lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2021-10-07 08:56:03 +02:00
parent afdd0edde7
commit 0dab750863
9 changed files with 57 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week40/html/._week40-bs009.html
+1 -1
View File
@@ -315,7 +315,7 @@ MathJax.Hub.Config({
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each mini-batche</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">10</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
doc/pub/week40/html/._week40-bs011.html
+3
View File
@@ -352,6 +352,9 @@ j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;gamma_j after </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> epochs: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> (n_epochs,gamma_j))
</pre></div>
<p>
We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters \( t_0 \) and \( t_1 \).
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
doc/pub/week40/html/._week40-bs012.html
+6 -6
View File
@@ -319,9 +319,9 @@ MathJax.Hub.Config({
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
@@ -339,7 +339,7 @@ Niterations <span style="color: #666666">=</span> <span style="color: #666666">1
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>n<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own gd&quot;</span>)
<span style="color: #008000">print</span>(theta)
@@ -351,7 +351,7 @@ ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666"
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
M <span style="color: #666666">=</span> <span style="color: #666666">10</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</span>, <span style="color: #666666">50</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">learning_schedule</span>(t):
@@ -364,7 +364,7 @@ theta <span style="color: #666666">=</span> np<span style="color: #666666">.</sp
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+1</span>]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
gradients <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>m) <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own sdg&quot;</span>)
doc/pub/week40/html/week40-reveal.html
+9 -7
View File
@@ -330,7 +330,7 @@ the number of minibatches, as exemplified in the code below.
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each mini-batche</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
n_epochs = <span style="color: #B452CD">10</span> <span style="color: #228B22">#number of epochs</span>
@@ -418,6 +418,8 @@ j = <span style="color: #B452CD">0</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;gamma_j after %d epochs: %g&quot;</span> % (n_epochs,gamma_j))
</pre></div>
<p>
We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters \( t_0 \) and \( t_1 \).
</section>
@@ -434,9 +436,9 @@ j = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
m = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
@@ -454,7 +456,7 @@ Niterations = <span style="color: #B452CD">1000</span>
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = <span style="color: #B452CD">2.0</span>/m*X.T @ ((X @ theta)-y)
gradients = <span style="color: #B452CD">2.0</span>/n*X.T @ ((X @ theta)-y)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
@@ -466,7 +468,7 @@ ypredict2 = Xnew.dot(theta_linreg)
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">10</span> <span style="color: #228B22">#size of each minibatch</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
@@ -479,7 +481,7 @@ theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color
random_index = np.random.randint(m)
xi = X[random_index:random_index+<span style="color: #B452CD">1</span>]
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
gradients = <span style="color: #B452CD">2</span> * xi.T @ ((xi @ theta)-yi)
gradients = (<span style="color: #B452CD">2.0</span>/m) * xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
theta = theta - eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own sdg&quot;</span>)
doc/pub/week40/html/week40-solarized.html
+10 -7
View File
@@ -421,7 +421,7 @@ the number of minibatches, as exemplified in the code below.
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each mini-batche</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
n_epochs = <span style="color: #B452CD">10</span> <span style="color: #228B22">#number of epochs</span>
@@ -507,6 +507,9 @@ j = <span style="color: #B452CD">0</span>
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;gamma_j after %d epochs: %g&quot;</span> % (n_epochs,gamma_j))
</pre></div>
<p>
We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters \( t_0 \) and \( t_1 \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
@@ -522,9 +525,9 @@ j = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
m = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
@@ -542,7 +545,7 @@ Niterations = <span style="color: #B452CD">1000</span>
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = <span style="color: #B452CD">2.0</span>/m*X.T @ ((X @ theta)-y)
gradients = <span style="color: #B452CD">2.0</span>/n*X.T @ ((X @ theta)-y)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
@@ -554,7 +557,7 @@ ypredict2 = Xnew.dot(theta_linreg)
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">10</span> <span style="color: #228B22">#size of each minibatch</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
@@ -567,7 +570,7 @@ theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color
random_index = np.random.randint(m)
xi = X[random_index:random_index+<span style="color: #B452CD">1</span>]
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
gradients = <span style="color: #B452CD">2</span> * xi.T @ ((xi @ theta)-yi)
gradients = (<span style="color: #B452CD">2.0</span>/m) * xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
theta = theta - eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own sdg&quot;</span>)
doc/pub/week40/html/week40.html
+10 -7
View File
@@ -426,7 +426,7 @@ the number of minibatches, as exemplified in the code below.
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each mini-batche</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">10</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
@@ -512,6 +512,9 @@ j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;gamma_j after </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> epochs: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">&quot;</span> <span style="color: #666666">%</span> (n_epochs,gamma_j))
</pre></div>
<p>
We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters \( t_0 \) and \( t_1 \).
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
@@ -527,9 +530,9 @@ j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
@@ -547,7 +550,7 @@ Niterations <span style="color: #666666">=</span> <span style="color: #666666">1
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>n<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own gd&quot;</span>)
<span style="color: #008000">print</span>(theta)
@@ -559,7 +562,7 @@ ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666"
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
M <span style="color: #666666">=</span> <span style="color: #666666">10</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</span>, <span style="color: #666666">50</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">learning_schedule</span>(t):
@@ -572,7 +575,7 @@ theta <span style="color: #666666">=</span> np<span style="color: #666666">.</sp
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+1</span>]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
gradients <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>m) <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own sdg&quot;</span>)
doc/pub/week40/ipynb/ipynb-week40-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week40/ipynb/week40.ipynb
+10 -7
View File
@@ -195,7 +195,7 @@
"import numpy as np \n",
"\n",
"n = 100 #100 datapoints \n",
"M = 5 #size of each minibatch\n",
"M = 5 #size of each mini-batche\n",
"m = int(n/M) #number of minibatches\n",
"n_epochs = 10 #number of epochs\n",
"\n",
@@ -287,6 +287,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters $t_0$ and $t_1$.\n",
"\n",
"\n",
"## Program for stochastic gradient"
]
},
@@ -308,9 +311,9 @@
"import matplotlib.pyplot as plt\n",
"from sklearn.linear_model import SGDRegressor\n",
"\n",
"m = 100\n",
"x = 2*np.random.rand(m,1)\n",
"y = 4+3*x+np.random.randn(m,1)\n",
"n = 100\n",
"x = 2*np.random.rand(n,1)\n",
"y = 4+3*x+np.random.randn(n,1)\n",
"\n",
"X = np.c_[np.ones((m,1)), x]\n",
"theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n",
@@ -328,7 +331,7 @@
"\n",
"\n",
"for iter in range(Niterations):\n",
" gradients = 2.0/m*X.T @ ((X @ theta)-y)\n",
" gradients = 2.0/n*X.T @ ((X @ theta)-y)\n",
" theta -= eta*gradients\n",
"print(\"theta from own gd\")\n",
"print(theta)\n",
@@ -340,7 +343,7 @@
"\n",
"\n",
"n_epochs = 50\n",
"M = 10 #size of each minibatch\n",
"M = 5 #size of each minibatch\n",
"m = int(n/M) #number of minibatches\n",
"t0, t1 = 5, 50\n",
"def learning_schedule(t):\n",
@@ -353,7 +356,7 @@
" random_index = np.random.randint(m)\n",
" xi = X[random_index:random_index+1]\n",
" yi = y[random_index:random_index+1]\n",
" gradients = 2 * xi.T @ ((xi @ theta)-yi)\n",
" gradients = (2.0/m) * xi.T @ ((xi @ theta)-yi)\n",
" eta = learning_schedule(epoch*m+i)\n",
" theta = theta - eta*gradients\n",
"print(\"theta from own sdg\")\n",
doc/src/week40/week40.do.txt
+8 -8
View File
@@ -144,7 +144,7 @@ the number of minibatches, as exemplified in the code below.
import numpy as np
n = 100 #100 datapoints
M = 5 #size of each minibatch
M = 5 #size of each mini-batche
m = int(n/M) #number of minibatches
n_epochs = 10 #number of epochs
@@ -222,7 +222,7 @@ print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))
!ec
We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters $t_0$ and $t_1$.
!split
@@ -236,9 +236,9 @@ import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import SGDRegressor
m = 100
x = 2*np.random.rand(m,1)
y = 4+3*x+np.random.randn(m,1)
n = 100
x = 2*np.random.rand(n,1)
y = 4+3*x+np.random.randn(n,1)
X = np.c_[np.ones((m,1)), x]
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
@@ -256,7 +256,7 @@ Niterations = 1000
for iter in range(Niterations):
gradients = 2.0/m*X.T @ ((X @ theta)-y)
gradients = 2.0/n*X.T @ ((X @ theta)-y)
theta -= eta*gradients
print("theta from own gd")
print(theta)
@@ -268,7 +268,7 @@ ypredict2 = Xnew.dot(theta_linreg)
n_epochs = 50
M = 10 #size of each minibatch
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
t0, t1 = 5, 50
def learning_schedule(t):
@@ -281,7 +281,7 @@ for epoch in range(n_epochs):
random_index = np.random.randint(m)
xi = X[random_index:random_index+1]
yi = y[random_index:random_index+1]
gradients = 2 * xi.T @ ((xi @ theta)-yi)
gradients = (2.0/m) * xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
theta = theta - eta*gradients
print("theta from own sdg")