lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update on week 40

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2021-11-03 07:51:06 +01:00
parent 45305909b3
commit 15b4a4ad26
9 changed files with 306 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week40/html/._week40-bs000.html
+1 -1
View File
@@ -325,7 +325,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Nov 2, 2021</h4>
<h4>Nov 3, 2021</h4>
</center> <!-- date -->
<br>
doc/pub/week40/html/._week40-bs012.html
+15 -12
View File
@@ -317,28 +317,28 @@ MathJax.Hub.Config({
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Importing various packages</span>
<span style="color: #408080; font-style: italic"># Importing various packages</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
n <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Own inversion&quot;</span>)
<span style="color: #008000">print</span>(theta_linreg)
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, eta0<span style="color: #666666">=0.1</span>)
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;sgdreg from scikit&quot;</span>)
<span style="color: #008000">print</span>(sgdreg<span style="color: #666666">.</span>intercept_, sgdreg<span style="color: #666666">.</span>coef_)
<span style="color: #408080; font-style: italic"># Hessian matrix</span>
H <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>n)<span style="color: #666666">*</span> XT_X
EigValues, EigVectors <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>eig(H)
<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;Eigenvalues of Hessian Matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>EigValues<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
eta <span style="color: #666666">=</span> <span style="color: #666666">1.0/</span>np<span style="color: #666666">.</span>max(EigValues)
Niterations <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
@@ -353,7 +353,6 @@ Xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</spa
ypredict <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta)
ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta_linreg)
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
@@ -364,16 +363,20 @@ t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</spa
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
<span style="color: #408080; font-style: italic"># Can you figure out a better way of setting up the contributions to each batch?</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>m]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>m]
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
xi <span style="color: #666666">=</span> X[random_index<span style="color: #666666">*</span>M:random_index<span style="color: #666666">*</span>M<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index<span style="color: #666666">*</span>M:random_index<span style="color: #666666">*</span>M<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>M)<span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own sdg&quot;</span>)
<span style="color: #008000">print</span>(theta)
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">&quot;r-&quot;</span>)
plt<span style="color: #666666">.</span>plot(xnew, ypredict2, <span style="color: #BA2121">&quot;b-&quot;</span>)
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">&#39;ro&#39;</span>)
doc/pub/week40/html/week40-bs.html
+1 -1
View File
@@ -325,7 +325,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Nov 2, 2021</h4>
<h4>Nov 3, 2021</h4>
</center> <!-- date -->
<br>
doc/pub/week40/html/week40-reveal.html
+16 -13
View File
@@ -184,7 +184,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Nov 2, 2021</h4>
<h4>Nov 3, 2021</h4>
</center> <!-- date -->
<br>
@@ -499,28 +499,28 @@ j = <span style="color: #B452CD">0</span>
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Importing various packages</span>
<span style="color: #228B22"># Importing various packages</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> exp, sqrt
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
n = <span style="color: #B452CD">1000</span>
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
sgdreg.fit(x,y.ravel())
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;sgdreg from scikit&quot;</span>)
<span style="color: #658b00">print</span>(sgdreg.intercept_, sgdreg.coef_)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">0.1</span>
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">1000</span>
@@ -535,7 +535,6 @@ Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color:
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
@@ -546,16 +545,20 @@ t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50<
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #228B22"># Can you figure out a better way of setting up the contributions to each batch?</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = np.random.randint(m)
xi = X[random_index:random_index+m]
yi = y[random_index:random_index+m]
gradients = <span style="color: #B452CD">2.0</span>* xi.T @ ((xi @ theta)-yi)
xi = X[random_index*M:random_index*M+M]
yi = y[random_index*M:random_index*M+M]
gradients = (<span style="color: #B452CD">2.0</span>/M)* xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
theta = theta - eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own sdg&quot;</span>)
<span style="color: #658b00">print</span>(theta)
plt.plot(xnew, ypredict, <span style="color: #CD5555">&quot;r-&quot;</span>)
plt.plot(xnew, ypredict2, <span style="color: #CD5555">&quot;b-&quot;</span>)
plt.plot(x, y ,<span style="color: #CD5555">&#39;ro&#39;</span>)
doc/pub/week40/html/week40-solarized.html
+16 -13
View File
@@ -261,7 +261,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Nov 2, 2021</h4>
<h4>Nov 3, 2021</h4>
</center> <!-- date -->
<br>
@@ -546,28 +546,28 @@ j = <span style="color: #B452CD">0</span>
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="line-height: 125%;"><span style="color: #228B22"># Importing various packages</span>
<span style="color: #228B22"># Importing various packages</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> exp, sqrt
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
n = <span style="color: #B452CD">1000</span>
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
sgdreg.fit(x,y.ravel())
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;sgdreg from scikit&quot;</span>)
<span style="color: #658b00">print</span>(sgdreg.intercept_, sgdreg.coef_)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">0.1</span>
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">1000</span>
@@ -582,7 +582,6 @@ Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color:
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
@@ -593,16 +592,20 @@ t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50<
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #228B22"># Can you figure out a better way of setting up the contributions to each batch?</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = np.random.randint(m)
xi = X[random_index:random_index+m]
yi = y[random_index:random_index+m]
gradients = <span style="color: #B452CD">2.0</span>* xi.T @ ((xi @ theta)-yi)
xi = X[random_index*M:random_index*M+M]
yi = y[random_index*M:random_index*M+M]
gradients = (<span style="color: #B452CD">2.0</span>/M)* xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
theta = theta - eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own sdg&quot;</span>)
<span style="color: #658b00">print</span>(theta)
plt.plot(xnew, ypredict, <span style="color: #CD5555">&quot;r-&quot;</span>)
plt.plot(xnew, ypredict2, <span style="color: #CD5555">&quot;b-&quot;</span>)
plt.plot(x, y ,<span style="color: #CD5555">&#39;ro&#39;</span>)
doc/pub/week40/html/week40.html
+16 -13
View File
@@ -338,7 +338,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Nov 2, 2021</h4>
<h4>Nov 3, 2021</h4>
</center> <!-- date -->
<br>
@@ -623,28 +623,28 @@ j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Importing various packages</span>
<span style="color: #408080; font-style: italic"># Importing various packages</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
n <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Own inversion&quot;</span>)
<span style="color: #008000">print</span>(theta_linreg)
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, eta0<span style="color: #666666">=0.1</span>)
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;sgdreg from scikit&quot;</span>)
<span style="color: #008000">print</span>(sgdreg<span style="color: #666666">.</span>intercept_, sgdreg<span style="color: #666666">.</span>coef_)
<span style="color: #408080; font-style: italic"># Hessian matrix</span>
H <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>n)<span style="color: #666666">*</span> XT_X
EigValues, EigVectors <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>eig(H)
<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;Eigenvalues of Hessian Matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>EigValues<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
eta <span style="color: #666666">=</span> <span style="color: #666666">1.0/</span>np<span style="color: #666666">.</span>max(EigValues)
Niterations <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
@@ -659,7 +659,6 @@ Xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</spa
ypredict <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta)
ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta_linreg)
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
@@ -670,16 +669,20 @@ t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</spa
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
<span style="color: #408080; font-style: italic"># Can you figure out a better way of setting up the contributions to each batch?</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>m]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>m]
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
xi <span style="color: #666666">=</span> X[random_index<span style="color: #666666">*</span>M:random_index<span style="color: #666666">*</span>M<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index<span style="color: #666666">*</span>M:random_index<span style="color: #666666">*</span>M<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>M)<span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own sdg&quot;</span>)
<span style="color: #008000">print</span>(theta)
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">&quot;r-&quot;</span>)
plt<span style="color: #666666">.</span>plot(xnew, ypredict2, <span style="color: #BA2121">&quot;b-&quot;</span>)
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">&#39;ro&#39;</span>)
doc/pub/week40/ipynb/ipynb-week40-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week40/ipynb/week40.ipynb
+226 -223
View File
File diff suppressed because it is too large Load Diff
doc/src/week40/week40.do.txt
+15 -12
View File
@@ -231,28 +231,28 @@ We note that we have defined several hyperparameters. These are now the number o
!bc pycod
# Importing various packages
# Importing various packages
from math import exp, sqrt
from random import random, seed
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import SGDRegressor
n = 1000
n = 100
x = 2*np.random.rand(n,1)
y = 4+3*x+np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x]
XT_X = X.T @ X
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)
sgdreg.fit(x,y.ravel())
print("sgdreg from scikit")
print(sgdreg.intercept_, sgdreg.coef_)
# Hessian matrix
H = (2.0/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
print(f"Eigenvalues of Hessian Matrix:{EigValues}")
theta = np.random.randn(2,1)
eta = 0.1
eta = 1.0/np.max(EigValues)
Niterations = 1000
@@ -267,7 +267,6 @@ Xnew = np.c_[np.ones((2,1)), xnew]
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
@@ -278,16 +277,20 @@ def learning_schedule(t):
theta = np.random.randn(2,1)
for epoch in range(n_epochs):
# Can you figure out a better way of setting up the contributions to each batch?
for i in range(m):
random_index = np.random.randint(m)
xi = X[random_index:random_index+m]
yi = y[random_index:random_index+m]
gradients = 2.0* xi.T @ ((xi @ theta)-yi)
xi = X[random_index*M:random_index*M+M]
yi = y[random_index*M:random_index*M+M]
gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
theta = theta - eta*gradients
print("theta from own sdg")
print(theta)
plt.plot(xnew, ypredict, "r-")
plt.plot(xnew, ypredict2, "b-")
plt.plot(x, y ,'ro')