lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2022-10-04 14:12:48 +02:00
parent 2a4d745265
commit 585ea681bd
9 changed files with 1368 additions and 958 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week39/html/week39-bs.html
+5 -4
View File
@@ -248,10 +248,11 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'same-code-but-now-with-momentum-gradient-descent'),
('Same problem but now with AdaGrad',
('Similar (second order function now) problem but now with '
'AdaGrad',
2,
None,
'same-problem-but-now-with-adagrad'),
'similar-second-order-function-now-problem-but-now-with-adagrad'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
@@ -373,7 +374,7 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._week39-bs079.html#but-noen-of-these-can-compete-with-newton-s-method" style="font-size: 80%;">But noen of these can compete with Newton's method</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs080.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs081.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs082.html#same-problem-but-now-with-adagrad" style="font-size: 80%;">Same problem but now with AdaGrad</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs082.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs083.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs084.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs085.html#weekend-challenge" style="font-size: 80%;">Weekend challenge</a></li>
@@ -406,7 +407,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 30, 2022</h4>
<h4>Oct 4, 2022</h4>
</center> <!-- date -->
<br>
doc/pub/week39/html/week39-reveal.html
+25 -31
View File
@@ -184,7 +184,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 30, 2022</h4>
<h4>Oct 4, 2022</h4>
</center> <!-- date -->
<br>
@@ -3486,7 +3486,7 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
</section>
<section>
<h2 id="same-problem-but-now-with-adagrad">Same problem but now with AdaGrad </h2>
<h2 id="similar-second-order-function-now-problem-but-now-with-adagrad">Similar (second order function now) problem but now with AdaGrad </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
@@ -3494,8 +3494,8 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #228B22"># OLS example with AdaGrad</span>
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
@@ -3506,53 +3506,45 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
n = <span style="color: #B452CD">10000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">100</span>
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = (<span style="color: #B452CD">1.0</span>/n)*training_gradient(y, X, theta)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
<span style="color: #658b00">print</span>(np.size(gradients))
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Including AdaGrad</span>
delta = <span style="color: #B452CD">0.000001</span>
r = [<span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">for</span> _ <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(gradients.shape[<span style="color: #B452CD">0</span>])]
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #228B22"># The outer product is calculated from scratch for each epoch</span>
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># calculate squared gradient by Hadamard multiplication</span>
r -= gradients*gradients
<span style="color: #228B22"># Calculate the outer product of the gradients</span>
Giter +=gradients @ gradients.T
<span style="color: #228B22"># Simpler algorithm with only diagonal elements</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
<span style="color: #228B22"># compute update</span>
update = <span style="color: #B452CD">1.0</span>/(delta+np.sqrt(r))*gradients
theta -= eta*update
update = np.multiply(Ginverse,gradients)
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
@@ -3569,6 +3561,8 @@ r = [<span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-w
</div>
</div>
</div>
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
</section>
<section>
doc/pub/week39/html/week39-solarized.html
+27 -33
View File
@@ -275,10 +275,11 @@ div.toc p,a {
2,
None,
'same-code-but-now-with-momentum-gradient-descent'),
('Same problem but now with AdaGrad',
('Similar (second order function now) problem but now with '
'AdaGrad',
2,
None,
'same-problem-but-now-with-adagrad'),
'similar-second-order-function-now-problem-but-now-with-adagrad'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
@@ -322,7 +323,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 30, 2022</h4>
<h4>Oct 4, 2022</h4>
</center> <!-- date -->
<br>
@@ -3413,7 +3414,7 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="same-problem-but-now-with-adagrad">Same problem but now with AdaGrad </h2>
<h2 id="similar-second-order-function-now-problem-but-now-with-adagrad">Similar (second order function now) problem but now with AdaGrad </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
@@ -3421,8 +3422,8 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #228B22"># OLS example with AdaGrad</span>
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
@@ -3433,53 +3434,45 @@ delta_momentum = <span style="color: #B452CD">0.3</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
n = <span style="color: #B452CD">10000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Hessian matrix</span>
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f&quot;Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}&quot;</span>)
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
Niterations = <span style="color: #B452CD">100</span>
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = (<span style="color: #B452CD">1.0</span>/n)*training_gradient(y, X, theta)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
<span style="color: #658b00">print</span>(np.size(gradients))
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Including AdaGrad</span>
delta = <span style="color: #B452CD">0.000001</span>
r = [<span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-weight: bold">for</span> _ <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(gradients.shape[<span style="color: #B452CD">0</span>])]
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #228B22"># The outer product is calculated from scratch for each epoch</span>
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># calculate squared gradient by Hadamard multiplication</span>
r -= gradients*gradients
<span style="color: #228B22"># Calculate the outer product of the gradients</span>
Giter +=gradients @ gradients.T
<span style="color: #228B22"># Simpler algorithm with only diagonal elements</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
<span style="color: #228B22"># compute update</span>
update = <span style="color: #B452CD">1.0</span>/(delta+np.sqrt(r))*gradients
theta -= eta*update
update = np.multiply(Ginverse,gradients)
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
@@ -3497,6 +3490,7 @@ r = [<span style="color: #B452CD">0.0</span> <span style="color: #8B008B; font-w
</div>
</div>
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
doc/pub/week39/html/week39.html
+27 -33
View File
@@ -352,10 +352,11 @@ div.toc p,a {
2,
None,
'same-code-but-now-with-momentum-gradient-descent'),
('Same problem but now with AdaGrad',
('Similar (second order function now) problem but now with '
'AdaGrad',
2,
None,
'same-problem-but-now-with-adagrad'),
'similar-second-order-function-now-problem-but-now-with-adagrad'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
@@ -399,7 +400,7 @@ MathJax.Hub.Config({
</center>
<br>
<center>
<h4>Sep 30, 2022</h4>
<h4>Oct 4, 2022</h4>
</center> <!-- date -->
<br>
@@ -3490,7 +3491,7 @@ delta_momentum <span style="color: #666666">=</span> <span style="color: #666666
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="same-problem-but-now-with-adagrad">Same problem but now with AdaGrad </h2>
<h2 id="similar-second-order-function-now-problem-but-now-with-adagrad">Similar (second order function now) problem but now with AdaGrad </h2>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="cell border-box-sizing code_cell rendered">
@@ -3498,8 +3499,8 @@ delta_momentum <span style="color: #666666">=</span> <span style="color: #666666
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Using Autograd to calculate gradients using SGD</span>
<span style="color: #408080; font-style: italic"># OLS example with AdaGrad</span>
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent</span>
<span style="color: #408080; font-style: italic"># OLS example</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
@@ -3510,53 +3511,45 @@ delta_momentum <span style="color: #666666">=</span> <span style="color: #666666
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostOLS</span>(y,X,theta):
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum((y<span style="color: #666666">-</span>X <span style="color: #666666">@</span> theta)<span style="color: #666666">**2</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">10000</span>
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+3*</span>x <span style="color: #666666">+4*</span>x<span style="color: #666666">*</span>x<span style="color: #408080; font-style: italic"># +np.random.randn(n,1)</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x]
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x, x<span style="color: #666666">*</span>x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(XT_X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Own inversion&quot;</span>)
<span style="color: #008000">print</span>(theta_linreg)
<span style="color: #408080; font-style: italic"># Hessian matrix</span>
H <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>n)<span style="color: #666666">*</span> XT_X
EigValues, EigVectors <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>eig(H)
<span style="color: #008000">print</span>(<span style="color: #BA2121">f&quot;Eigenvalues of Hessian Matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>EigValues<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">&quot;</span>)
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
eta <span style="color: #666666">=</span> <span style="color: #666666">1.0/</span>np<span style="color: #666666">.</span>max(EigValues)
Niterations <span style="color: #666666">=</span> <span style="color: #666666">100</span>
<span style="color: #408080; font-style: italic"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient <span style="color: #666666">=</span> grad(CostOLS,<span style="color: #666666">2</span>)
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
gradients <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>n)<span style="color: #666666">*</span>training_gradient(y, X, theta)
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own gd&quot;</span>)
<span style="color: #008000">print</span>(theta)
<span style="color: #008000">print</span>(np<span style="color: #666666">.</span>size(gradients))
<span style="color: #408080; font-style: italic"># Define parameters for Stochastic Gradient Descent</span>
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
<span style="color: #408080; font-style: italic"># Guess for unknown parameters theta</span>
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">3</span>,<span style="color: #666666">1</span>)
<span style="color: #408080; font-style: italic"># Including AdaGrad</span>
delta <span style="color: #666666">=</span> <span style="color: #666666">0.000001</span>
r <span style="color: #666666">=</span> [<span style="color: #666666">0.0</span> <span style="color: #008000; font-weight: bold">for</span> _ <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(gradients<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>])]
<span style="color: #408080; font-style: italic"># Value for learning rate</span>
eta <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
<span style="color: #408080; font-style: italic"># Including AdaGrad parameter to avoid possible division by zero</span>
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span>
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
<span style="color: #408080; font-style: italic"># The outer product is calculated from scratch for each epoch</span>
Giter <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(shape<span style="color: #666666">=</span>(<span style="color: #666666">3</span>,<span style="color: #666666">3</span>))
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> M<span style="color: #666666">*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>M)<span style="color: #666666">*</span>training_gradient(yi, xi, theta)
<span style="color: #408080; font-style: italic"># calculate squared gradient by Hadamard multiplication</span>
r <span style="color: #666666">-=</span> gradients<span style="color: #666666">*</span>gradients
<span style="color: #408080; font-style: italic"># Calculate the outer product of the gradients</span>
Giter <span style="color: #666666">+=</span>gradients <span style="color: #666666">@</span> gradients<span style="color: #666666">.</span>T
<span style="color: #408080; font-style: italic"># Simpler algorithm with only diagonal elements</span>
Ginverse <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[eta<span style="color: #666666">/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(np<span style="color: #666666">.</span>diagonal(Giter)))]
<span style="color: #408080; font-style: italic"># compute update</span>
update <span style="color: #666666">=</span> <span style="color: #666666">1.0/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(r))<span style="color: #666666">*</span>gradients
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>update
update <span style="color: #666666">=</span> np<span style="color: #666666">.</span>multiply(Ginverse,gradients)
theta <span style="color: #666666">-=</span> update
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own AdaGrad&quot;</span>)
<span style="color: #008000">print</span>(theta)
</pre>
@@ -3574,6 +3567,7 @@ r <span style="color: #666666">=</span> [<span style="color: #666666">0.0</span>
</div>
</div>
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week39/ipynb/week39.ipynb
+1191 -787
View File
File diff suppressed because one or more lines are too long
doc/src/week39/adagrad.py
+15 -40
View File
@@ -4,53 +4,28 @@ from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
y = 4*x+3*x*x
# Setting up Design matrix
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
XTX = X.T @ X
XTy = X.T @ y
theta_linreg = np.linalg.pinv(XTX) @ (XTy)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
gradients = np.zeros(theta.shape)
r = gradients*gradients
print(r.shape)
print(gradients.shape)
# Value for learning rate
beta = np.random.randn(3,1)
eta = 0.01
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# calculate squared gradient by Hadamard multiplication
# r += (gradients*gradients)
r = np.sum(gradients*gradients)
# compute update
update = 1.0/(delta+np.sqrt(r))*gradients
theta -= eta*update
print("theta from own AdaGrad")
print(theta)
delta = 1e-8
Niterations = 10000
Giter = np.zeros(shape=(3,3))
for iter in range(Niterations):
gradient = (2.0/n)*(XTX @ beta - XTy)
Giter +=gradient @ gradient.T
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
beta -= np.multiply(Ginverse,gradient)
print("Optimal parameters with AdaGrad",beta)
doc/src/week39/adagradSGD.py
+55
View File
@@ -0,0 +1,55 @@
# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Value for learning rate
eta = 0.01
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
Giter +=gradients @ gradients.T
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# calculate squared gradient by Hadamard multiplication
# r += (gradients*gradients)
# r = np.sum(gradients*gradients)
# compute update
update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own AdaGrad")
print(theta)
doc/src/week39/week39.do.txt
+23 -30
View File
@@ -2432,10 +2432,10 @@ print(theta)
!split
===== Same problem but now with AdaGrad =====
===== Similar (second order function now) problem but now with AdaGrad =====
!bc pycod
# Using Autograd to calculate gradients using SGD
# OLS example with AdaGrad
# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
@@ -2446,57 +2446,50 @@ from autograd import grad
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 100
x = 2*np.random.rand(n,1)
y = 4+3*x+np.random.randn(n,1)
n = 10000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x]
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Hessian matrix
H = (2.0/n)* XT_X
EigValues, EigVectors = np.linalg.eig(H)
print(f"Eigenvalues of Hessian Matrix:{EigValues}")
theta = np.random.randn(2,1)
eta = 1.0/np.max(EigValues)
Niterations = 100
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
for iter in range(Niterations):
gradients = (1.0/n)*training_gradient(y, X, theta)
theta -= eta*gradients
print("theta from own gd")
print(theta)
print(np.size(gradients))
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
theta = np.random.randn(2,1)
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Including AdaGrad
delta = 0.000001
r = [0.0 for _ in range(gradients.shape[0])]
# Value for learning rate
eta = 0.01
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
# The outer product is calculated from scratch for each epoch
Giter = np.zeros(shape=(3,3))
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# calculate squared gradient by Hadamard multiplication
r -= gradients*gradients
# Calculate the outer product of the gradients
Giter +=gradients @ gradients.T
# Simpler algorithm with only diagonal elements
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# compute update
update = 1.0/(delta+np.sqrt(r))*gradients
theta -= eta*update
update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own AdaGrad")
print(theta)
!ec
Running this code we note an almost perfect agreement with the results from matrix inversion.
!split
===== And Logistic Regression =====