lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added adam, had forgotten and updated RMSprop

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2022-10-31 16:26:24 +01:00
parent 9950370561
commit 9bb691b916
9 changed files with 768 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week39/html/week39-bs.html
+8 -5
View File
@@ -262,12 +262,15 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
2,
None,
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
None,
'introducing-jax-https-jax-readthedocs-io-en-latest'),
('Weekend challenge', 2, None, 'weekend-challenge')]}
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
end of tocinfo -->
<body>
@@ -386,9 +389,9 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._week39-bs082.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs083.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs084.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs085.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs086.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs087.html#weekend-challenge" style="font-size: 80%;">Weekend challenge</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs085.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;">And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs086.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs087.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
</ul>
</li>
doc/pub/week39/html/week39-reveal.html
+86 -15
View File
@@ -3639,16 +3639,12 @@ delta = <span style="color: #B452CD">1e-8</span>
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Previous value for the outer product of gradients</span>
Previous = Giter
<span style="color: #228B22"># Accumulated gradient</span>
<span style="color: #228B22"># Giter +=gradients @ gradients.T</span>
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
Giter = (rho*Giter+(<span style="color: #B452CD">1</span>-rho)*gradients*gradients)
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
<span style="color: #228B22"># Hadamard product</span>
update = np.multiply(Ginverse,gradients)
update = Ginverse*gradients
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #658b00">print</span>(theta)
@@ -3668,6 +3664,91 @@ delta = <span style="color: #B452CD">1e-8</span>
</div>
</section>
<section>
<h2 id="and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf">And finally <a href="https://arxiv.org/pdf/1412.6980.pdf" target="_blank">ADAM</a> </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980</span>
beta1 = <span style="color: #B452CD">0.9</span>
beta2 = <span style="color: #B452CD">0.999</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-7</span>
<span style="color: #658b00">iter</span> = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
first_moment = <span style="color: #B452CD">0.0</span>
second_moment = <span style="color: #B452CD">0.0</span>
<span style="color: #658b00">iter</span> += <span style="color: #B452CD">1</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Computing moments first</span>
first_moment = beta1*first_moment + (<span style="color: #B452CD">1</span>-beta1)*gradients
second_moment = beta2*second_moment+(<span style="color: #B452CD">1</span>-beta2)*gradients*gradients
first_term = first_moment/(<span style="color: #B452CD">1.0</span>-beta1**<span style="color: #658b00">iter</span>)
second_term = second_moment/(<span style="color: #B452CD">1.0</span>-beta2**<span style="color: #658b00">iter</span>)
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
update = eta*first_term/(np.sqrt(second_term)+delta)
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own ADAM&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
@@ -3771,16 +3852,6 @@ derivative_fn = grad(sum_logistic)
</div>
</section>
<section>
<h2 id="weekend-challenge">Weekend challenge </h2>
<ul>
<p><li> Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms.</li>
<p><li> Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods</li>
<p><li> Extend from linear regression to logistic regression.</li>
</ul>
</section>
</div> <!-- class="slides" -->
doc/pub/week39/html/week39-solarized.html
+91 -15
View File
@@ -289,12 +289,15 @@ div.toc p,a {
2,
None,
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
2,
None,
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
None,
'introducing-jax-https-jax-readthedocs-io-en-latest'),
('Weekend challenge', 2, None, 'weekend-challenge')]}
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
end of tocinfo -->
<body>
@@ -3572,16 +3575,12 @@ delta = <span style="color: #B452CD">1e-8</span>
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Previous value for the outer product of gradients</span>
Previous = Giter
<span style="color: #228B22"># Accumulated gradient</span>
<span style="color: #228B22"># Giter +=gradients @ gradients.T</span>
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
Giter = (rho*Giter+(<span style="color: #B452CD">1</span>-rho)*gradients*gradients)
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
<span style="color: #228B22"># Hadamard product</span>
update = np.multiply(Ginverse,gradients)
update = Ginverse*gradients
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #658b00">print</span>(theta)
@@ -3601,6 +3600,91 @@ delta = <span style="color: #B452CD">1e-8</span>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf">And finally <a href="https://arxiv.org/pdf/1412.6980.pdf" target="_blank">ADAM</a> </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">1000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980</span>
beta1 = <span style="color: #B452CD">0.9</span>
beta2 = <span style="color: #B452CD">0.999</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-7</span>
<span style="color: #658b00">iter</span> = <span style="color: #B452CD">0</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
first_moment = <span style="color: #B452CD">0.0</span>
second_moment = <span style="color: #B452CD">0.0</span>
<span style="color: #658b00">iter</span> += <span style="color: #B452CD">1</span>
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Computing moments first</span>
first_moment = beta1*first_moment + (<span style="color: #B452CD">1</span>-beta1)*gradients
second_moment = beta2*second_moment+(<span style="color: #B452CD">1</span>-beta2)*gradients*gradients
first_term = first_moment/(<span style="color: #B452CD">1.0</span>-beta1**<span style="color: #658b00">iter</span>)
second_term = second_moment/(<span style="color: #B452CD">1.0</span>-beta2**<span style="color: #658b00">iter</span>)
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
update = eta*first_term/(np.sqrt(second_term)+delta)
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own ADAM&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
@@ -3704,14 +3788,6 @@ derivative_fn = grad(sum_logistic)
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="weekend-challenge">Weekend challenge </h2>
<ul>
<li> Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms.</li>
<li> Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods</li>
<li> Extend from linear regression to logistic regression.</li>
</ul>
<!-- ------------------- end of main content --------------- -->
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
doc/pub/week39/html/week39.html
+91 -15
View File
@@ -366,12 +366,15 @@ div.toc p,a {
2,
None,
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
2,
None,
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
None,
'introducing-jax-https-jax-readthedocs-io-en-latest'),
('Weekend challenge', 2, None, 'weekend-challenge')]}
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
end of tocinfo -->
<body>
@@ -3649,16 +3652,12 @@ delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>M)<span style="color: #666666">*</span>training_gradient(yi, xi, theta)
<span style="color: #408080; font-style: italic"># Previous value for the outer product of gradients</span>
Previous <span style="color: #666666">=</span> Giter
<span style="color: #408080; font-style: italic"># Accumulated gradient</span>
<span style="color: #408080; font-style: italic"># Giter +=gradients @ gradients.T</span>
<span style="color: #408080; font-style: italic"># Scaling with rho the new and the previous results</span>
Giter <span style="color: #666666">=</span> (rho<span style="color: #666666">*</span>Giter<span style="color: #666666">+</span>(<span style="color: #666666">1-</span>rho)<span style="color: #666666">*</span>gradients<span style="color: #666666">*</span>gradients)
<span style="color: #408080; font-style: italic"># Taking the diagonal only and inverting</span>
Ginverse <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[eta<span style="color: #666666">/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(np<span style="color: #666666">.</span>diagonal(Giter)))]
<span style="color: #408080; font-style: italic"># Hadamard product</span>
update <span style="color: #666666">=</span> np<span style="color: #666666">.</span>multiply(Ginverse,gradients)
update <span style="color: #666666">=</span> Ginverse<span style="color: #666666">*</span>gradients
theta <span style="color: #666666">-=</span> update
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #008000">print</span>(theta)
@@ -3678,6 +3677,91 @@ delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf">And finally <a href="https://arxiv.org/pdf/1412.6980.pdf" target="_blank">ADAM</a> </h2>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #408080; font-style: italic"># OLS example</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> grad
<span style="color: #408080; font-style: italic"># Note change from previous example</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostOLS</span>(y,X,theta):
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum((y<span style="color: #666666">-</span>X <span style="color: #666666">@</span> theta)<span style="color: #666666">**2</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+3*</span>x <span style="color: #666666">+4*</span>x<span style="color: #666666">*</span>x<span style="color: #408080; font-style: italic"># +np.random.randn(n,1)</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x, x<span style="color: #666666">*</span>x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(XT_X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Own inversion&quot;</span>)
<span style="color: #008000">print</span>(theta_linreg)
<span style="color: #408080; font-style: italic"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient <span style="color: #666666">=</span> grad(CostOLS,<span style="color: #666666">2</span>)
<span style="color: #408080; font-style: italic"># Define parameters for Stochastic Gradient Descent</span>
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
<span style="color: #408080; font-style: italic"># Guess for unknown parameters theta</span>
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">3</span>,<span style="color: #666666">1</span>)
<span style="color: #408080; font-style: italic"># Value for learning rate</span>
eta <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
<span style="color: #408080; font-style: italic"># Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980</span>
beta1 <span style="color: #666666">=</span> <span style="color: #666666">0.9</span>
beta2 <span style="color: #666666">=</span> <span style="color: #666666">0.999</span>
<span style="color: #408080; font-style: italic"># Including AdaGrad parameter to avoid possible division by zero</span>
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-7</span>
<span style="color: #008000">iter</span> <span style="color: #666666">=</span> <span style="color: #666666">0</span>
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
first_moment <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
second_moment <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
<span style="color: #008000">iter</span> <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> M<span style="color: #666666">*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>M)<span style="color: #666666">*</span>training_gradient(yi, xi, theta)
<span style="color: #408080; font-style: italic"># Computing moments first</span>
first_moment <span style="color: #666666">=</span> beta1<span style="color: #666666">*</span>first_moment <span style="color: #666666">+</span> (<span style="color: #666666">1-</span>beta1)<span style="color: #666666">*</span>gradients
second_moment <span style="color: #666666">=</span> beta2<span style="color: #666666">*</span>second_moment<span style="color: #666666">+</span>(<span style="color: #666666">1-</span>beta2)<span style="color: #666666">*</span>gradients<span style="color: #666666">*</span>gradients
first_term <span style="color: #666666">=</span> first_moment<span style="color: #666666">/</span>(<span style="color: #666666">1.0-</span>beta1<span style="color: #666666">**</span><span style="color: #008000">iter</span>)
second_term <span style="color: #666666">=</span> second_moment<span style="color: #666666">/</span>(<span style="color: #666666">1.0-</span>beta2<span style="color: #666666">**</span><span style="color: #008000">iter</span>)
<span style="color: #408080; font-style: italic"># Scaling with rho the new and the previous results</span>
update <span style="color: #666666">=</span> eta<span style="color: #666666">*</span>first_term<span style="color: #666666">/</span>(np<span style="color: #666666">.</span>sqrt(second_term)<span style="color: #666666">+</span>delta)
theta <span style="color: #666666">-=</span> update
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own ADAM&quot;</span>)
<span style="color: #008000">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
@@ -3781,14 +3865,6 @@ derivative_fn <span style="color: #666666">=</span> grad(sum_logistic)
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="weekend-challenge">Weekend challenge </h2>
<ul>
<li> Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms.</li>
<li> Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods</li>
<li> Extend from linear regression to logistic regression.</li>
</ul>
<!-- ------------------- end of main content --------------- -->
<center style="font-size:80%">
<!-- copyright --> &copy; 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week39/ipynb/week39.ipynb
+365 -304
View File
File diff suppressed because it is too large Load Diff
doc/src/week39/adam.py
+59
View File
@@ -0,0 +1,59 @@
# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 1000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Value for learning rate
eta = 0.01
# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
beta1 = 0.9
beta2 = 0.999
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-7
iter = 0
for epoch in range(n_epochs):
first_moment = 0.0
second_moment = 0.0
iter += 1
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# Computing moments first
first_moment = beta1*first_moment + (1-beta1)*gradients
second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
first_term = first_moment/(1.0-beta1**iter)
second_term = second_moment/(1.0-beta2**iter)
# Scaling with rho the new and the previous results
update = eta*first_term/(np.sqrt(second_term)+delta)
theta -= update
print("theta from own ADAM")
print(theta)
doc/src/week39/rmsprop.py
+2 -4
View File
@@ -43,16 +43,14 @@ for epoch in range(n_epochs):
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# Previous value for the outer product of gradients
Previous = Giter
# Accumulated gradient
# Giter +=gradients @ gradients.T
# Scaling with rho the new and the previous results
Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# Hadamard product
update = np.multiply(Ginverse,gradients)
update = Ginverse*gradients
# update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own RMSprop")
print(theta)
doc/src/week39/week39.do.txt
+66 -12
View File
@@ -2550,21 +2550,82 @@ for epoch in range(n_epochs):
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# Previous value for the outer product of gradients
Previous = Giter
# Accumulated gradient
# Giter +=gradients @ gradients.T
# Scaling with rho the new and the previous results
Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# Hadamard product
update = np.multiply(Ginverse,gradients)
update = Ginverse*gradients
theta -= update
print("theta from own RMSprop")
print(theta)
!ec
!split
===== And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf" =====
!bc pycod
# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 1000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Value for learning rate
eta = 0.01
# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
beta1 = 0.9
beta2 = 0.999
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-7
iter = 0
for epoch in range(n_epochs):
first_moment = 0.0
second_moment = 0.0
iter += 1
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# Computing moments first
first_moment = beta1*first_moment + (1-beta1)*gradients
second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
first_term = first_moment/(1.0-beta1**iter)
second_term = second_moment/(1.0-beta2**iter)
# Scaling with rho the new and the previous results
update = eta*first_term/(np.sqrt(second_term)+delta)
theta -= update
print("theta from own ADAM")
print(theta)
!ec
!split
===== And Logistic Regression =====
@@ -2630,11 +2691,4 @@ print(derivative_fn(x_small))
!ec
!split
===== Weekend challenge =====
* Try to run the above codes and implement the stochastic gradient descent with the ADAM. Here you can use as examples the Adagrad and the RMSprop algorithms.
* Add a more complicated function and study the rate of convergence for the derivatives as function of the different methods
* Extend from linear regression to logistic regression.