lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added rmsprop

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2022-10-04 14:29:48 +02:00
parent 585ea681bd
commit 170dde1c94
12 changed files with 848 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week39/html/week39-bs.html
+10 -4
View File
@@ -253,6 +253,11 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'similar-second-order-function-now-problem-but-now-with-adagrad'),
('RMSprop for adaptive learning rate with Stochastic Gradient '
'Descent',
2,
None,
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
@@ -375,9 +380,10 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._week39-bs080.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs081.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs082.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs083.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs084.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs085.html#weekend-challenge" style="font-size: 80%;">Weekend challenge</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs083.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs084.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs085.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
<!-- navigation toc: --> <li><a href="._week39-bs086.html#weekend-challenge" style="font-size: 80%;">Weekend challenge</a></li>
</ul>
</li>
@@ -432,7 +438,7 @@ MathJax.Hub.Config({
<li><a href="._week39-bs008.html">9</a></li>
<li><a href="._week39-bs009.html">10</a></li>
<li><a href="">...</a></li>
<li><a href="._week39-bs085.html">86</a></li>
<li><a href="._week39-bs086.html">87</a></li>
<li><a href="._week39-bs001.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
doc/pub/week39/html/week39-reveal.html
+83
View File
@@ -3565,6 +3565,89 @@ delta = <span style="color: #B452CD">1e-8</span>
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
</section>
<section>
<h2 id="rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">RMSprop for adaptive learning rate with Stochastic Gradient Descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">10000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Value for parameter rho</span>
rho = <span style="color: #B452CD">0.99</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Previous value for the outer product of gradients</span>
Previous = Giter
<span style="color: #228B22"># Accumulated gradient</span>
Giter +=gradients @ gradients.T
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
Gnew = (rho*Previous+(<span style="color: #B452CD">1</span>-rho)*Giter)
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
<span style="color: #228B22"># Hadamard product</span>
update = np.multiply(Ginverse,gradients)
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
</section>
<section>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
doc/pub/week39/html/week39-solarized.html
+88
View File
@@ -280,6 +280,11 @@ div.toc p,a {
2,
None,
'similar-second-order-function-now-problem-but-now-with-adagrad'),
('RMSprop for adaptive learning rate with Stochastic Gradient '
'Descent',
2,
None,
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
@@ -3492,6 +3497,89 @@ delta = <span style="color: #B452CD">1e-8</span>
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">RMSprop for adaptive learning rate with Stochastic Gradient Descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #eeeedd">
<pre style="line-height: 125%;"><span style="color: #228B22"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #228B22"># OLS example</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">autograd.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">autograd</span> <span style="color: #8B008B; font-weight: bold">import</span> grad
<span style="color: #228B22"># Note change from previous example</span>
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">CostOLS</span>(y,X,theta):
<span style="color: #8B008B; font-weight: bold">return</span> np.sum((y-X @ theta)**<span style="color: #B452CD">2</span>)
n = <span style="color: #B452CD">10000</span>
x = np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">2.0</span>+<span style="color: #B452CD">3</span>*x +<span style="color: #B452CD">4</span>*x*x<span style="color: #228B22"># +np.random.randn(n,1)</span>
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
<span style="color: #228B22"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient = grad(CostOLS,<span style="color: #B452CD">2</span>)
<span style="color: #228B22"># Define parameters for Stochastic Gradient Descent</span>
n_epochs = <span style="color: #B452CD">50</span>
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
<span style="color: #228B22"># Guess for unknown parameters theta</span>
theta = np.random.randn(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">1</span>)
<span style="color: #228B22"># Value for learning rate</span>
eta = <span style="color: #B452CD">0.01</span>
<span style="color: #228B22"># Value for parameter rho</span>
rho = <span style="color: #B452CD">0.99</span>
<span style="color: #228B22"># Including AdaGrad parameter to avoid possible division by zero</span>
delta = <span style="color: #B452CD">1e-8</span>
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
Giter = np.zeros(shape=(<span style="color: #B452CD">3</span>,<span style="color: #B452CD">3</span>))
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (<span style="color: #B452CD">1.0</span>/M)*training_gradient(yi, xi, theta)
<span style="color: #228B22"># Previous value for the outer product of gradients</span>
Previous = Giter
<span style="color: #228B22"># Accumulated gradient</span>
Giter +=gradients @ gradients.T
<span style="color: #228B22"># Scaling with rho the new and the previous results</span>
Gnew = (rho*Previous+(<span style="color: #B452CD">1</span>-rho)*Giter)
<span style="color: #228B22"># Taking the diagonal only and inverting</span>
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
<span style="color: #228B22"># Hadamard product</span>
update = np.multiply(Ginverse,gradients)
theta -= update
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #658b00">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
doc/pub/week39/html/week39.html
+88
View File
@@ -357,6 +357,11 @@ div.toc p,a {
2,
None,
'similar-second-order-function-now-problem-but-now-with-adagrad'),
('RMSprop for adaptive learning rate with Stochastic Gradient '
'Descent',
2,
None,
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
('And Logistic Regression', 2, None, 'and-logistic-regression'),
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
2,
@@ -3569,6 +3574,89 @@ delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</
<p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent">RMSprop for adaptive learning rate with Stochastic Gradient Descent </h2>
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="cell border-box-sizing code_cell rendered">
<div class="input">
<div class="inner_cell">
<div class="input_area">
<div class="highlight" style="background: #f8f8f8">
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent</span>
<span style="color: #408080; font-style: italic"># OLS example</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> grad
<span style="color: #408080; font-style: italic"># Note change from previous example</span>
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostOLS</span>(y,X,theta):
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>sum((y<span style="color: #666666">-</span>X <span style="color: #666666">@</span> theta)<span style="color: #666666">**2</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">10000</span>
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">2.0+3*</span>x <span style="color: #666666">+4*</span>x<span style="color: #666666">*</span>x<span style="color: #408080; font-style: italic"># +np.random.randn(n,1)</span>
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x, x<span style="color: #666666">*</span>x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>pinv(XT_X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Own inversion&quot;</span>)
<span style="color: #008000">print</span>(theta_linreg)
<span style="color: #408080; font-style: italic"># Note that we request the derivative wrt third argument (theta, 2 here)</span>
training_gradient <span style="color: #666666">=</span> grad(CostOLS,<span style="color: #666666">2</span>)
<span style="color: #408080; font-style: italic"># Define parameters for Stochastic Gradient Descent</span>
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
<span style="color: #408080; font-style: italic"># Guess for unknown parameters theta</span>
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">3</span>,<span style="color: #666666">1</span>)
<span style="color: #408080; font-style: italic"># Value for learning rate</span>
eta <span style="color: #666666">=</span> <span style="color: #666666">0.01</span>
<span style="color: #408080; font-style: italic"># Value for parameter rho</span>
rho <span style="color: #666666">=</span> <span style="color: #666666">0.99</span>
<span style="color: #408080; font-style: italic"># Including AdaGrad parameter to avoid possible division by zero</span>
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span>
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
Giter <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(shape<span style="color: #666666">=</span>(<span style="color: #666666">3</span>,<span style="color: #666666">3</span>))
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> M<span style="color: #666666">*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>M]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>M]
gradients <span style="color: #666666">=</span> (<span style="color: #666666">1.0/</span>M)<span style="color: #666666">*</span>training_gradient(yi, xi, theta)
<span style="color: #408080; font-style: italic"># Previous value for the outer product of gradients</span>
Previous <span style="color: #666666">=</span> Giter
<span style="color: #408080; font-style: italic"># Accumulated gradient</span>
Giter <span style="color: #666666">+=</span>gradients <span style="color: #666666">@</span> gradients<span style="color: #666666">.</span>T
<span style="color: #408080; font-style: italic"># Scaling with rho the new and the previous results</span>
Gnew <span style="color: #666666">=</span> (rho<span style="color: #666666">*</span>Previous<span style="color: #666666">+</span>(<span style="color: #666666">1-</span>rho)<span style="color: #666666">*</span>Giter)
<span style="color: #408080; font-style: italic"># Taking the diagonal only and inverting</span>
Ginverse <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[eta<span style="color: #666666">/</span>(delta<span style="color: #666666">+</span>np<span style="color: #666666">.</span>sqrt(np<span style="color: #666666">.</span>diagonal(Gnew)))]
<span style="color: #408080; font-style: italic"># Hadamard product</span>
update <span style="color: #666666">=</span> np<span style="color: #666666">.</span>multiply(Ginverse,gradients)
theta <span style="color: #666666">-=</span> update
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own RMSprop&quot;</span>)
<span style="color: #008000">print</span>(theta)
</pre>
</div>
</div>
</div>
</div>
<div class="output_wrapper">
<div class="output">
<div class="output_area">
<div class="output_subarea output_stream output_stdout output_text">
</div>
</div>
</div>
</div>
</div>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
<h2 id="and-logistic-regression">And Logistic Regression </h2>
doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week39/ipynb/week39.ipynb
+361 -281
View File
File diff suppressed because it is too large Load Diff
doc/src/week39/adagrad.py
-31
View File
@@ -1,31 +0,0 @@
# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
n = 10000
x = np.random.rand(n,1)
y = 4*x+3*x*x
# Setting up Design matrix
X = np.c_[np.ones((n,1)), x, x*x]
XTX = X.T @ X
XTy = X.T @ y
theta_linreg = np.linalg.pinv(XTX) @ (XTy)
print("Own inversion")
print(theta_linreg)
beta = np.random.randn(3,1)
eta = 0.01
delta = 1e-8
Niterations = 10000
Giter = np.zeros(shape=(3,3))
for iter in range(Niterations):
gradient = (2.0/n)*(XTX @ beta - XTy)
Giter +=gradient @ gradient.T
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
beta -= np.multiply(Ginverse,gradient)
print("Optimal parameters with AdaGrad",beta)
doc/src/week39/adagradSGD.py
-5
View File
@@ -43,11 +43,6 @@ for epoch in range(n_epochs):
gradients = (1.0/M)*training_gradient(yi, xi, theta)
Giter +=gradients @ gradients.T
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# calculate squared gradient by Hadamard multiplication
# r += (gradients*gradients)
# r = np.sum(gradients*gradients)
# compute update
update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own AdaGrad")
doc/src/week39/codes/adagradSGD.py
+50
View File
@@ -0,0 +1,50 @@
# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Value for learning rate
eta = 0.01
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
Giter +=gradients @ gradients.T
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own AdaGrad")
print(theta)
doc/src/week39/codes/rmspronoplot.py
+52
View File
@@ -0,0 +1,52 @@
# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Value for learning rate
eta = 0.01
rho = 0.99
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
Previous = Giter
Giter +=gradients @ gradients.T
Gnew = (rho*Previous+(1-rho)*Giter)
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own AdaGrad")
print(theta)
doc/src/week39/rmsprop.py
+52
View File
@@ -0,0 +1,52 @@
# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Value for learning rate
eta = 0.01
rho = 0.99
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
Previous = Giter
Giter +=gradients @ gradients.T
Gnew = (rho*Previous+(1-rho)*Giter)
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own AdaGrad")
print(theta)
doc/src/week39/week39.do.txt
+64
View File
@@ -2491,6 +2491,70 @@ print(theta)
Running this code we note an almost perfect agreement with the results from matrix inversion.
!split
===== RMSprop for adaptive learning rate with Stochastic Gradient Descent =====
!bc pycod
# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent
# OLS example
from random import random, seed
import numpy as np
import autograd.numpy as np
import matplotlib.pyplot as plt
from autograd import grad
# Note change from previous example
def CostOLS(y,X,theta):
return np.sum((y-X @ theta)**2)
n = 10000
x = np.random.rand(n,1)
y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
X = np.c_[np.ones((n,1)), x, x*x]
XT_X = X.T @ X
theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
# Note that we request the derivative wrt third argument (theta, 2 here)
training_gradient = grad(CostOLS,2)
# Define parameters for Stochastic Gradient Descent
n_epochs = 50
M = 5 #size of each minibatch
m = int(n/M) #number of minibatches
# Guess for unknown parameters theta
theta = np.random.randn(3,1)
# Value for learning rate
eta = 0.01
# Value for parameter rho
rho = 0.99
# Including AdaGrad parameter to avoid possible division by zero
delta = 1e-8
for epoch in range(n_epochs):
Giter = np.zeros(shape=(3,3))
for i in range(m):
random_index = M*np.random.randint(m)
xi = X[random_index:random_index+M]
yi = y[random_index:random_index+M]
gradients = (1.0/M)*training_gradient(yi, xi, theta)
# Previous value for the outer product of gradients
Previous = Giter
# Accumulated gradient
Giter +=gradients @ gradients.T
# Scaling with rho the new and the previous results
Gnew = (rho*Previous+(1-rho)*Giter)
# Taking the diagonal only and inverting
Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
# Hadamard product
update = np.multiply(Ginverse,gradients)
theta -= update
print("theta from own RMSprop")
print(theta)
!ec
!split
===== And Logistic Regression =====