Morten Hjorth-Jensen
486 lines
30 KiB
HTML
486 lines
30 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week39-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 39: Optimization and Gradient Methods">
|
|
<title>Week 39: Optimization and Gradient Methods</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week39-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plan for week 39, September 23-27, 2024',
|
|
2,
|
|
None,
|
|
'plan-for-week-39-september-23-27-2024'),
|
|
('Lecture Monday September 23',
|
|
2,
|
|
None,
|
|
'lecture-monday-september-23'),
|
|
('Lab sessions week 39', 2, None, 'lab-sessions-week-39'),
|
|
('Lecture Monday September 23, Optimization, the central part of '
|
|
'any Machine Learning algortithm',
|
|
2,
|
|
None,
|
|
'lecture-monday-september-23-optimization-the-central-part-of-any-machine-learning-algortithm'),
|
|
('Revisiting our Logistic Regression case',
|
|
2,
|
|
None,
|
|
'revisiting-our-logistic-regression-case'),
|
|
('The equations to solve', 2, None, 'the-equations-to-solve'),
|
|
("Solving using Newton-Raphson's method",
|
|
2,
|
|
None,
|
|
'solving-using-newton-raphson-s-method'),
|
|
("Brief reminder on Newton-Raphson's method",
|
|
2,
|
|
None,
|
|
'brief-reminder-on-newton-raphson-s-method'),
|
|
('The equations', 2, None, 'the-equations'),
|
|
('Simple geometric interpretation',
|
|
2,
|
|
None,
|
|
'simple-geometric-interpretation'),
|
|
('Extending to more than one variable',
|
|
2,
|
|
None,
|
|
'extending-to-more-than-one-variable'),
|
|
('Steepest descent', 2, None, 'steepest-descent'),
|
|
('More on Steepest descent', 2, None, 'more-on-steepest-descent'),
|
|
('The ideal', 2, None, 'the-ideal'),
|
|
('The sensitiveness of the gradient descent',
|
|
2,
|
|
None,
|
|
'the-sensitiveness-of-the-gradient-descent'),
|
|
('Convex functions', 2, None, 'convex-functions'),
|
|
('Convex function', 2, None, 'convex-function'),
|
|
('Conditions on convex functions',
|
|
2,
|
|
None,
|
|
'conditions-on-convex-functions'),
|
|
('More on convex functions', 2, None, 'more-on-convex-functions'),
|
|
('Some simple problems', 2, None, 'some-simple-problems'),
|
|
('Standard steepest descent',
|
|
2,
|
|
None,
|
|
'standard-steepest-descent'),
|
|
('Gradient method', 2, None, 'gradient-method'),
|
|
('Steepest descent method', 2, None, 'steepest-descent-method'),
|
|
('Steepest descent method', 2, None, 'steepest-descent-method'),
|
|
('Final expressions', 2, None, 'final-expressions'),
|
|
('Steepest descent example', 2, None, 'steepest-descent-example'),
|
|
('Conjugate gradient method',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method'),
|
|
('Conjugate gradient method',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method'),
|
|
('Conjugate gradient method',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method'),
|
|
('Conjugate gradient method',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method'),
|
|
('Conjugate gradient method and iterations',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method-and-iterations'),
|
|
('Conjugate gradient method',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method'),
|
|
('Conjugate gradient method',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method'),
|
|
('Conjugate gradient method',
|
|
2,
|
|
None,
|
|
'conjugate-gradient-method'),
|
|
('Revisiting our first homework',
|
|
2,
|
|
None,
|
|
'revisiting-our-first-homework'),
|
|
('Gradient descent example', 2, None, 'gradient-descent-example'),
|
|
('The derivative of the cost/loss function',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-cost-loss-function'),
|
|
('The Hessian matrix', 2, None, 'the-hessian-matrix'),
|
|
('Simple program', 2, None, 'simple-program'),
|
|
('Gradient Descent Example', 2, None, 'gradient-descent-example'),
|
|
('And a corresponding example using _scikit-learn_',
|
|
2,
|
|
None,
|
|
'and-a-corresponding-example-using-scikit-learn'),
|
|
('Gradient descent and Ridge',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-ridge'),
|
|
('The Hessian matrix for Ridge Regression',
|
|
2,
|
|
None,
|
|
'the-hessian-matrix-for-ridge-regression'),
|
|
('Program example for gradient descent with Ridge Regression',
|
|
2,
|
|
None,
|
|
'program-example-for-gradient-descent-with-ridge-regression'),
|
|
('Using gradient descent methods, limitations',
|
|
2,
|
|
None,
|
|
'using-gradient-descent-methods-limitations'),
|
|
('Improving gradient descent with momentum',
|
|
2,
|
|
None,
|
|
'improving-gradient-descent-with-momentum'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Overview video on Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'overview-video-on-stochastic-gradient-descent'),
|
|
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
|
('Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent-sgd'),
|
|
('Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent'),
|
|
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
|
('SGD example', 2, None, 'sgd-example'),
|
|
('The gradient step', 2, None, 'the-gradient-step'),
|
|
('Simple example code', 2, None, 'simple-example-code'),
|
|
('When do we stop?', 2, None, 'when-do-we-stop'),
|
|
('Slightly different approach',
|
|
2,
|
|
None,
|
|
'slightly-different-approach'),
|
|
('Time decay rate', 2, None, 'time-decay-rate'),
|
|
('Code with a Number of Minibatches which varies',
|
|
2,
|
|
None,
|
|
'code-with-a-number-of-minibatches-which-varies'),
|
|
('Replace or not', 2, None, 'replace-or-not'),
|
|
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
|
('More on momentum based approaches',
|
|
2,
|
|
None,
|
|
'more-on-momentum-based-approaches'),
|
|
('Momentum parameter', 2, None, 'momentum-parameter'),
|
|
('Second moment of the gradient',
|
|
2,
|
|
None,
|
|
'second-moment-of-the-gradient'),
|
|
('RMS prop', 2, None, 'rms-prop'),
|
|
('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
|
|
2,
|
|
None,
|
|
'adam-optimizer-https-arxiv-org-abs-1412-6980'),
|
|
('Algorithms and codes for Adagrad, RMSprop and Adam',
|
|
2,
|
|
None,
|
|
'algorithms-and-codes-for-adagrad-rmsprop-and-adam'),
|
|
('Practical tips', 2, None, 'practical-tips'),
|
|
('Automatic differentiation',
|
|
2,
|
|
None,
|
|
'automatic-differentiation'),
|
|
('Using autograd', 2, None, 'using-autograd'),
|
|
('Autograd with more complicated functions',
|
|
2,
|
|
None,
|
|
'autograd-with-more-complicated-functions'),
|
|
('More complicated functions using the elements of their '
|
|
'arguments directly',
|
|
2,
|
|
None,
|
|
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
|
('Functions using mathematical functions from Numpy',
|
|
2,
|
|
None,
|
|
'functions-using-mathematical-functions-from-numpy'),
|
|
('More autograd', 2, None, 'more-autograd'),
|
|
('And with loops', 2, None, 'and-with-loops'),
|
|
('Using recursion', 2, None, 'using-recursion'),
|
|
('Unsupported functions', 2, None, 'unsupported-functions'),
|
|
('The syntax a.dot(b) when finding the dot product',
|
|
2,
|
|
None,
|
|
'the-syntax-a-dot-b-when-finding-the-dot-product'),
|
|
('Using Autograd with OLS', 2, None, 'using-autograd-with-ols'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
("But none of these can compete with Newton's method",
|
|
2,
|
|
None,
|
|
'but-none-of-these-can-compete-with-newton-s-method'),
|
|
('Including Stochastic Gradient Descent with Autograd',
|
|
2,
|
|
None,
|
|
'including-stochastic-gradient-descent-with-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Similar (second order function now) problem but now with '
|
|
'AdaGrad',
|
|
2,
|
|
None,
|
|
'similar-second-order-function-now-problem-but-now-with-adagrad'),
|
|
('RMSprop for adaptive learning rate with Stochastic Gradient '
|
|
'Descent',
|
|
2,
|
|
None,
|
|
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
|
|
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
|
|
2,
|
|
None,
|
|
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
|
|
('And Logistic Regression', 2, None, 'and-logistic-regression'),
|
|
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
|
|
2,
|
|
None,
|
|
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week39-bs.html">Week 39: Optimization and Gradient Methods</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week39-bs001.html#plan-for-week-39-september-23-27-2024" style="font-size: 80%;">Plan for week 39, September 23-27, 2024</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs002.html#lecture-monday-september-23" style="font-size: 80%;">Lecture Monday September 23</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs003.html#lab-sessions-week-39" style="font-size: 80%;">Lab sessions week 39</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs004.html#lecture-monday-september-23-optimization-the-central-part-of-any-machine-learning-algortithm" style="font-size: 80%;">Lecture Monday September 23, Optimization, the central part of any Machine Learning algortithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs005.html#revisiting-our-logistic-regression-case" style="font-size: 80%;">Revisiting our Logistic Regression case</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs006.html#the-equations-to-solve" style="font-size: 80%;">The equations to solve</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs007.html#solving-using-newton-raphson-s-method" style="font-size: 80%;">Solving using Newton-Raphson's method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs008.html#brief-reminder-on-newton-raphson-s-method" style="font-size: 80%;">Brief reminder on Newton-Raphson's method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs009.html#the-equations" style="font-size: 80%;">The equations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs010.html#simple-geometric-interpretation" style="font-size: 80%;">Simple geometric interpretation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs011.html#extending-to-more-than-one-variable" style="font-size: 80%;">Extending to more than one variable</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs012.html#steepest-descent" style="font-size: 80%;">Steepest descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs013.html#more-on-steepest-descent" style="font-size: 80%;">More on Steepest descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs014.html#the-ideal" style="font-size: 80%;">The ideal</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs015.html#the-sensitiveness-of-the-gradient-descent" style="font-size: 80%;">The sensitiveness of the gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs016.html#convex-functions" style="font-size: 80%;">Convex functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs017.html#convex-function" style="font-size: 80%;">Convex function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs018.html#conditions-on-convex-functions" style="font-size: 80%;">Conditions on convex functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs019.html#more-on-convex-functions" style="font-size: 80%;">More on convex functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs020.html#some-simple-problems" style="font-size: 80%;">Some simple problems</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs021.html#standard-steepest-descent" style="font-size: 80%;">Standard steepest descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs022.html#gradient-method" style="font-size: 80%;">Gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs024.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs024.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs025.html#final-expressions" style="font-size: 80%;">Final expressions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs026.html#steepest-descent-example" style="font-size: 80%;">Steepest descent example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs031.html#conjugate-gradient-method-and-iterations" style="font-size: 80%;">Conjugate gradient method and iterations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs034.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs035.html#revisiting-our-first-homework" style="font-size: 80%;">Revisiting our first homework</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs040.html#gradient-descent-example" style="font-size: 80%;">Gradient descent example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs037.html#the-derivative-of-the-cost-loss-function" style="font-size: 80%;">The derivative of the cost/loss function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs038.html#the-hessian-matrix" style="font-size: 80%;">The Hessian matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs039.html#simple-program" style="font-size: 80%;">Simple program</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs040.html#gradient-descent-example" style="font-size: 80%;">Gradient Descent Example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs041.html#and-a-corresponding-example-using-scikit-learn" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs042.html#gradient-descent-and-ridge" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs043.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;">The Hessian matrix for Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs044.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs045.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs046.html#improving-gradient-descent-with-momentum" style="font-size: 80%;">Improving gradient descent with momentum</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs083.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs048.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;">Overview video on Stochastic Gradient Descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs049.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs050.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs051.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs052.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs053.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs054.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs055.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs056.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs057.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs058.html#time-decay-rate" style="font-size: 80%;">Time decay rate</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs059.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs060.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs061.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs062.html#more-on-momentum-based-approaches" style="font-size: 80%;">More on momentum based approaches</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs063.html#momentum-parameter" style="font-size: 80%;">Momentum parameter</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs064.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
|
|
<!-- navigation toc: --> <li><a href="#rms-prop" style="font-size: 80%;">RMS prop</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs066.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs067.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs068.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs069.html#automatic-differentiation" style="font-size: 80%;">Automatic differentiation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs070.html#using-autograd" style="font-size: 80%;">Using autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs071.html#autograd-with-more-complicated-functions" style="font-size: 80%;">Autograd with more complicated functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs072.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;">More complicated functions using the elements of their arguments directly</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs073.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;">Functions using mathematical functions from Numpy</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs074.html#more-autograd" style="font-size: 80%;">More autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs075.html#and-with-loops" style="font-size: 80%;">And with loops</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs076.html#using-recursion" style="font-size: 80%;">Using recursion</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs077.html#unsupported-functions" style="font-size: 80%;">Unsupported functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs078.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;">The syntax a.dot(b) when finding the dot product</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs079.html#using-autograd-with-ols" style="font-size: 80%;">Using Autograd with OLS</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs083.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs081.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;">But none of these can compete with Newton's method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs082.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs083.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs084.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs085.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs086.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;">And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs087.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week39-bs088.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0065"></a>
|
|
<!-- !split -->
|
|
<h2 id="rms-prop" class="anchor">RMS prop </h2>
|
|
|
|
<p>In RMS prop, in addition to keeping a running average of the first
|
|
moment of the gradient, we also keep track of the second moment
|
|
denoted by \( \mathbf{s}_t=\mathbb{E}[\mathbf{g}_t^2] \). The update rule
|
|
for RMS prop is given by
|
|
</p>
|
|
|
|
$$
|
|
\begin{align}
|
|
\mathbf{g}_t &= \nabla_\theta E(\boldsymbol{\theta})
|
|
\tag{4}\\
|
|
\mathbf{s}_t &=\beta \mathbf{s}_{t-1} +(1-\beta)\mathbf{g}_t^2 \nonumber \\
|
|
\boldsymbol{\theta}_{t+1}&=&\boldsymbol{\theta}_t - \eta_t { \mathbf{g}_t \over \sqrt{\mathbf{s}_t +\epsilon}}, \nonumber
|
|
\end{align}
|
|
$$
|
|
|
|
<p>where \( \beta \) controls the averaging time of the second moment and is
|
|
typically taken to be about \( \beta=0.9 \), \( \eta_t \) is a learning rate
|
|
typically chosen to be \( 10^{-3} \), and \( \epsilon\sim 10^{-8} \) is a
|
|
small regularization constant to prevent divergences. Multiplication
|
|
and division by vectors is understood as an element-wise operation. It
|
|
is clear from this formula that the learning rate is reduced in
|
|
directions where the norm of the gradient is consistently large. This
|
|
greatly speeds up the convergence by allowing us to use a larger
|
|
learning rate for flat directions.
|
|
</p>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week39-bs064.html">«</a></li>
|
|
<li><a href="._week39-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week39-bs057.html">58</a></li>
|
|
<li><a href="._week39-bs058.html">59</a></li>
|
|
<li><a href="._week39-bs059.html">60</a></li>
|
|
<li><a href="._week39-bs060.html">61</a></li>
|
|
<li><a href="._week39-bs061.html">62</a></li>
|
|
<li><a href="._week39-bs062.html">63</a></li>
|
|
<li><a href="._week39-bs063.html">64</a></li>
|
|
<li><a href="._week39-bs064.html">65</a></li>
|
|
<li class="active"><a href="._week39-bs065.html">66</a></li>
|
|
<li><a href="._week39-bs066.html">67</a></li>
|
|
<li><a href="._week39-bs067.html">68</a></li>
|
|
<li><a href="._week39-bs068.html">69</a></li>
|
|
<li><a href="._week39-bs069.html">70</a></li>
|
|
<li><a href="._week39-bs070.html">71</a></li>
|
|
<li><a href="._week39-bs071.html">72</a></li>
|
|
<li><a href="._week39-bs072.html">73</a></li>
|
|
<li><a href="._week39-bs073.html">74</a></li>
|
|
<li><a href="._week39-bs074.html">75</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week39-bs088.html">89</a></li>
|
|
<li><a href="._week39-bs066.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|