527 lines
33 KiB
HTML
527 lines
33 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 37: Gradient descent methods">
|
|
<title>Week 37: Gradient descent methods</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 37, lecture Monday',
|
|
2,
|
|
None,
|
|
'plans-for-week-37-lecture-monday'),
|
|
('Readings and Videos:', 2, None, 'readings-and-videos'),
|
|
('Material for lecture Monday September 8',
|
|
2,
|
|
None,
|
|
'material-for-lecture-monday-september-8'),
|
|
('Gradient descent and revisiting Ordinary Least Squares from '
|
|
'last week',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-revisiting-ordinary-least-squares-from-last-week'),
|
|
('Gradient descent example', 2, None, 'gradient-descent-example'),
|
|
('The derivative of the cost/loss function',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-cost-loss-function'),
|
|
('The Hessian matrix', 2, None, 'the-hessian-matrix'),
|
|
('Simple program', 2, None, 'simple-program'),
|
|
('Gradient Descent Example', 2, None, 'gradient-descent-example'),
|
|
('Gradient descent and Ridge',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-ridge'),
|
|
('The Hessian matrix for Ridge Regression',
|
|
2,
|
|
None,
|
|
'the-hessian-matrix-for-ridge-regression'),
|
|
('Program example for gradient descent with Ridge Regression',
|
|
2,
|
|
None,
|
|
'program-example-for-gradient-descent-with-ridge-regression'),
|
|
('Using gradient descent methods, limitations',
|
|
2,
|
|
None,
|
|
'using-gradient-descent-methods-limitations'),
|
|
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
|
('Improving gradient descent with momentum',
|
|
2,
|
|
None,
|
|
'improving-gradient-descent-with-momentum'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Overview video on Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'overview-video-on-stochastic-gradient-descent-sgd'),
|
|
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
|
('Pros and cons', 2, None, 'pros-and-cons'),
|
|
('Convergence rates', 2, None, 'convergence-rates'),
|
|
('Accuracy', 2, None, 'accuracy'),
|
|
('Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent-sgd'),
|
|
('Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent'),
|
|
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
|
('SGD example', 2, None, 'sgd-example'),
|
|
('The gradient step', 2, None, 'the-gradient-step'),
|
|
('Simple example code', 2, None, 'simple-example-code'),
|
|
('When do we stop?', 2, None, 'when-do-we-stop'),
|
|
('Slightly different approach',
|
|
2,
|
|
None,
|
|
'slightly-different-approach'),
|
|
('Time decay rate', 2, None, 'time-decay-rate'),
|
|
('Code with a Number of Minibatches which varies',
|
|
2,
|
|
None,
|
|
'code-with-a-number-of-minibatches-which-varies'),
|
|
('Replace or not', 2, None, 'replace-or-not'),
|
|
('SGD vs Full-Batch GD: Convergence Speed and Memory Comparison',
|
|
2,
|
|
None,
|
|
'sgd-vs-full-batch-gd-convergence-speed-and-memory-comparison'),
|
|
('Theoretical Convergence Speed and convex optimization',
|
|
3,
|
|
None,
|
|
'theoretical-convergence-speed-and-convex-optimization'),
|
|
('Strongly Convex Case', 3, None, 'strongly-convex-case'),
|
|
('Non-Convex Problems', 3, None, 'non-convex-problems'),
|
|
('Memory Usage and Scalability',
|
|
2,
|
|
None,
|
|
'memory-usage-and-scalability'),
|
|
('Empirical Evidence: Convergence Time and Memory in Practice',
|
|
2,
|
|
None,
|
|
'empirical-evidence-convergence-time-and-memory-in-practice'),
|
|
('Deep Neural Networks', 3, None, 'deep-neural-networks'),
|
|
('Memory constraints', 3, None, 'memory-constraints'),
|
|
('Second moment of the gradient',
|
|
2,
|
|
None,
|
|
'second-moment-of-the-gradient'),
|
|
('Challenge: Choosing a Fixed Learning Rate',
|
|
2,
|
|
None,
|
|
'challenge-choosing-a-fixed-learning-rate'),
|
|
('Motivation for Adaptive Step Sizes',
|
|
2,
|
|
None,
|
|
'motivation-for-adaptive-step-sizes'),
|
|
('AdaGrad algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Derivation of the AdaGrad Algorithm',
|
|
2,
|
|
None,
|
|
'derivation-of-the-adagrad-algorithm'),
|
|
('AdaGrad Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adagrad-update-rule-derivation'),
|
|
('AdaGrad Properties', 2, None, 'adagrad-properties'),
|
|
('RMSProp: Adaptive Learning Rates',
|
|
2,
|
|
None,
|
|
'rmsprop-adaptive-learning-rates'),
|
|
('RMSProp algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Adam Optimizer', 2, None, 'adam-optimizer'),
|
|
('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
|
|
2,
|
|
None,
|
|
'adam-optimizer-https-arxiv-org-abs-1412-6980'),
|
|
('Why Combine Momentum and RMSProp?',
|
|
2,
|
|
None,
|
|
'why-combine-momentum-and-rmsprop'),
|
|
('Adam: Exponential Moving Averages (Moments)',
|
|
2,
|
|
None,
|
|
'adam-exponential-moving-averages-moments'),
|
|
('Adam: Bias Correction', 2, None, 'adam-bias-correction'),
|
|
('Adam: Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adam-update-rule-derivation'),
|
|
('Adam vs. AdaGrad and RMSProp',
|
|
2,
|
|
None,
|
|
'adam-vs-adagrad-and-rmsprop'),
|
|
('Adaptivity Across Dimensions',
|
|
2,
|
|
None,
|
|
'adaptivity-across-dimensions'),
|
|
('ADAM algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Algorithms and codes for Adagrad, RMSprop and Adam',
|
|
2,
|
|
None,
|
|
'algorithms-and-codes-for-adagrad-rmsprop-and-adam'),
|
|
('Practical tips', 2, None, 'practical-tips'),
|
|
('Sneaking in automatic differentiation using Autograd',
|
|
2,
|
|
None,
|
|
'sneaking-in-automatic-differentiation-using-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Including Stochastic Gradient Descent with Autograd',
|
|
2,
|
|
None,
|
|
'including-stochastic-gradient-descent-with-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
("But none of these can compete with Newton's method",
|
|
2,
|
|
None,
|
|
'but-none-of-these-can-compete-with-newton-s-method'),
|
|
('Similar (second order function now) problem but now with '
|
|
'AdaGrad',
|
|
2,
|
|
None,
|
|
'similar-second-order-function-now-problem-but-now-with-adagrad'),
|
|
('RMSprop for adaptive learning rate with Stochastic Gradient '
|
|
'Descent',
|
|
2,
|
|
None,
|
|
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
|
|
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
|
|
2,
|
|
None,
|
|
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
|
|
('Material for the lab sessions',
|
|
2,
|
|
None,
|
|
'material-for-the-lab-sessions'),
|
|
('Reminder on different scaling methods',
|
|
2,
|
|
None,
|
|
'reminder-on-different-scaling-methods'),
|
|
('Functionality in Scikit-Learn',
|
|
2,
|
|
None,
|
|
'functionality-in-scikit-learn'),
|
|
('More preprocessing', 2, None, 'more-preprocessing'),
|
|
('Frequently used scaling functions',
|
|
2,
|
|
None,
|
|
'frequently-used-scaling-functions')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week37-bs.html">Week 37: Gradient descent methods</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37-lecture-monday" style="font-size: 80%;"><b>Plans for week 37, lecture Monday</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs002.html#readings-and-videos" style="font-size: 80%;"><b>Readings and Videos:</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs003.html#material-for-lecture-monday-september-8" style="font-size: 80%;"><b>Material for lecture Monday September 8</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs004.html#gradient-descent-and-revisiting-ordinary-least-squares-from-last-week" style="font-size: 80%;"><b>Gradient descent and revisiting Ordinary Least Squares from last week</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#gradient-descent-example" style="font-size: 80%;"><b>Gradient descent example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs006.html#the-derivative-of-the-cost-loss-function" style="font-size: 80%;"><b>The derivative of the cost/loss function</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs007.html#the-hessian-matrix" style="font-size: 80%;"><b>The Hessian matrix</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs008.html#simple-program" style="font-size: 80%;"><b>Simple program</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#gradient-descent-example" style="font-size: 80%;"><b>Gradient Descent Example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs010.html#gradient-descent-and-ridge" style="font-size: 80%;"><b>Gradient descent and Ridge</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs011.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;"><b>The Hessian matrix for Ridge Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs012.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;"><b>Program example for gradient descent with Ridge Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs013.html#using-gradient-descent-methods-limitations" style="font-size: 80%;"><b>Using gradient descent methods, limitations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs014.html#momentum-based-gd" style="font-size: 80%;"><b>Momentum based GD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs015.html#improving-gradient-descent-with-momentum" style="font-size: 80%;"><b>Improving gradient descent with momentum</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs056.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;"><b>Same code but now with momentum gradient descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs017.html#overview-video-on-stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Overview video on Stochastic Gradient Descent (SGD)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs018.html#batches-and-mini-batches" style="font-size: 80%;"><b>Batches and mini-batches</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs019.html#pros-and-cons" style="font-size: 80%;"><b>Pros and cons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs020.html#convergence-rates" style="font-size: 80%;"><b>Convergence rates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs021.html#accuracy" style="font-size: 80%;"><b>Accuracy</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs022.html#stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Stochastic Gradient Descent (SGD)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs023.html#stochastic-gradient-descent" style="font-size: 80%;"><b>Stochastic Gradient Descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs024.html#computation-of-gradients" style="font-size: 80%;"><b>Computation of gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs025.html#sgd-example" style="font-size: 80%;"><b>SGD example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs026.html#the-gradient-step" style="font-size: 80%;"><b>The gradient step</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs027.html#simple-example-code" style="font-size: 80%;"><b>Simple example code</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs028.html#when-do-we-stop" style="font-size: 80%;"><b>When do we stop?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs029.html#slightly-different-approach" style="font-size: 80%;"><b>Slightly different approach</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs030.html#time-decay-rate" style="font-size: 80%;"><b>Time decay rate</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs031.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;"><b>Code with a Number of Minibatches which varies</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs032.html#replace-or-not" style="font-size: 80%;"><b>Replace or not</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#sgd-vs-full-batch-gd-convergence-speed-and-memory-comparison" style="font-size: 80%;"><b>SGD vs Full-Batch GD: Convergence Speed and Memory Comparison</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#theoretical-convergence-speed-and-convex-optimization" style="font-size: 80%;"> Theoretical Convergence Speed and convex optimization</a></li>
|
|
<!-- navigation toc: --> <li><a href="#strongly-convex-case" style="font-size: 80%;"> Strongly Convex Case</a></li>
|
|
<!-- navigation toc: --> <li><a href="#non-convex-problems" style="font-size: 80%;"> Non-Convex Problems</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs034.html#memory-usage-and-scalability" style="font-size: 80%;"><b>Memory Usage and Scalability</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#empirical-evidence-convergence-time-and-memory-in-practice" style="font-size: 80%;"><b>Empirical Evidence: Convergence Time and Memory in Practice</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#deep-neural-networks" style="font-size: 80%;"> Deep Neural Networks</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#memory-constraints" style="font-size: 80%;"> Memory constraints</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs036.html#second-moment-of-the-gradient" style="font-size: 80%;"><b>Second moment of the gradient</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs037.html#challenge-choosing-a-fixed-learning-rate" style="font-size: 80%;"><b>Challenge: Choosing a Fixed Learning Rate</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs038.html#motivation-for-adaptive-step-sizes" style="font-size: 80%;"><b>Motivation for Adaptive Step Sizes</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs038.html#adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;"><b>AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs039.html#derivation-of-the-adagrad-algorithm" style="font-size: 80%;"><b>Derivation of the AdaGrad Algorithm</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs040.html#adagrad-update-rule-derivation" style="font-size: 80%;"><b>AdaGrad Update Rule Derivation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs041.html#adagrad-properties" style="font-size: 80%;"><b>AdaGrad Properties</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs042.html#rmsprop-adaptive-learning-rates" style="font-size: 80%;"><b>RMSProp: Adaptive Learning Rates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs042.html#rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;"><b>RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs043.html#adam-optimizer" style="font-size: 80%;"><b>Adam Optimizer</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs044.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;"><b>"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs045.html#why-combine-momentum-and-rmsprop" style="font-size: 80%;"><b>Why Combine Momentum and RMSProp?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs046.html#adam-exponential-moving-averages-moments" style="font-size: 80%;"><b>Adam: Exponential Moving Averages (Moments)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs047.html#adam-bias-correction" style="font-size: 80%;"><b>Adam: Bias Correction</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs048.html#adam-update-rule-derivation" style="font-size: 80%;"><b>Adam: Update Rule Derivation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs049.html#adam-vs-adagrad-and-rmsprop" style="font-size: 80%;"><b>Adam vs. AdaGrad and RMSProp</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs050.html#adaptivity-across-dimensions" style="font-size: 80%;"><b>Adaptivity Across Dimensions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs050.html#adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;"><b>ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs051.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;"><b>Algorithms and codes for Adagrad, RMSprop and Adam</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs052.html#practical-tips" style="font-size: 80%;"><b>Practical tips</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#sneaking-in-automatic-differentiation-using-autograd" style="font-size: 80%;"><b>Sneaking in automatic differentiation using Autograd</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs056.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;"><b>Same code but now with momentum gradient descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs055.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;"><b>Including Stochastic Gradient Descent with Autograd</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs056.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;"><b>Same code but now with momentum gradient descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs057.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;"><b>But none of these can compete with Newton's method</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs058.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;"><b>Similar (second order function now) problem but now with AdaGrad</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs059.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;"><b>RMSprop for adaptive learning rate with Stochastic Gradient Descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs060.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;"><b>And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs061.html#material-for-the-lab-sessions" style="font-size: 80%;"><b>Material for the lab sessions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs062.html#reminder-on-different-scaling-methods" style="font-size: 80%;"><b>Reminder on different scaling methods</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs063.html#functionality-in-scikit-learn" style="font-size: 80%;"><b>Functionality in Scikit-Learn</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs064.html#more-preprocessing" style="font-size: 80%;"><b>More preprocessing</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs065.html#frequently-used-scaling-functions" style="font-size: 80%;"><b>Frequently used scaling functions</b></a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0033"></a>
|
|
<!-- !split -->
|
|
<h2 id="sgd-vs-full-batch-gd-convergence-speed-and-memory-comparison" class="anchor">SGD vs Full-Batch GD: Convergence Speed and Memory Comparison </h2>
|
|
<h3 id="theoretical-convergence-speed-and-convex-optimization" class="anchor">Theoretical Convergence Speed and convex optimization </h3>
|
|
|
|
<p>Consider minimizing an empirical cost function</p>
|
|
$$
|
|
C(\theta) =\frac{1}{N}\sum_{i=1}^N l_i(\theta),
|
|
$$
|
|
|
|
<p>where each \( l_i(\theta) \) is a
|
|
differentiable loss term. Gradient Descent (GD) updates parameters
|
|
using the full gradient \( \nabla C(\theta) \), while Stochastic Gradient
|
|
Descent (SGD) uses a single sample (or mini-batch) gradient \( \nabla
|
|
l_i(\theta) \) selected at random. In equation form, one GD step is:
|
|
</p>
|
|
|
|
$$
|
|
\theta_{t+1} = \theta_t-\eta \nabla C(\theta_t) =\theta_t -\eta \frac{1}{N}\sum_{i=1}^N \nabla l_i(\theta_t),
|
|
$$
|
|
|
|
<p>whereas one SGD step is:</p>
|
|
|
|
$$
|
|
\theta_{t+1} = \theta_t -\eta \nabla l_{i_t}(\theta_t),
|
|
$$
|
|
|
|
<p>with \( i_t \) randomly chosen. On smooth convex problems, GD and SGD both
|
|
converge to the global minimum, but their rates differ. GD can take
|
|
larger, more stable steps since it uses the exact gradient, achieving
|
|
an error that decreases on the order of \( O(1/t) \) per iteration for
|
|
convex objectives (and even exponentially fast for strongly convex
|
|
cases). In contrast, plain SGD has more variance in each step, leading
|
|
to sublinear convergence in expectation – typically \( O(1/\sqrt{t}) \)
|
|
for general convex objectives (\thetaith appropriate diminishing step
|
|
sizes) . Intuitively, GD’s trajectory is smoother and more
|
|
predictable, while SGD’s path oscillates due to noise but costs far
|
|
less per iteration, enabling many more updates in the same time.
|
|
</p>
|
|
<h3 id="strongly-convex-case" class="anchor">Strongly Convex Case </h3>
|
|
|
|
<p>If \( C(\theta) \) is strongly convex and \( L \)-smooth (so GD enjoys linear
|
|
convergence), the gap \( C(\theta_t)-C(\theta^*) \) for GD shrinks as
|
|
</p>
|
|
$$
|
|
C(\theta_t) - C(\theta^* ) \le \Big(1 - \frac{\mu}{L}\Big)^t [C(\theta_0)-C(\theta^*)],
|
|
$$
|
|
|
|
<p>a geometric (linear) convergence per iteration . Achieving an
|
|
\( \epsilon \)-accurate solution thus takes on the order of
|
|
\( \log(1/\epsilon) \) iterations for GD. However, each GD iteration costs
|
|
\( O(N) \) gradient evaluations. SGD cannot exploit strong convexity to
|
|
obtain a linear rate – instead, with a properly decaying step size
|
|
(e.g. \( \eta_t = \frac{1}{\mu t} \)) or iterate averaging, SGD attains an
|
|
\( O(1/t) \) convergence rate in expectation . For example, one result
|
|
of Moulines and Bach 2011, see <a href="https://papers.nips.cc/paper_files/paper/2011/hash/40008b9a5380fcacce3976bf7c08af5b-Abstract.html" target="_self"><tt>https://papers.nips.cc/paper_files/paper/2011/hash/40008b9a5380fcacce3976bf7c08af5b-Abstract.html</tt></a> shows that with \( \eta_t = \Theta(1/t) \),
|
|
</p>
|
|
$$
|
|
\mathbb{E}[C(\theta_t) - C(\theta^*)] = O(1/t),
|
|
$$
|
|
|
|
<p>for strongly convex, smooth \( F \) . This \( 1/t \) rate is slower per
|
|
iteration than GD’s exponential decay, but each SGD iteration is \( N \)
|
|
times cheaper. In fact, to reach error \( \epsilon \), plain SGD needs on
|
|
the order of \( T=O(1/\epsilon) \) iterations (sub-linear convergence),
|
|
while GD needs \( O(\log(1/\epsilon)) \) iterations. When accounting for
|
|
cost-per-iteration, GD requires \( O(N \log(1/\epsilon)) \) total gradient
|
|
computations versus SGD’s \( O(1/\epsilon) \) single-sample
|
|
computations. In large-scale regimes (huge \( N \)), SGD can be
|
|
faster in wall-clock time because \( N \log(1/\epsilon) \) may far exceed
|
|
\( 1/\epsilon \) for reasonable accuracy levels. In other words,
|
|
with millions of data points, one epoch of GD (one full gradient) is
|
|
extremely costly, whereas SGD can make \( N \) cheap updates in the time
|
|
GD makes one – often yielding a good solution faster in practice, even
|
|
though SGD’s asymptotic error decays more slowly. As one lecture
|
|
succinctly puts it: “SGD can be super effective in terms of iteration
|
|
cost and memory, but SGD is slow to converge and can’t adapt to strong
|
|
convexity” . Thus, the break-even point depends on \( N \) and the desired
|
|
accuracy: for moderate accuracy on very large \( N \), SGD’s cheaper
|
|
updates win; for extremely high precision (very small \( \epsilon \)) on a
|
|
modest \( N \), GD’s fast convergence per step can be advantageous.
|
|
</p>
|
|
<h3 id="non-convex-problems" class="anchor">Non-Convex Problems </h3>
|
|
|
|
<p>In non-convex optimization (e.g. deep neural networks), neither GD nor
|
|
SGD guarantees global minima, but SGD often displays faster progress
|
|
in finding useful minima. Theoretical results here are weaker, usually
|
|
showing convergence to a stationary point \( \theta \) (\( |\nabla C| \) is
|
|
small) in expectation. For example, GD might require \( O(1/\epsilon^2) \)
|
|
iterations to ensure \( |\nabla C(\theta)| < \epsilon \), and SGD typically has
|
|
similar polynomial complexity (often worse due to gradient
|
|
noise). However, a noteworthy difference is that SGD’s stochasticity
|
|
can help escape saddle points or poor local minima. Random gradient
|
|
fluctuations act like implicit noise, helping the iterate “jump” out
|
|
of flat saddle regions where full-batch GD could stagnate . In fact,
|
|
research has shown that adding noise to GD can guarantee escaping
|
|
saddle points in polynomial time, and the inherent noise in SGD often
|
|
serves this role. Empirically, this means SGD can sometimes find a
|
|
lower loss basin faster, whereas full-batch GD might get “stuck” near
|
|
saddle points or need a very small learning rate to navigate complex
|
|
error surfaces . Overall, in modern high-dimensional machine learning,
|
|
SGD (or mini-batch SGD) is the workhorse for large non-convex problems
|
|
because it converges to good solutions much faster in practice,
|
|
despite the lack of a linear convergence guarantee. Full-batch GD is
|
|
rarely used on large neural networks, as it would require tiny steps
|
|
to avoid divergence and is extremely slow per iteration .
|
|
</p>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week37-bs032.html">«</a></li>
|
|
<li><a href="._week37-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs025.html">26</a></li>
|
|
<li><a href="._week37-bs026.html">27</a></li>
|
|
<li><a href="._week37-bs027.html">28</a></li>
|
|
<li><a href="._week37-bs028.html">29</a></li>
|
|
<li><a href="._week37-bs029.html">30</a></li>
|
|
<li><a href="._week37-bs030.html">31</a></li>
|
|
<li><a href="._week37-bs031.html">32</a></li>
|
|
<li><a href="._week37-bs032.html">33</a></li>
|
|
<li class="active"><a href="._week37-bs033.html">34</a></li>
|
|
<li><a href="._week37-bs034.html">35</a></li>
|
|
<li><a href="._week37-bs035.html">36</a></li>
|
|
<li><a href="._week37-bs036.html">37</a></li>
|
|
<li><a href="._week37-bs037.html">38</a></li>
|
|
<li><a href="._week37-bs038.html">39</a></li>
|
|
<li><a href="._week37-bs039.html">40</a></li>
|
|
<li><a href="._week37-bs040.html">41</a></li>
|
|
<li><a href="._week37-bs041.html">42</a></li>
|
|
<li><a href="._week37-bs042.html">43</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs065.html">66</a></li>
|
|
<li><a href="._week37-bs034.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|