405 lines
25 KiB
HTML
405 lines
25 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 37: Statistical interpretations and Resampling Methods">
|
|
<title>Week 37: Statistical interpretations and Resampling Methods</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 37, lecture Monday',
|
|
2,
|
|
None,
|
|
'plans-for-week-37-lecture-monday'),
|
|
('Readings and Videos:', 2, None, 'readings-and-videos'),
|
|
('Material for lecture Monday September 8',
|
|
2,
|
|
None,
|
|
'material-for-lecture-monday-september-8'),
|
|
('Gradient descent and revisiting Ordinary Least Squares from '
|
|
'last week',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-revisiting-ordinary-least-squares-from-last-week'),
|
|
('Gradient descent example', 2, None, 'gradient-descent-example'),
|
|
('The derivative of the cost/loss function',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-cost-loss-function'),
|
|
('The Hessian matrix', 2, None, 'the-hessian-matrix'),
|
|
('Simple program', 2, None, 'simple-program'),
|
|
('Gradient Descent Example', 2, None, 'gradient-descent-example'),
|
|
('Gradient descent and Ridge',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-ridge'),
|
|
('The Hessian matrix for Ridge Regression',
|
|
2,
|
|
None,
|
|
'the-hessian-matrix-for-ridge-regression'),
|
|
('Program example for gradient descent with Ridge Regression',
|
|
2,
|
|
None,
|
|
'program-example-for-gradient-descent-with-ridge-regression'),
|
|
('Using gradient descent methods, limitations',
|
|
2,
|
|
None,
|
|
'using-gradient-descent-methods-limitations'),
|
|
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
|
('Improving gradient descent with momentum',
|
|
2,
|
|
None,
|
|
'improving-gradient-descent-with-momentum'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Overview video on Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'overview-video-on-stochastic-gradient-descent-sgd'),
|
|
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
|
('Pros and cons', 2, None, 'pros-and-cons'),
|
|
('Convergence rates', 2, None, 'convergence-rates'),
|
|
('Accuracy', 2, None, 'accuracy'),
|
|
('Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent-sgd'),
|
|
('Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent'),
|
|
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
|
('SGD example', 2, None, 'sgd-example'),
|
|
('The gradient step', 2, None, 'the-gradient-step'),
|
|
('Simple example code', 2, None, 'simple-example-code'),
|
|
('When do we stop?', 2, None, 'when-do-we-stop'),
|
|
('Slightly different approach',
|
|
2,
|
|
None,
|
|
'slightly-different-approach'),
|
|
('Time decay rate', 2, None, 'time-decay-rate'),
|
|
('Code with a Number of Minibatches which varies',
|
|
2,
|
|
None,
|
|
'code-with-a-number-of-minibatches-which-varies'),
|
|
('Replace or not', 2, None, 'replace-or-not'),
|
|
('Second moment of the gradient',
|
|
2,
|
|
None,
|
|
'second-moment-of-the-gradient'),
|
|
('Challenge: Choosing a Fixed Learning Rate',
|
|
2,
|
|
None,
|
|
'challenge-choosing-a-fixed-learning-rate'),
|
|
('Motivation for Adaptive Step Sizes',
|
|
2,
|
|
None,
|
|
'motivation-for-adaptive-step-sizes'),
|
|
('AdaGrad algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Derivation of the AdaGrad Algorithm',
|
|
2,
|
|
None,
|
|
'derivation-of-the-adagrad-algorithm'),
|
|
('AdaGrad Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adagrad-update-rule-derivation'),
|
|
('AdaGrad Properties', 2, None, 'adagrad-properties'),
|
|
('RMSProp: Adaptive Learning Rates',
|
|
2,
|
|
None,
|
|
'rmsprop-adaptive-learning-rates'),
|
|
('RMSProp algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Adam Optimizer', 2, None, 'adam-optimizer'),
|
|
('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
|
|
2,
|
|
None,
|
|
'adam-optimizer-https-arxiv-org-abs-1412-6980'),
|
|
('Why Combine Momentum and RMSProp?',
|
|
2,
|
|
None,
|
|
'why-combine-momentum-and-rmsprop'),
|
|
('Adam: Exponential Moving Averages (Moments)',
|
|
2,
|
|
None,
|
|
'adam-exponential-moving-averages-moments'),
|
|
('Adam: Bias Correction', 2, None, 'adam-bias-correction'),
|
|
('Adam: Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adam-update-rule-derivation'),
|
|
('Adam vs. AdaGrad and RMSProp',
|
|
2,
|
|
None,
|
|
'adam-vs-adagrad-and-rmsprop'),
|
|
('Adaptivity Across Dimensions',
|
|
2,
|
|
None,
|
|
'adaptivity-across-dimensions'),
|
|
('ADAM algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Algorithms and codes for Adagrad, RMSprop and Adam',
|
|
2,
|
|
None,
|
|
'algorithms-and-codes-for-adagrad-rmsprop-and-adam'),
|
|
('Practical tips', 2, None, 'practical-tips'),
|
|
('Sneaking in automatic differentiation using Autograd',
|
|
2,
|
|
None,
|
|
'sneaking-in-automatic-differentiation-using-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Including Stochastic Gradient Descent with Autograd',
|
|
2,
|
|
None,
|
|
'including-stochastic-gradient-descent-with-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
("But none of these can compete with Newton's method",
|
|
2,
|
|
None,
|
|
'but-none-of-these-can-compete-with-newton-s-method'),
|
|
('Similar (second order function now) problem but now with '
|
|
'AdaGrad',
|
|
2,
|
|
None,
|
|
'similar-second-order-function-now-problem-but-now-with-adagrad'),
|
|
('RMSprop for adaptive learning rate with Stochastic Gradient '
|
|
'Descent',
|
|
2,
|
|
None,
|
|
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
|
|
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
|
|
2,
|
|
None,
|
|
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
|
|
('Material for the lab sessions',
|
|
2,
|
|
None,
|
|
'material-for-the-lab-sessions'),
|
|
('Reminder on different scaling methods',
|
|
2,
|
|
None,
|
|
'reminder-on-different-scaling-methods'),
|
|
('Functionality in Scikit-Learn',
|
|
2,
|
|
None,
|
|
'functionality-in-scikit-learn'),
|
|
('More preprocessing', 2, None, 'more-preprocessing'),
|
|
('Frequently used scaling functions',
|
|
2,
|
|
None,
|
|
'frequently-used-scaling-functions')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week37-bs.html">Week 37: Statistical interpretations and Resampling Methods</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37-lecture-monday" style="font-size: 80%;">Plans for week 37, lecture Monday</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs002.html#readings-and-videos" style="font-size: 80%;">Readings and Videos:</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs003.html#material-for-lecture-monday-september-8" style="font-size: 80%;">Material for lecture Monday September 8</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs004.html#gradient-descent-and-revisiting-ordinary-least-squares-from-last-week" style="font-size: 80%;">Gradient descent and revisiting Ordinary Least Squares from last week</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#gradient-descent-example" style="font-size: 80%;">Gradient descent example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs006.html#the-derivative-of-the-cost-loss-function" style="font-size: 80%;">The derivative of the cost/loss function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs007.html#the-hessian-matrix" style="font-size: 80%;">The Hessian matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs008.html#simple-program" style="font-size: 80%;">Simple program</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#gradient-descent-example" style="font-size: 80%;">Gradient Descent Example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs010.html#gradient-descent-and-ridge" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs011.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;">The Hessian matrix for Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs012.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs013.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs014.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs015.html#improving-gradient-descent-with-momentum" style="font-size: 80%;">Improving gradient descent with momentum</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs017.html#overview-video-on-stochastic-gradient-descent-sgd" style="font-size: 80%;">Overview video on Stochastic Gradient Descent (SGD)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs018.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs019.html#pros-and-cons" style="font-size: 80%;">Pros and cons</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs020.html#convergence-rates" style="font-size: 80%;">Convergence rates</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs021.html#accuracy" style="font-size: 80%;">Accuracy</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs022.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs023.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs024.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs025.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs026.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs027.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs028.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs029.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs030.html#time-decay-rate" style="font-size: 80%;">Time decay rate</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs031.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs032.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs033.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs034.html#challenge-choosing-a-fixed-learning-rate" style="font-size: 80%;">Challenge: Choosing a Fixed Learning Rate</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#motivation-for-adaptive-step-sizes" style="font-size: 80%;">Motivation for Adaptive Step Sizes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs036.html#derivation-of-the-adagrad-algorithm" style="font-size: 80%;">Derivation of the AdaGrad Algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs037.html#adagrad-update-rule-derivation" style="font-size: 80%;">AdaGrad Update Rule Derivation</a></li>
|
|
<!-- navigation toc: --> <li><a href="#adagrad-properties" style="font-size: 80%;">AdaGrad Properties</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs039.html#rmsprop-adaptive-learning-rates" style="font-size: 80%;">RMSProp: Adaptive Learning Rates</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs039.html#rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs040.html#adam-optimizer" style="font-size: 80%;">Adam Optimizer</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs041.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs042.html#why-combine-momentum-and-rmsprop" style="font-size: 80%;">Why Combine Momentum and RMSProp?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs043.html#adam-exponential-moving-averages-moments" style="font-size: 80%;">Adam: Exponential Moving Averages (Moments)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs044.html#adam-bias-correction" style="font-size: 80%;">Adam: Bias Correction</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs045.html#adam-update-rule-derivation" style="font-size: 80%;">Adam: Update Rule Derivation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs046.html#adam-vs-adagrad-and-rmsprop" style="font-size: 80%;">Adam vs. AdaGrad and RMSProp</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs047.html#adaptivity-across-dimensions" style="font-size: 80%;">Adaptivity Across Dimensions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs047.html#adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs048.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs049.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs050.html#sneaking-in-automatic-differentiation-using-autograd" style="font-size: 80%;">Sneaking in automatic differentiation using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs052.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs054.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;">But none of these can compete with Newton's method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs055.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs056.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs057.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;">And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs058.html#material-for-the-lab-sessions" style="font-size: 80%;">Material for the lab sessions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs059.html#reminder-on-different-scaling-methods" style="font-size: 80%;">Reminder on different scaling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs060.html#functionality-in-scikit-learn" style="font-size: 80%;">Functionality in Scikit-Learn</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs061.html#more-preprocessing" style="font-size: 80%;">More preprocessing</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs062.html#frequently-used-scaling-functions" style="font-size: 80%;">Frequently used scaling functions</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0038"></a>
|
|
<!-- !split -->
|
|
<h2 id="adagrad-properties" class="anchor">AdaGrad Properties </h2>
|
|
|
|
<ol>
|
|
<li> AdaGrad automatically tunes the step size for each parameter. Parameters with more <em>volatile or large gradients</em> get smaller steps, and those with <em>small or infrequent gradients</em> get relatively larger steps</li>
|
|
<li> No manual schedule needed: The accumulation \( r_t \) keeps increasing (or stays the same if gradient is zero), so step sizes \( \eta/\sqrt{r_t} \) are non-increasing. This has a similar effect to a learning rate schedule, but individualized per coordinate.</li>
|
|
<li> Sparse data benefit: For very sparse features, \( r_{t,j} \) grows slowly, so that feature’s parameter retains a higher learning rate for longer, allowing it to make significant updates when it does get a gradient signal</li>
|
|
<li> Convergence: In convex optimization, AdaGrad can be shown to achieve a sub-linear convergence rate comparable to the best fixed learning rate tuned for the problem</li>
|
|
</ol>
|
|
<p>It effectively reduces the need to tune \( \eta \) by hand.</p>
|
|
<ol>
|
|
<li> Limitations: Because \( r_t \) accumulates without bound, AdaGrad’s learning rates can become extremely small over long training, potentially slowing progress. (Later variants like RMSProp, AdaDelta, Adam address this by modifying the accumulation rule.)</li>
|
|
</ol>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week37-bs037.html">«</a></li>
|
|
<li><a href="._week37-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs030.html">31</a></li>
|
|
<li><a href="._week37-bs031.html">32</a></li>
|
|
<li><a href="._week37-bs032.html">33</a></li>
|
|
<li><a href="._week37-bs033.html">34</a></li>
|
|
<li><a href="._week37-bs034.html">35</a></li>
|
|
<li><a href="._week37-bs035.html">36</a></li>
|
|
<li><a href="._week37-bs036.html">37</a></li>
|
|
<li><a href="._week37-bs037.html">38</a></li>
|
|
<li class="active"><a href="._week37-bs038.html">39</a></li>
|
|
<li><a href="._week37-bs039.html">40</a></li>
|
|
<li><a href="._week37-bs040.html">41</a></li>
|
|
<li><a href="._week37-bs041.html">42</a></li>
|
|
<li><a href="._week37-bs042.html">43</a></li>
|
|
<li><a href="._week37-bs043.html">44</a></li>
|
|
<li><a href="._week37-bs044.html">45</a></li>
|
|
<li><a href="._week37-bs045.html">46</a></li>
|
|
<li><a href="._week37-bs046.html">47</a></li>
|
|
<li><a href="._week37-bs047.html">48</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs062.html">63</a></li>
|
|
<li><a href="._week37-bs039.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|