Morten Hjorth-Jensen
401 lines
26 KiB
HTML
401 lines
26 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 37: Statistical interpretations and Resampling Methods">
|
|
<title>Week 37: Statistical interpretations and Resampling Methods</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week37-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 37, lecture Monday',
|
|
2,
|
|
None,
|
|
'plans-for-week-37-lecture-monday'),
|
|
('Readings and Videos:', 2, None, 'readings-and-videos'),
|
|
('Material for lecture Monday September 8',
|
|
2,
|
|
None,
|
|
'material-for-lecture-monday-september-8'),
|
|
('Gradient descent and revisiting Ordinary Least Squares from '
|
|
'last week',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-revisiting-ordinary-least-squares-from-last-week'),
|
|
('Gradient descent example', 2, None, 'gradient-descent-example'),
|
|
('The derivative of the cost/loss function',
|
|
2,
|
|
None,
|
|
'the-derivative-of-the-cost-loss-function'),
|
|
('The Hessian matrix', 2, None, 'the-hessian-matrix'),
|
|
('Simple program', 2, None, 'simple-program'),
|
|
('Gradient Descent Example', 2, None, 'gradient-descent-example'),
|
|
('Gradient descent and Ridge',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-ridge'),
|
|
('The Hessian matrix for Ridge Regression',
|
|
2,
|
|
None,
|
|
'the-hessian-matrix-for-ridge-regression'),
|
|
('Program example for gradient descent with Ridge Regression',
|
|
2,
|
|
None,
|
|
'program-example-for-gradient-descent-with-ridge-regression'),
|
|
('Using gradient descent methods, limitations',
|
|
2,
|
|
None,
|
|
'using-gradient-descent-methods-limitations'),
|
|
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
|
('Improving gradient descent with momentum',
|
|
2,
|
|
None,
|
|
'improving-gradient-descent-with-momentum'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Overview video on Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'overview-video-on-stochastic-gradient-descent-sgd'),
|
|
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
|
('Pros and cons', 2, None, 'pros-and-cons'),
|
|
('Convergence rates', 2, None, 'convergence-rates'),
|
|
('Accuracy', 2, None, 'accuracy'),
|
|
('Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent-sgd'),
|
|
('Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent'),
|
|
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
|
('SGD example', 2, None, 'sgd-example'),
|
|
('The gradient step', 2, None, 'the-gradient-step'),
|
|
('Simple example code', 2, None, 'simple-example-code'),
|
|
('When do we stop?', 2, None, 'when-do-we-stop'),
|
|
('Slightly different approach',
|
|
2,
|
|
None,
|
|
'slightly-different-approach'),
|
|
('Time decay rate', 2, None, 'time-decay-rate'),
|
|
('Code with a Number of Minibatches which varies',
|
|
2,
|
|
None,
|
|
'code-with-a-number-of-minibatches-which-varies'),
|
|
('Replace or not', 2, None, 'replace-or-not'),
|
|
('Second moment of the gradient',
|
|
2,
|
|
None,
|
|
'second-moment-of-the-gradient'),
|
|
('Challenge: Choosing a Fixed Learning Rate',
|
|
2,
|
|
None,
|
|
'challenge-choosing-a-fixed-learning-rate'),
|
|
('Motivation for Adaptive Step Sizes',
|
|
2,
|
|
None,
|
|
'motivation-for-adaptive-step-sizes'),
|
|
('AdaGrad algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Derivation of the AdaGrad Algorithm',
|
|
2,
|
|
None,
|
|
'derivation-of-the-adagrad-algorithm'),
|
|
('AdaGrad Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adagrad-update-rule-derivation'),
|
|
('AdaGrad Properties', 2, None, 'adagrad-properties'),
|
|
('RMSProp: Adaptive Learning Rates',
|
|
2,
|
|
None,
|
|
'rmsprop-adaptive-learning-rates'),
|
|
('RMSProp algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Adam Optimizer', 2, None, 'adam-optimizer'),
|
|
('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
|
|
2,
|
|
None,
|
|
'adam-optimizer-https-arxiv-org-abs-1412-6980'),
|
|
('Why Combine Momentum and RMSProp?',
|
|
2,
|
|
None,
|
|
'why-combine-momentum-and-rmsprop'),
|
|
('Adam: Exponential Moving Averages (Moments)',
|
|
2,
|
|
None,
|
|
'adam-exponential-moving-averages-moments'),
|
|
('Adam: Bias Correction', 2, None, 'adam-bias-correction'),
|
|
('Adam: Update Rule Derivation',
|
|
2,
|
|
None,
|
|
'adam-update-rule-derivation'),
|
|
('Adam vs. AdaGrad and RMSProp',
|
|
2,
|
|
None,
|
|
'adam-vs-adagrad-and-rmsprop'),
|
|
('Adaptivity Across Dimensions',
|
|
2,
|
|
None,
|
|
'adaptivity-across-dimensions'),
|
|
('ADAM algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Algorithms and codes for Adagrad, RMSprop and Adam',
|
|
2,
|
|
None,
|
|
'algorithms-and-codes-for-adagrad-rmsprop-and-adam'),
|
|
('Practical tips', 2, None, 'practical-tips'),
|
|
('Sneaking in automatic differentiation using Autograd',
|
|
2,
|
|
None,
|
|
'sneaking-in-automatic-differentiation-using-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Including Stochastic Gradient Descent with Autograd',
|
|
2,
|
|
None,
|
|
'including-stochastic-gradient-descent-with-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
("But none of these can compete with Newton's method",
|
|
2,
|
|
None,
|
|
'but-none-of-these-can-compete-with-newton-s-method'),
|
|
('Similar (second order function now) problem but now with '
|
|
'AdaGrad',
|
|
2,
|
|
None,
|
|
'similar-second-order-function-now-problem-but-now-with-adagrad'),
|
|
('RMSprop for adaptive learning rate with Stochastic Gradient '
|
|
'Descent',
|
|
2,
|
|
None,
|
|
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
|
|
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
|
|
2,
|
|
None,
|
|
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
|
|
('Material for the lab sessions',
|
|
2,
|
|
None,
|
|
'material-for-the-lab-sessions'),
|
|
('Reminder on different scaling methods',
|
|
2,
|
|
None,
|
|
'reminder-on-different-scaling-methods'),
|
|
('Functionality in Scikit-Learn',
|
|
2,
|
|
None,
|
|
'functionality-in-scikit-learn'),
|
|
('More preprocessing', 2, None, 'more-preprocessing'),
|
|
('Frequently used scaling functions',
|
|
2,
|
|
None,
|
|
'frequently-used-scaling-functions')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week37-bs.html">Week 37: Statistical interpretations and Resampling Methods</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week37-bs001.html#plans-for-week-37-lecture-monday" style="font-size: 80%;">Plans for week 37, lecture Monday</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs002.html#readings-and-videos" style="font-size: 80%;">Readings and Videos:</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs003.html#material-for-lecture-monday-september-8" style="font-size: 80%;">Material for lecture Monday September 8</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs004.html#gradient-descent-and-revisiting-ordinary-least-squares-from-last-week" style="font-size: 80%;">Gradient descent and revisiting Ordinary Least Squares from last week</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#gradient-descent-example" style="font-size: 80%;">Gradient descent example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs006.html#the-derivative-of-the-cost-loss-function" style="font-size: 80%;">The derivative of the cost/loss function</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs007.html#the-hessian-matrix" style="font-size: 80%;">The Hessian matrix</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs008.html#simple-program" style="font-size: 80%;">Simple program</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs009.html#gradient-descent-example" style="font-size: 80%;">Gradient Descent Example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs010.html#gradient-descent-and-ridge" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs011.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;">The Hessian matrix for Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs012.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs013.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs014.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs015.html#improving-gradient-descent-with-momentum" style="font-size: 80%;">Improving gradient descent with momentum</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs017.html#overview-video-on-stochastic-gradient-descent-sgd" style="font-size: 80%;">Overview video on Stochastic Gradient Descent (SGD)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs018.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs019.html#pros-and-cons" style="font-size: 80%;">Pros and cons</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs020.html#convergence-rates" style="font-size: 80%;">Convergence rates</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs021.html#accuracy" style="font-size: 80%;">Accuracy</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs022.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs023.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs024.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs025.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs026.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs027.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs028.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs029.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs030.html#time-decay-rate" style="font-size: 80%;">Time decay rate</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs031.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs032.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs033.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs034.html#challenge-choosing-a-fixed-learning-rate" style="font-size: 80%;">Challenge: Choosing a Fixed Learning Rate</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#motivation-for-adaptive-step-sizes" style="font-size: 80%;">Motivation for Adaptive Step Sizes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs035.html#adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs036.html#derivation-of-the-adagrad-algorithm" style="font-size: 80%;">Derivation of the AdaGrad Algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs037.html#adagrad-update-rule-derivation" style="font-size: 80%;">AdaGrad Update Rule Derivation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs038.html#adagrad-properties" style="font-size: 80%;">AdaGrad Properties</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs039.html#rmsprop-adaptive-learning-rates" style="font-size: 80%;">RMSProp: Adaptive Learning Rates</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs039.html#rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs040.html#adam-optimizer" style="font-size: 80%;">Adam Optimizer</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs041.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;">"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs042.html#why-combine-momentum-and-rmsprop" style="font-size: 80%;">Why Combine Momentum and RMSProp?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs043.html#adam-exponential-moving-averages-moments" style="font-size: 80%;">Adam: Exponential Moving Averages (Moments)</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs044.html#adam-bias-correction" style="font-size: 80%;">Adam: Bias Correction</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs045.html#adam-update-rule-derivation" style="font-size: 80%;">Adam: Update Rule Derivation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs046.html#adam-vs-adagrad-and-rmsprop" style="font-size: 80%;">Adam vs. AdaGrad and RMSProp</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs047.html#adaptivity-across-dimensions" style="font-size: 80%;">Adaptivity Across Dimensions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs047.html#adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;">ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs048.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;">Algorithms and codes for Adagrad, RMSprop and Adam</a></li>
|
|
<!-- navigation toc: --> <li><a href="#practical-tips" style="font-size: 80%;">Practical tips</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs050.html#sneaking-in-automatic-differentiation-using-autograd" style="font-size: 80%;">Sneaking in automatic differentiation using Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs052.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs053.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;">Same code but now with momentum gradient descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs054.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;">But none of these can compete with Newton's method</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs055.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;">Similar (second order function now) problem but now with AdaGrad</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs056.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;">RMSprop for adaptive learning rate with Stochastic Gradient Descent</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs057.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;">And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs058.html#material-for-the-lab-sessions" style="font-size: 80%;">Material for the lab sessions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs059.html#reminder-on-different-scaling-methods" style="font-size: 80%;">Reminder on different scaling methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs060.html#functionality-in-scikit-learn" style="font-size: 80%;">Functionality in Scikit-Learn</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs061.html#more-preprocessing" style="font-size: 80%;">More preprocessing</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week37-bs062.html#frequently-used-scaling-functions" style="font-size: 80%;">Frequently used scaling functions</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0049"></a>
|
|
<!-- !split -->
|
|
<h2 id="practical-tips" class="anchor">Practical tips </h2>
|
|
|
|
<ul>
|
|
<li> <b>Randomize the data when making mini-batches</b>. It is always important to randomly shuffle the data when forming mini-batches. Otherwise, the gradient descent method can fit spurious correlations resulting from the order in which data is presented.</li>
|
|
<li> <b>Transform your inputs</b>. Learning becomes difficult when our landscape has a mixture of steep and flat directions. One simple trick for minimizing these situations is to standardize the data by subtracting the mean and normalizing the variance of input variables. Whenever possible, also decorrelate the inputs. To understand why this is helpful, consider the case of linear regression. It is easy to show that for the squared error cost function, the Hessian of the cost function is just the correlation matrix between the inputs. Thus, by standardizing the inputs, we are ensuring that the landscape looks homogeneous in all directions in parameter space. Since most deep networks can be viewed as linear transformations followed by a non-linearity at each layer, we expect this intuition to hold beyond the linear case.</li>
|
|
<li> <b>Monitor the out-of-sample performance.</b> Always monitor the performance of your model on a validation set (a small portion of the training data that is held out of the training process to serve as a proxy for the test set. If the validation error starts increasing, then the model is beginning to overfit. Terminate the learning process. This <em>early stopping</em> significantly improves performance in many settings.</li>
|
|
<li> <b>Adaptive optimization methods don't always have good generalization.</b> Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications.</li>
|
|
</ul>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week37-bs048.html">«</a></li>
|
|
<li><a href="._week37-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs041.html">42</a></li>
|
|
<li><a href="._week37-bs042.html">43</a></li>
|
|
<li><a href="._week37-bs043.html">44</a></li>
|
|
<li><a href="._week37-bs044.html">45</a></li>
|
|
<li><a href="._week37-bs045.html">46</a></li>
|
|
<li><a href="._week37-bs046.html">47</a></li>
|
|
<li><a href="._week37-bs047.html">48</a></li>
|
|
<li><a href="._week37-bs048.html">49</a></li>
|
|
<li class="active"><a href="._week37-bs049.html">50</a></li>
|
|
<li><a href="._week37-bs050.html">51</a></li>
|
|
<li><a href="._week37-bs051.html">52</a></li>
|
|
<li><a href="._week37-bs052.html">53</a></li>
|
|
<li><a href="._week37-bs053.html">54</a></li>
|
|
<li><a href="._week37-bs054.html">55</a></li>
|
|
<li><a href="._week37-bs055.html">56</a></li>
|
|
<li><a href="._week37-bs056.html">57</a></li>
|
|
<li><a href="._week37-bs057.html">58</a></li>
|
|
<li><a href="._week37-bs058.html">59</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week37-bs062.html">63</a></li>
|
|
<li><a href="._week37-bs050.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|