Morten Hjorth-Jensen
461 lines
29 KiB
HTML
461 lines
29 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week40.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week40-bs --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 40: Gradient descent methods (continued) and start Neural networks">
|
|
<title>Week 40: Gradient descent methods (continued) and start Neural networks</title>
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<!-- doconce format html week40.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week40-bs --no_mako -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
<style type="text/css">
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Plans for week 40', 2, None, 'plans-for-week-40'),
|
|
('Lecture Monday September 30, 2024',
|
|
2,
|
|
None,
|
|
'lecture-monday-september-30-2024'),
|
|
('Suggested readings and videos',
|
|
2,
|
|
None,
|
|
'suggested-readings-and-videos'),
|
|
('Lab sessions Tuesday and Wednesday',
|
|
2,
|
|
None,
|
|
'lab-sessions-tuesday-and-wednesday'),
|
|
('Summary from last week, using gradient descent methods, '
|
|
'limitations',
|
|
2,
|
|
None,
|
|
'summary-from-last-week-using-gradient-descent-methods-limitations'),
|
|
('Simple implementation of GD for OLS, Ridge and Lasso',
|
|
2,
|
|
None,
|
|
'simple-implementation-of-gd-for-ols-ridge-and-lasso'),
|
|
("But none of these can compete with Newton's method",
|
|
2,
|
|
None,
|
|
'but-none-of-these-can-compete-with-newton-s-method'),
|
|
('Gradient descent and Logistic regression',
|
|
2,
|
|
None,
|
|
'gradient-descent-and-logistic-regression'),
|
|
('Overview video on Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'overview-video-on-stochastic-gradient-descent'),
|
|
('Batches and mini-batches', 2, None, 'batches-and-mini-batches'),
|
|
('Stochastic Gradient Descent (SGD)',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent-sgd'),
|
|
('Stochastic Gradient Descent',
|
|
2,
|
|
None,
|
|
'stochastic-gradient-descent'),
|
|
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
|
('SGD example', 2, None, 'sgd-example'),
|
|
('The gradient step', 2, None, 'the-gradient-step'),
|
|
('Simple example code', 2, None, 'simple-example-code'),
|
|
('When do we stop?', 2, None, 'when-do-we-stop'),
|
|
('Slightly different approach',
|
|
2,
|
|
None,
|
|
'slightly-different-approach'),
|
|
('Time decay rate', 2, None, 'time-decay-rate'),
|
|
('Code with a Number of Minibatches which varies',
|
|
2,
|
|
None,
|
|
'code-with-a-number-of-minibatches-which-varies'),
|
|
('Replace or not', 2, None, 'replace-or-not'),
|
|
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
|
('More on momentum based approaches',
|
|
2,
|
|
None,
|
|
'more-on-momentum-based-approaches'),
|
|
('Momentum parameter', 2, None, 'momentum-parameter'),
|
|
('Second moment of the gradient',
|
|
2,
|
|
None,
|
|
'second-moment-of-the-gradient'),
|
|
('RMS prop', 2, None, 'rms-prop'),
|
|
('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"',
|
|
2,
|
|
None,
|
|
'adam-optimizer-https-arxiv-org-abs-1412-6980'),
|
|
('Algorithms and codes for Adagrad, RMSprop and Adam',
|
|
2,
|
|
None,
|
|
'algorithms-and-codes-for-adagrad-rmsprop-and-adam'),
|
|
('AdaGrad algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('RMSProp algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('ADAM algorithm, taken from "Goodfellow et '
|
|
'al":"https://www.deeplearningbook.org/contents/optimization.html"',
|
|
2,
|
|
None,
|
|
'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'),
|
|
('Practical tips', 2, None, 'practical-tips'),
|
|
('Automatic differentiation',
|
|
2,
|
|
None,
|
|
'automatic-differentiation'),
|
|
('Using autograd', 2, None, 'using-autograd'),
|
|
('Autograd with more complicated functions',
|
|
2,
|
|
None,
|
|
'autograd-with-more-complicated-functions'),
|
|
('More complicated functions using the elements of their '
|
|
'arguments directly',
|
|
2,
|
|
None,
|
|
'more-complicated-functions-using-the-elements-of-their-arguments-directly'),
|
|
('Functions using mathematical functions from Numpy',
|
|
2,
|
|
None,
|
|
'functions-using-mathematical-functions-from-numpy'),
|
|
('More autograd', 2, None, 'more-autograd'),
|
|
('And with loops', 2, None, 'and-with-loops'),
|
|
('Using recursion', 2, None, 'using-recursion'),
|
|
('Using Autograd with OLS', 2, None, 'using-autograd-with-ols'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Including Stochastic Gradient Descent with Autograd',
|
|
2,
|
|
None,
|
|
'including-stochastic-gradient-descent-with-autograd'),
|
|
('Same code but now with momentum gradient descent',
|
|
2,
|
|
None,
|
|
'same-code-but-now-with-momentum-gradient-descent'),
|
|
('Similar (second order function now) problem but now with '
|
|
'AdaGrad',
|
|
2,
|
|
None,
|
|
'similar-second-order-function-now-problem-but-now-with-adagrad'),
|
|
('RMSprop for adaptive learning rate with Stochastic Gradient '
|
|
'Descent',
|
|
2,
|
|
None,
|
|
'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'),
|
|
('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"',
|
|
2,
|
|
None,
|
|
'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'),
|
|
('And Logistic Regression', 2, None, 'and-logistic-regression'),
|
|
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
|
|
2,
|
|
None,
|
|
'introducing-jax-https-jax-readthedocs-io-en-latest'),
|
|
('Getting started with Jax, note the way we import numpy',
|
|
3,
|
|
None,
|
|
'getting-started-with-jax-note-the-way-we-import-numpy'),
|
|
('A warm-up example', 3, None, 'a-warm-up-example'),
|
|
('A more advanced example', 3, None, 'a-more-advanced-example'),
|
|
('Introduction to Neural networks',
|
|
2,
|
|
None,
|
|
'introduction-to-neural-networks'),
|
|
('Artificial neurons', 2, None, 'artificial-neurons'),
|
|
('Neural network types', 2, None, 'neural-network-types'),
|
|
('Feed-forward neural networks',
|
|
2,
|
|
None,
|
|
'feed-forward-neural-networks'),
|
|
('Convolutional Neural Network',
|
|
2,
|
|
None,
|
|
'convolutional-neural-network'),
|
|
('Recurrent neural networks',
|
|
2,
|
|
None,
|
|
'recurrent-neural-networks'),
|
|
('Other types of networks', 2, None, 'other-types-of-networks'),
|
|
('Multilayer perceptrons', 2, None, 'multilayer-perceptrons'),
|
|
('Why multilayer perceptrons?',
|
|
2,
|
|
None,
|
|
'why-multilayer-perceptrons'),
|
|
('Illustration of a single perceptron model and a '
|
|
'multi-perceptron model',
|
|
2,
|
|
None,
|
|
'illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model'),
|
|
('Examples of XOR, OR and AND gates',
|
|
2,
|
|
None,
|
|
'examples-of-xor-or-and-and-gates'),
|
|
('Does Logistic Regression do a better Job?',
|
|
2,
|
|
None,
|
|
'does-logistic-regression-do-a-better-job'),
|
|
('Adding Neural Networks', 2, None, 'adding-neural-networks'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Mathematical model', 2, None, 'mathematical-model'),
|
|
('Matrix-vector notation', 3, None, 'matrix-vector-notation'),
|
|
('Matrix-vector notation and activation',
|
|
3,
|
|
None,
|
|
'matrix-vector-notation-and-activation'),
|
|
('Activation functions', 3, None, 'activation-functions'),
|
|
('Activation functions, Logistic and Hyperbolic ones',
|
|
3,
|
|
None,
|
|
'activation-functions-logistic-and-hyperbolic-ones'),
|
|
('Relevance', 3, None, 'relevance')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="week40-bs.html">Week 40: Gradient descent methods (continued) and start Neural networks</a>
|
|
</div>
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._week40-bs001.html#plans-for-week-40" style="font-size: 80%;"><b>Plans for week 40</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs002.html#lecture-monday-september-30-2024" style="font-size: 80%;"><b>Lecture Monday September 30, 2024</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs003.html#suggested-readings-and-videos" style="font-size: 80%;"><b>Suggested readings and videos</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs004.html#lab-sessions-tuesday-and-wednesday" style="font-size: 80%;"><b>Lab sessions Tuesday and Wednesday</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs005.html#summary-from-last-week-using-gradient-descent-methods-limitations" style="font-size: 80%;"><b>Summary from last week, using gradient descent methods, limitations</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs006.html#simple-implementation-of-gd-for-ols-ridge-and-lasso" style="font-size: 80%;"><b>Simple implementation of GD for OLS, Ridge and Lasso</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs007.html#but-none-of-these-can-compete-with-newton-s-method" style="font-size: 80%;"><b>But none of these can compete with Newton's method</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs008.html#gradient-descent-and-logistic-regression" style="font-size: 80%;"><b>Gradient descent and Logistic regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs009.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;"><b>Overview video on Stochastic Gradient Descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs010.html#batches-and-mini-batches" style="font-size: 80%;"><b>Batches and mini-batches</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs011.html#stochastic-gradient-descent-sgd" style="font-size: 80%;"><b>Stochastic Gradient Descent (SGD)</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs012.html#stochastic-gradient-descent" style="font-size: 80%;"><b>Stochastic Gradient Descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs013.html#computation-of-gradients" style="font-size: 80%;"><b>Computation of gradients</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs014.html#sgd-example" style="font-size: 80%;"><b>SGD example</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs015.html#the-gradient-step" style="font-size: 80%;"><b>The gradient step</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs016.html#simple-example-code" style="font-size: 80%;"><b>Simple example code</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs017.html#when-do-we-stop" style="font-size: 80%;"><b>When do we stop?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs018.html#slightly-different-approach" style="font-size: 80%;"><b>Slightly different approach</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs019.html#time-decay-rate" style="font-size: 80%;"><b>Time decay rate</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs020.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;"><b>Code with a Number of Minibatches which varies</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs021.html#replace-or-not" style="font-size: 80%;"><b>Replace or not</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs022.html#momentum-based-gd" style="font-size: 80%;"><b>Momentum based GD</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs023.html#more-on-momentum-based-approaches" style="font-size: 80%;"><b>More on momentum based approaches</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="#momentum-parameter" style="font-size: 80%;"><b>Momentum parameter</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs025.html#second-moment-of-the-gradient" style="font-size: 80%;"><b>Second moment of the gradient</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs026.html#rms-prop" style="font-size: 80%;"><b>RMS prop</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs027.html#adam-optimizer-https-arxiv-org-abs-1412-6980" style="font-size: 80%;"><b>"ADAM optimizer":"https://arxiv.org/abs/1412.6980"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs028.html#algorithms-and-codes-for-adagrad-rmsprop-and-adam" style="font-size: 80%;"><b>Algorithms and codes for Adagrad, RMSprop and Adam</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs028.html#adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;"><b>AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs028.html#rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;"><b>RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs028.html#adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html" style="font-size: 80%;"><b>ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs029.html#practical-tips" style="font-size: 80%;"><b>Practical tips</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs030.html#automatic-differentiation" style="font-size: 80%;"><b>Automatic differentiation</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs031.html#using-autograd" style="font-size: 80%;"><b>Using autograd</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs032.html#autograd-with-more-complicated-functions" style="font-size: 80%;"><b>Autograd with more complicated functions</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs033.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;"><b>More complicated functions using the elements of their arguments directly</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs034.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;"><b>Functions using mathematical functions from Numpy</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs035.html#more-autograd" style="font-size: 80%;"><b>More autograd</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs036.html#and-with-loops" style="font-size: 80%;"><b>And with loops</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs037.html#using-recursion" style="font-size: 80%;"><b>Using recursion</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs038.html#using-autograd-with-ols" style="font-size: 80%;"><b>Using Autograd with OLS</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs041.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;"><b>Same code but now with momentum gradient descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs040.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;"><b>Including Stochastic Gradient Descent with Autograd</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs041.html#same-code-but-now-with-momentum-gradient-descent" style="font-size: 80%;"><b>Same code but now with momentum gradient descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs042.html#similar-second-order-function-now-problem-but-now-with-adagrad" style="font-size: 80%;"><b>Similar (second order function now) problem but now with AdaGrad</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs043.html#rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent" style="font-size: 80%;"><b>RMSprop for adaptive learning rate with Stochastic Gradient Descent</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs044.html#and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf" style="font-size: 80%;"><b>And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs045.html#and-logistic-regression" style="font-size: 80%;"><b>And Logistic Regression</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs045.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;"><b>Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs045.html#getting-started-with-jax-note-the-way-we-import-numpy" style="font-size: 80%;"> Getting started with Jax, note the way we import numpy</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs045.html#a-warm-up-example" style="font-size: 80%;"> A warm-up example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs045.html#a-more-advanced-example" style="font-size: 80%;"> A more advanced example</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs046.html#introduction-to-neural-networks" style="font-size: 80%;"><b>Introduction to Neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs047.html#artificial-neurons" style="font-size: 80%;"><b>Artificial neurons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs048.html#neural-network-types" style="font-size: 80%;"><b>Neural network types</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs049.html#feed-forward-neural-networks" style="font-size: 80%;"><b>Feed-forward neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs050.html#convolutional-neural-network" style="font-size: 80%;"><b>Convolutional Neural Network</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs051.html#recurrent-neural-networks" style="font-size: 80%;"><b>Recurrent neural networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs052.html#other-types-of-networks" style="font-size: 80%;"><b>Other types of networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs053.html#multilayer-perceptrons" style="font-size: 80%;"><b>Multilayer perceptrons</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs054.html#why-multilayer-perceptrons" style="font-size: 80%;"><b>Why multilayer perceptrons?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs055.html#illustration-of-a-single-perceptron-model-and-a-multi-perceptron-model" style="font-size: 80%;"><b>Illustration of a single perceptron model and a multi-perceptron model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs056.html#examples-of-xor-or-and-and-gates" style="font-size: 80%;"><b>Examples of XOR, OR and AND gates</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs057.html#does-logistic-regression-do-a-better-job" style="font-size: 80%;"><b>Does Logistic Regression do a better Job?</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs058.html#adding-neural-networks" style="font-size: 80%;"><b>Adding Neural Networks</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs063.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs063.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs063.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs063.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs063.html#mathematical-model" style="font-size: 80%;"><b>Mathematical model</b></a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs064.html#matrix-vector-notation" style="font-size: 80%;"> Matrix-vector notation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs065.html#matrix-vector-notation-and-activation" style="font-size: 80%;"> Matrix-vector notation and activation</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs066.html#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs067.html#activation-functions-logistic-and-hyperbolic-ones" style="font-size: 80%;"> Activation functions, Logistic and Hyperbolic ones</a></li>
|
|
<!-- navigation toc: --> <li><a href="._week40-bs068.html#relevance" style="font-size: 80%;"> Relevance</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
<div class="container">
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
<a name="part0024"></a>
|
|
<!-- !split -->
|
|
<h2 id="momentum-parameter" class="anchor">Momentum parameter </h2>
|
|
|
|
<p>Notice that this equation is identical to previous one if we identify
|
|
the position of the particle, \( \mathbf{w} \), with the parameters
|
|
\( \boldsymbol{\theta} \). This allows us to identify the momentum
|
|
parameter and learning rate with the mass of the particle and the
|
|
viscous drag as:
|
|
</p>
|
|
|
|
$$
|
|
\gamma= {m \over m +\mu \Delta t }, \qquad \eta = {(\Delta t)^2 \over m +\mu \Delta t}.
|
|
$$
|
|
|
|
<p>Thus, as the name suggests, the momentum parameter is proportional to
|
|
the mass of the particle and effectively provides inertia.
|
|
Furthermore, in the large viscosity/small learning rate limit, our
|
|
memory time scales as \( (1-\gamma)^{-1} \approx m/(\mu \Delta t) \).
|
|
</p>
|
|
|
|
<p>Why is momentum useful? SGD momentum helps the gradient descent
|
|
algorithm gain speed in directions with persistent but small gradients
|
|
even in the presence of stochasticity, while suppressing oscillations
|
|
in high-curvature directions. This becomes especially important in
|
|
situations where the landscape is shallow and flat in some directions
|
|
and narrow and steep in others. It has been argued that first-order
|
|
methods (with appropriate initial conditions) can perform comparable
|
|
to more expensive second order methods, especially in the context of
|
|
complex deep learning models.
|
|
</p>
|
|
|
|
<p>These beneficial properties of momentum can sometimes become even more
|
|
pronounced by using a slight modification of the classical momentum
|
|
algorithm called Nesterov Accelerated Gradient (NAG).
|
|
</p>
|
|
|
|
<p>In the NAG algorithm, rather than calculating the gradient at the
|
|
current parameters, \( \nabla_\theta E(\boldsymbol{\theta}_t) \), one
|
|
calculates the gradient at the expected value of the parameters given
|
|
our current momentum, \( \nabla_\theta E(\boldsymbol{\theta}_t +\gamma
|
|
\mathbf{v}_{t-1}) \). This yields the NAG update rule
|
|
</p>
|
|
|
|
$$
|
|
\begin{align}
|
|
\mathbf{v}_{t}&=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t +\gamma \mathbf{v}_{t-1}) \nonumber \\
|
|
\boldsymbol{\theta}_{t+1}&= \boldsymbol{\theta}_t -\mathbf{v}_{t}.
|
|
\tag{2}
|
|
\end{align}
|
|
$$
|
|
|
|
<p>One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of \( \gamma \).</p>
|
|
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._week40-bs023.html">«</a></li>
|
|
<li><a href="._week40-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week40-bs016.html">17</a></li>
|
|
<li><a href="._week40-bs017.html">18</a></li>
|
|
<li><a href="._week40-bs018.html">19</a></li>
|
|
<li><a href="._week40-bs019.html">20</a></li>
|
|
<li><a href="._week40-bs020.html">21</a></li>
|
|
<li><a href="._week40-bs021.html">22</a></li>
|
|
<li><a href="._week40-bs022.html">23</a></li>
|
|
<li><a href="._week40-bs023.html">24</a></li>
|
|
<li class="active"><a href="._week40-bs024.html">25</a></li>
|
|
<li><a href="._week40-bs025.html">26</a></li>
|
|
<li><a href="._week40-bs026.html">27</a></li>
|
|
<li><a href="._week40-bs027.html">28</a></li>
|
|
<li><a href="._week40-bs028.html">29</a></li>
|
|
<li><a href="._week40-bs029.html">30</a></li>
|
|
<li><a href="._week40-bs030.html">31</a></li>
|
|
<li><a href="._week40-bs031.html">32</a></li>
|
|
<li><a href="._week40-bs032.html">33</a></li>
|
|
<li><a href="._week40-bs033.html">34</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._week40-bs068.html">69</a></li>
|
|
<li><a href="._week40-bs025.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
|
</footer>
|
|
-->
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|