Merge branch 'master' of https://github.com/CompPhysics/MachineLearning
This commit is contained in:
Morten Hjorth-Jensen
@@ -37,7 +37,6 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 39', 2, None, 'plan-for-week-39'),
|
||||
('Thursday September 29', 2, None, 'thursday-september-29'),
|
||||
('Optimization, the central part of any Machine Learning '
|
||||
'algortithm',
|
||||
2,
|
||||
@@ -167,11 +166,6 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
@@ -180,26 +174,11 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Time decay rate', 2, None, 'time-decay-rate'),
|
||||
('Code with a Number of Minibatches which varies',
|
||||
2,
|
||||
None,
|
||||
'code-with-a-number-of-minibatches-which-varies'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Replace or not', 2, None, 'replace-or-not'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
@@ -246,7 +225,11 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d
|
||||
2,
|
||||
None,
|
||||
'including-stochastic-gradient-descent-with-autograd'),
|
||||
('And Logistic Regression', 2, None, 'and-logistic-regression')]}
|
||||
('And Logistic Regression', 2, None, 'and-logistic-regression'),
|
||||
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
|
||||
2,
|
||||
None,
|
||||
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -282,91 +265,83 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs001.html#plan-for-week-39" style="font-size: 80%;">Plan for week 39</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs002.html#thursday-september-29" style="font-size: 80%;">Thursday September 29</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs003.html#optimization-the-central-part-of-any-machine-learning-algortithm" style="font-size: 80%;">Optimization, the central part of any Machine Learning algortithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs004.html#revisiting-our-logistic-regression-case" style="font-size: 80%;">Revisiting our Logistic Regression case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs005.html#the-equations-to-solve" style="font-size: 80%;">The equations to solve</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs006.html#solving-using-newton-raphson-s-method" style="font-size: 80%;">Solving using Newton-Raphson's method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs007.html#brief-reminder-on-newton-raphson-s-method" style="font-size: 80%;">Brief reminder on Newton-Raphson's method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs008.html#the-equations" style="font-size: 80%;">The equations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs009.html#simple-geometric-interpretation" style="font-size: 80%;">Simple geometric interpretation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs010.html#extending-to-more-than-one-variable" style="font-size: 80%;">Extending to more than one variable</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs011.html#steepest-descent" style="font-size: 80%;">Steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs012.html#more-on-steepest-descent" style="font-size: 80%;">More on Steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs013.html#the-ideal" style="font-size: 80%;">The ideal</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs014.html#the-sensitiveness-of-the-gradient-descent" style="font-size: 80%;">The sensitiveness of the gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs015.html#convex-functions" style="font-size: 80%;">Convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs016.html#convex-function" style="font-size: 80%;">Convex function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs017.html#conditions-on-convex-functions" style="font-size: 80%;">Conditions on convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs018.html#more-on-convex-functions" style="font-size: 80%;">More on convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs019.html#some-simple-problems" style="font-size: 80%;">Some simple problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs020.html#standard-steepest-descent" style="font-size: 80%;">Standard steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs021.html#gradient-method" style="font-size: 80%;">Gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs023.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs023.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs024.html#final-expressions" style="font-size: 80%;">Final expressions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs025.html#steepest-descent-example" style="font-size: 80%;">Steepest descent example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs030.html#conjugate-gradient-method-and-iterations" style="font-size: 80%;">Conjugate gradient method and iterations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs034.html#revisiting-our-first-homework" style="font-size: 80%;">Revisiting our first homework</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs039.html#gradient-descent-example" style="font-size: 80%;">Gradient descent example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs036.html#the-derivative-of-the-cost-loss-function" style="font-size: 80%;">The derivative of the cost/loss function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs037.html#the-hessian-matrix" style="font-size: 80%;">The Hessian matrix</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs038.html#simple-program" style="font-size: 80%;">Simple program</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs039.html#gradient-descent-example" style="font-size: 80%;">Gradient Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs040.html#and-a-corresponding-example-using-scikit-learn" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs041.html#gradient-descent-and-ridge" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs042.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;">The Hessian matrix for Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs043.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs044.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs045.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;">Overview video on Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs046.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs047.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs050.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs051.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs050.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs051.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs059.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs060.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs061.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs062.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs063.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs064.html#program-for-stochastic-gradient" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs058.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs059.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs060.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs061.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs062.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs063.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs064.html#program-for-stochastic-gradient" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs065.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs066.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs067.html#more-on-momentum-based-approaches" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs068.html#momentum-parameter" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs069.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs070.html#rms-prop" style="font-size: 80%;">RMS prop</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs071.html#adam-optimizer" style="font-size: 80%;">ADAM optimizer</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs072.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs073.html#automatic-differentiation" style="font-size: 80%;">Automatic differentiation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs074.html#using-autograd" style="font-size: 80%;">Using autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs075.html#autograd-with-more-complicated-functions" style="font-size: 80%;">Autograd with more complicated functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs076.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;">More complicated functions using the elements of their arguments directly</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs077.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;">Functions using mathematical functions from Numpy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs078.html#more-autograd" style="font-size: 80%;">More autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs079.html#and-with-loops" style="font-size: 80%;">And with loops</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs080.html#using-recursion" style="font-size: 80%;">Using recursion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs081.html#unsupported-functions" style="font-size: 80%;">Unsupported functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs082.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;">The syntax a.dot(b) when finding the dot product</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs083.html#recommended-to-avoid" style="font-size: 80%;">Recommended to avoid</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs084.html#using-autograd-with-ols" style="font-size: 80%;">Using Autograd with OLS</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs085.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs086.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs002.html#optimization-the-central-part-of-any-machine-learning-algortithm" style="font-size: 80%;">Optimization, the central part of any Machine Learning algortithm</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs003.html#revisiting-our-logistic-regression-case" style="font-size: 80%;">Revisiting our Logistic Regression case</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs004.html#the-equations-to-solve" style="font-size: 80%;">The equations to solve</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs005.html#solving-using-newton-raphson-s-method" style="font-size: 80%;">Solving using Newton-Raphson's method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs006.html#brief-reminder-on-newton-raphson-s-method" style="font-size: 80%;">Brief reminder on Newton-Raphson's method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs007.html#the-equations" style="font-size: 80%;">The equations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs008.html#simple-geometric-interpretation" style="font-size: 80%;">Simple geometric interpretation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs009.html#extending-to-more-than-one-variable" style="font-size: 80%;">Extending to more than one variable</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs010.html#steepest-descent" style="font-size: 80%;">Steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs011.html#more-on-steepest-descent" style="font-size: 80%;">More on Steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs012.html#the-ideal" style="font-size: 80%;">The ideal</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs013.html#the-sensitiveness-of-the-gradient-descent" style="font-size: 80%;">The sensitiveness of the gradient descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs014.html#convex-functions" style="font-size: 80%;">Convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs015.html#convex-function" style="font-size: 80%;">Convex function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs016.html#conditions-on-convex-functions" style="font-size: 80%;">Conditions on convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs017.html#more-on-convex-functions" style="font-size: 80%;">More on convex functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs018.html#some-simple-problems" style="font-size: 80%;">Some simple problems</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs019.html#standard-steepest-descent" style="font-size: 80%;">Standard steepest descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs020.html#gradient-method" style="font-size: 80%;">Gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs022.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs022.html#steepest-descent-method" style="font-size: 80%;">Steepest descent method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs023.html#final-expressions" style="font-size: 80%;">Final expressions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs024.html#steepest-descent-example" style="font-size: 80%;">Steepest descent example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs029.html#conjugate-gradient-method-and-iterations" style="font-size: 80%;">Conjugate gradient method and iterations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs032.html#conjugate-gradient-method" style="font-size: 80%;">Conjugate gradient method</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs033.html#revisiting-our-first-homework" style="font-size: 80%;">Revisiting our first homework</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs038.html#gradient-descent-example" style="font-size: 80%;">Gradient descent example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs035.html#the-derivative-of-the-cost-loss-function" style="font-size: 80%;">The derivative of the cost/loss function</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs036.html#the-hessian-matrix" style="font-size: 80%;">The Hessian matrix</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs037.html#simple-program" style="font-size: 80%;">Simple program</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs038.html#gradient-descent-example" style="font-size: 80%;">Gradient Descent Example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs039.html#and-a-corresponding-example-using-scikit-learn" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs040.html#gradient-descent-and-ridge" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs041.html#the-hessian-matrix-for-ridge-regression" style="font-size: 80%;">The Hessian matrix for Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs042.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs043.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs044.html#overview-video-on-stochastic-gradient-descent" style="font-size: 80%;">Overview video on Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs045.html#batches-and-mini-batches" style="font-size: 80%;">Batches and mini-batches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs046.html#stochastic-gradient-descent-sgd" style="font-size: 80%;">Stochastic Gradient Descent (SGD)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs047.html#stochastic-gradient-descent" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs048.html#computation-of-gradients" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs049.html#sgd-example" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs050.html#the-gradient-step" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs051.html#simple-example-code" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs052.html#when-do-we-stop" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs053.html#slightly-different-approach" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs054.html#time-decay-rate" style="font-size: 80%;">Time decay rate</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs055.html#code-with-a-number-of-minibatches-which-varies" style="font-size: 80%;">Code with a Number of Minibatches which varies</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs056.html#replace-or-not" style="font-size: 80%;">Replace or not</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs057.html#momentum-based-gd" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs058.html#more-on-momentum-based-approaches" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs059.html#momentum-parameter" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs060.html#second-moment-of-the-gradient" style="font-size: 80%;">Second moment of the gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs061.html#rms-prop" style="font-size: 80%;">RMS prop</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs062.html#adam-optimizer" style="font-size: 80%;">ADAM optimizer</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs063.html#practical-tips" style="font-size: 80%;">Practical tips</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs064.html#automatic-differentiation" style="font-size: 80%;">Automatic differentiation</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs065.html#using-autograd" style="font-size: 80%;">Using autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs066.html#autograd-with-more-complicated-functions" style="font-size: 80%;">Autograd with more complicated functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs067.html#more-complicated-functions-using-the-elements-of-their-arguments-directly" style="font-size: 80%;">More complicated functions using the elements of their arguments directly</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs068.html#functions-using-mathematical-functions-from-numpy" style="font-size: 80%;">Functions using mathematical functions from Numpy</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs069.html#more-autograd" style="font-size: 80%;">More autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs070.html#and-with-loops" style="font-size: 80%;">And with loops</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs071.html#using-recursion" style="font-size: 80%;">Using recursion</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs072.html#unsupported-functions" style="font-size: 80%;">Unsupported functions</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs073.html#the-syntax-a-dot-b-when-finding-the-dot-product" style="font-size: 80%;">The syntax a.dot(b) when finding the dot product</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs074.html#recommended-to-avoid" style="font-size: 80%;">Recommended to avoid</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs075.html#using-autograd-with-ols" style="font-size: 80%;">Using Autograd with OLS</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs076.html#including-stochastic-gradient-descent-with-autograd" style="font-size: 80%;">Including Stochastic Gradient Descent with Autograd</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs077.html#and-logistic-regression" style="font-size: 80%;">And Logistic Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week39-bs078.html#introducing-jax-https-jax-readthedocs-io-en-latest" style="font-size: 80%;">Introducing "JAX":"https://jax.readthedocs.io/en/latest/"</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -421,7 +396,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week39-bs008.html">9</a></li>
|
||||
<li><a href="._week39-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week39-bs086.html">87</a></li>
|
||||
<li><a href="._week39-bs078.html">79</a></li>
|
||||
<li><a href="._week39-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -198,7 +198,7 @@ MathJax.Hub.Config({
|
||||
<h2 id="plan-for-week-39">Plan for week 39 </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression</li>
|
||||
<p><li> Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression and discussion of stochastic gradient descent</li>
|
||||
<p><li> Friday: Stochastic Gradient descent with examples and automatic differentiation</li>
|
||||
<p><li> Reading recommendations:</li>
|
||||
</ul>
|
||||
@@ -213,12 +213,6 @@ For a good discussion on gradient methods, we would like to recommend Goodfellow
|
||||
<p>These sections summarize neatly what we have done till now and point to what is coming with respect to deep learning. </p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="thursday-september-29">Thursday September 29 </h2>
|
||||
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/OverarchingAimsWeek39.mp4?vrtx=view-as-webpage" target="_blank">Overview Video, why do we care about gradient methods?</a>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="optimization-the-central-part-of-any-machine-learning-algortithm">Optimization, the central part of any Machine Learning algortithm </h2>
|
||||
|
||||
@@ -1718,46 +1712,6 @@ minibatches. We denote these minibatches by \( B_k \) where
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="stochastic-gradient-descent">Stochastic Gradient Descent </h2>
|
||||
|
||||
<p>Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
the shortcomings of the Gradient descent method discussed above.
|
||||
</p>
|
||||
|
||||
<p>The underlying idea of SGD comes from the observation that the cost
|
||||
function, which we want to minimize, can almost always be written as a
|
||||
sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \),
|
||||
</p>
|
||||
<p> <br>
|
||||
$$
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="computation-of-gradients">Computation of gradients </h2>
|
||||
|
||||
<p>This in turn means that the gradient can be
|
||||
computed as a sum over \( i \)-gradients
|
||||
</p>
|
||||
<p> <br>
|
||||
$$
|
||||
\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>Stochasticity/randomness is introduced by only taking the
|
||||
gradient on a subset of the data called minibatches. If there are \( n \)
|
||||
data points and the size of each minibatch is \( M \), there will be \( n/M \)
|
||||
minibatches. We denote these minibatches by \( B_k \) where
|
||||
\( k=1,\cdots,n/M \).
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="sgd-example">SGD example </h2>
|
||||
<p>As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
@@ -1873,8 +1827,18 @@ gave the lowest value.
|
||||
|
||||
<p>Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
reasonable time such that we do not move at all. Such approaches are
|
||||
also called scaling. There are many such ways to <a href="https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1" target="_blank">scale the learning
|
||||
rate</a>
|
||||
and <a href="https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf" target="_blank">discussions here</a>. See
|
||||
also
|
||||
<a href="https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1" target="_blank"><tt>https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1</tt></a>
|
||||
for a discussion of different scaling functions for the learning rate.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="time-decay-rate">Time decay rate </h2>
|
||||
|
||||
<p>As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function <p> <br>
|
||||
$$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$
|
||||
@@ -1934,99 +1898,6 @@ j = <span style="color: #B452CD">0</span>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="program-for-stochastic-gradient">Program for stochastic gradient </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Importing various packages</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
|
||||
|
||||
m = <span style="color: #B452CD">100</span>
|
||||
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
|
||||
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
|
||||
|
||||
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
|
||||
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
||||
<span style="color: #658b00">print</span>(theta_linreg)
|
||||
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
|
||||
sgdreg.fit(x,y.ravel())
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"sgdreg from scikit"</span>)
|
||||
<span style="color: #658b00">print</span>(sgdreg.intercept_, sgdreg.coef_)
|
||||
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
eta = <span style="color: #B452CD">0.1</span>
|
||||
Niterations = <span style="color: #B452CD">1000</span>
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
||||
gradients = <span style="color: #B452CD">2.0</span>/m*X.T @ ((X @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
|
||||
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
|
||||
ypredict = Xnew.dot(theta)
|
||||
ypredict2 = Xnew.dot(theta_linreg)
|
||||
|
||||
|
||||
n_epochs = <span style="color: #B452CD">50</span>
|
||||
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #228B22"># note: here the number of minibatches is equal to the number of points!!</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
random_index = np.random.randint(m)
|
||||
xi = X[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
gradients = <span style="color: #B452CD">2</span> * xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own sdg"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
plt.plot(xnew, ypredict, <span style="color: #CD5555">"r-"</span>)
|
||||
plt.plot(xnew, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
||||
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
||||
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
||||
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
||||
plt.title(<span style="color: #CD5555">r'Random numbers '</span>)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p><b>Challenge</b>: try to write a similar code for a Logistic Regression case.</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="code-with-a-number-of-minibatches-which-varies">Code with a Number of Minibatches which varies </h2>
|
||||
|
||||
@@ -2078,6 +1949,7 @@ n_epochs = 50
|
||||
M = 5 #size of each minibatch
|
||||
m = int(n/M) #number of minibatches
|
||||
t0, t1 = 5, 50
|
||||
|
||||
def learning_schedule(t):
|
||||
return t0/(t+t1)
|
||||
|
||||
@@ -2119,280 +1991,6 @@ plt.show()
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="sgd-example">SGD example </h2>
|
||||
|
||||
<p>As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 =
|
||||
(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you
|
||||
have only a single batch with all data points and on the other extreme,
|
||||
you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e
|
||||
\( B_k = \mathbf{x}_k \).
|
||||
</p>
|
||||
|
||||
<p>The idea is now to approximate the gradient by replacing the sum over
|
||||
all data points with a sum over the data points in one the minibatches
|
||||
picked at random in each gradient descent step
|
||||
</p>
|
||||
<p> <br>
|
||||
$$
|
||||
\nabla_{\beta}
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta
|
||||
c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="the-gradient-step">The gradient step </h2>
|
||||
|
||||
<p>Thus a gradient descent step now looks like </p>
|
||||
<p> <br>
|
||||
$$
|
||||
\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta})
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>where \( k \) is picked at random with equal
|
||||
probability from \( [1,n/M] \). An iteration over the number of
|
||||
minibathces (n/M) is commonly referred to as an epoch. Thus it is
|
||||
typical to choose a number of epochs and for each epoch iterate over
|
||||
the number of minibatches, as exemplified in the code below.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="simple-example-code">Simple example code </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
|
||||
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
|
||||
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each mini-batche</span>
|
||||
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
||||
n_epochs = <span style="color: #B452CD">10</span> <span style="color: #228B22">#number of epochs</span>
|
||||
|
||||
j = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n_epochs+<span style="color: #B452CD">1</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
k = np.random.randint(m) <span style="color: #228B22">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #228B22">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #228B22">#Compute new suggestion for </span>
|
||||
j += <span style="color: #B452CD">1</span>
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Taking the gradient only on a subset of the data has two important
|
||||
benefits. First, it introduces randomness which decreases the chance
|
||||
that our opmization scheme gets stuck in a local minima. Second, if
|
||||
the size of the minibatches are small relative to the number of
|
||||
datapoints (\( M < n \)), the computation of the gradient is much
|
||||
cheaper since we sum over the datapoints in the \( k-th \) minibatch and not
|
||||
all \( n \) datapoints.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="when-do-we-stop">When do we stop? </h2>
|
||||
|
||||
<p>A natural question is when do we stop the search for a new minimum?
|
||||
One possibility is to compute the full gradient after a given number
|
||||
of epochs and check if the norm of the gradient is smaller than some
|
||||
threshold and stop if true. However, the condition that the gradient
|
||||
is zero is valid also for local minima, so this would only tell us
|
||||
that we are close to a local/global minimum. However, we could also
|
||||
evaluate the cost function at this point, store the result and
|
||||
continue the search. If the test kicks in at a later stage we can
|
||||
compare the values of the cost function and keep the \( \beta \) that
|
||||
gave the lowest value.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="slightly-different-approach">Slightly different approach </h2>
|
||||
|
||||
<p>Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
</p>
|
||||
|
||||
<p>As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function <p> <br>
|
||||
$$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$
|
||||
<p> <br> goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).</p>
|
||||
|
||||
<p>In this way we can fix the number of epochs, compute \( \beta \) and
|
||||
evaluate the cost function at the end. Repeating the computation will
|
||||
give a different result since the scheme is random by design. Then we
|
||||
pick the final \( \beta \) that gives the lowest value of the cost
|
||||
function.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">step_length</span>(t,t0,t1):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
||||
|
||||
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
|
||||
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
||||
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
||||
n_epochs = <span style="color: #B452CD">500</span> <span style="color: #228B22">#number of epochs</span>
|
||||
t0 = <span style="color: #B452CD">1.0</span>
|
||||
t1 = <span style="color: #B452CD">10</span>
|
||||
|
||||
gamma_j = t0/t1
|
||||
j = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n_epochs+<span style="color: #B452CD">1</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
k = np.random.randint(m) <span style="color: #228B22">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #228B22">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #228B22">#Compute new suggestion for beta</span>
|
||||
t = epoch*m+i
|
||||
gamma_j = step_length(t,t0,t1)
|
||||
j += <span style="color: #B452CD">1</span>
|
||||
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"gamma_j after %d epochs: %g"</span> % (n_epochs,gamma_j))
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters \( t_0 \) and \( t_1 \).</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="program-for-stochastic-gradient">Program for stochastic gradient </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #228B22"># Importing various packages</span>
|
||||
<span style="color: #228B22"># Importing various packages</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
|
||||
n = <span style="color: #B452CD">100</span>
|
||||
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
||||
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
|
||||
|
||||
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
||||
XT_X = X.T @ X
|
||||
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
||||
<span style="color: #658b00">print</span>(theta_linreg)
|
||||
<span style="color: #228B22"># Hessian matrix</span>
|
||||
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
|
||||
EigValues, EigVectors = np.linalg.eig(H)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
||||
Niterations = <span style="color: #B452CD">1000</span>
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
||||
gradients = <span style="color: #B452CD">2.0</span>/n*X.T @ ((X @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
|
||||
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
|
||||
ypredict = Xnew.dot(theta)
|
||||
ypredict2 = Xnew.dot(theta_linreg)
|
||||
|
||||
n_epochs = <span style="color: #B452CD">50</span>
|
||||
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
||||
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
||||
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
||||
<span style="color: #228B22"># Can you figure out a better way of setting up the contributions to each batch?</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
random_index = M*np.random.randint(m)
|
||||
xi = X[random_index:random_index+M]
|
||||
yi = y[random_index:random_index+M]
|
||||
gradients = (<span style="color: #B452CD">2.0</span>/M)* xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own sdg"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
|
||||
|
||||
|
||||
plt.plot(xnew, ypredict, <span style="color: #CD5555">"r-"</span>)
|
||||
plt.plot(xnew, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
||||
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
||||
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
||||
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
||||
plt.title(<span style="color: #CD5555">r'Random numbers '</span>)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="replace-or-not">Replace or not </h2>
|
||||
|
||||
@@ -3536,6 +3134,50 @@ weights = np.array([<span style="color: #B452CD">0.0</span>, <span style="color:
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 id="introducing-jax-https-jax-readthedocs-io-en-latest">Introducing <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a> </h2>
|
||||
|
||||
<p>Presently, instead of using <b>autograd</b>, we recommend using <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a></p>
|
||||
|
||||
<p><b>JAX</b> is Autograd and <a href="https://www.tensorflow.org/xla" target="_blank">XLA (Accelerated Linear Algebra))</a>,
|
||||
brought together for high-performance numerical computing and machine learning research.
|
||||
It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more.
|
||||
</p>
|
||||
|
||||
<p>Here's a simple example on how you can use <b>JAX</b> to compute the derivate of the logistic function.</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="font-size: 80%; line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">jax.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">jnp</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">jax</span> <span style="color: #8B008B; font-weight: bold">import</span> grad, jit, vmap
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sum_logistic</span>(x):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> jnp.sum(<span style="color: #B452CD">1.0</span> / (<span style="color: #B452CD">1.0</span> + jnp.exp(-x)))
|
||||
|
||||
x_small = jnp.arange(<span style="color: #B452CD">3.</span>)
|
||||
derivative_fn = grad(sum_logistic)
|
||||
<span style="color: #658b00">print</span>(derivative_fn(x_small))
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
</div> <!-- class="slides" -->
|
||||
|
||||
@@ -64,7 +64,6 @@ div.toc p,a {
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 39', 2, None, 'plan-for-week-39'),
|
||||
('Thursday September 29', 2, None, 'thursday-september-29'),
|
||||
('Optimization, the central part of any Machine Learning '
|
||||
'algortithm',
|
||||
2,
|
||||
@@ -194,11 +193,6 @@ div.toc p,a {
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
@@ -207,26 +201,11 @@ div.toc p,a {
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Time decay rate', 2, None, 'time-decay-rate'),
|
||||
('Code with a Number of Minibatches which varies',
|
||||
2,
|
||||
None,
|
||||
'code-with-a-number-of-minibatches-which-varies'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Replace or not', 2, None, 'replace-or-not'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
@@ -273,7 +252,11 @@ div.toc p,a {
|
||||
2,
|
||||
None,
|
||||
'including-stochastic-gradient-descent-with-autograd'),
|
||||
('And Logistic Regression', 2, None, 'and-logistic-regression')]}
|
||||
('And Logistic Regression', 2, None, 'and-logistic-regression'),
|
||||
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
|
||||
2,
|
||||
None,
|
||||
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -319,7 +302,7 @@ MathJax.Hub.Config({
|
||||
<h2 id="plan-for-week-39">Plan for week 39 </h2>
|
||||
|
||||
<ul>
|
||||
<li> Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression</li>
|
||||
<li> Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression and discussion of stochastic gradient descent</li>
|
||||
<li> Friday: Stochastic Gradient descent with examples and automatic differentiation</li>
|
||||
<li> Reading recommendations:</li>
|
||||
</ul>
|
||||
@@ -332,11 +315,6 @@ For a good discussion on gradient methods, we would like to recommend Goodfellow
|
||||
<b>For more discussions of project 1, chapter 5 of Goodfellow et al is a good read, in particular sections 5.1-5.5 and 5.7-5.11</b>.
|
||||
<p>These sections summarize neatly what we have done till now and point to what is coming with respect to deep learning. </p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="thursday-september-29">Thursday September 29 </h2>
|
||||
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/OverarchingAimsWeek39.mp4?vrtx=view-as-webpage" target="_blank">Overview Video, why do we care about gradient methods?</a>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="optimization-the-central-part-of-any-machine-learning-algortithm">Optimization, the central part of any Machine Learning algortithm </h2>
|
||||
|
||||
@@ -1663,41 +1641,6 @@ C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="computation-of-gradients">Computation of gradients </h2>
|
||||
|
||||
<p>This in turn means that the gradient can be
|
||||
computed as a sum over \( i \)-gradients
|
||||
</p>
|
||||
$$
|
||||
\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
|
||||
<p>Stochasticity/randomness is introduced by only taking the
|
||||
gradient on a subset of the data called minibatches. If there are \( n \)
|
||||
data points and the size of each minibatch is \( M \), there will be \( n/M \)
|
||||
minibatches. We denote these minibatches by \( B_k \) where
|
||||
\( k=1,\cdots,n/M \).
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="stochastic-gradient-descent">Stochastic Gradient Descent </h2>
|
||||
|
||||
<p>Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
the shortcomings of the Gradient descent method discussed above.
|
||||
</p>
|
||||
|
||||
<p>The underlying idea of SGD comes from the observation that the cost
|
||||
function, which we want to minimize, can almost always be written as a
|
||||
sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \),
|
||||
</p>
|
||||
$$
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="computation-of-gradients">Computation of gradients </h2>
|
||||
|
||||
@@ -1824,9 +1767,18 @@ gave the lowest value.
|
||||
|
||||
<p>Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
reasonable time such that we do not move at all. Such approaches are
|
||||
also called scaling. There are many such ways to <a href="https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1" target="_blank">scale the learning
|
||||
rate</a>
|
||||
and <a href="https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf" target="_blank">discussions here</a>. See
|
||||
also
|
||||
<a href="https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1" target="_blank"><tt>https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1</tt></a>
|
||||
for a discussion of different scaling functions for the learning rate.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="time-decay-rate">Time decay rate </h2>
|
||||
|
||||
<p>As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).</p>
|
||||
|
||||
<p>In this way we can fix the number of epochs, compute \( \beta \) and
|
||||
@@ -1883,98 +1835,6 @@ j = <span style="color: #B452CD">0</span>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="program-for-stochastic-gradient">Program for stochastic gradient </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #228B22"># Importing various packages</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
|
||||
|
||||
m = <span style="color: #B452CD">100</span>
|
||||
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
|
||||
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
|
||||
|
||||
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
|
||||
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
||||
<span style="color: #658b00">print</span>(theta_linreg)
|
||||
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
|
||||
sgdreg.fit(x,y.ravel())
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"sgdreg from scikit"</span>)
|
||||
<span style="color: #658b00">print</span>(sgdreg.intercept_, sgdreg.coef_)
|
||||
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
eta = <span style="color: #B452CD">0.1</span>
|
||||
Niterations = <span style="color: #B452CD">1000</span>
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
||||
gradients = <span style="color: #B452CD">2.0</span>/m*X.T @ ((X @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
|
||||
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
|
||||
ypredict = Xnew.dot(theta)
|
||||
ypredict2 = Xnew.dot(theta_linreg)
|
||||
|
||||
|
||||
n_epochs = <span style="color: #B452CD">50</span>
|
||||
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #228B22"># note: here the number of minibatches is equal to the number of points!!</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
random_index = np.random.randint(m)
|
||||
xi = X[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
gradients = <span style="color: #B452CD">2</span> * xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own sdg"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
plt.plot(xnew, ypredict, <span style="color: #CD5555">"r-"</span>)
|
||||
plt.plot(xnew, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
||||
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
||||
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
||||
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
||||
plt.title(<span style="color: #CD5555">r'Random numbers '</span>)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p><b>Challenge</b>: try to write a similar code for a Logistic Regression case.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="code-with-a-number-of-minibatches-which-varies">Code with a Number of Minibatches which varies </h2>
|
||||
|
||||
@@ -2026,6 +1886,7 @@ n_epochs = 50
|
||||
M = 5 #size of each minibatch
|
||||
m = int(n/M) #number of minibatches
|
||||
t0, t1 = 5, 50
|
||||
|
||||
def learning_schedule(t):
|
||||
return t0/(t+t1)
|
||||
|
||||
@@ -2067,270 +1928,6 @@ plt.show()
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="sgd-example">SGD example </h2>
|
||||
|
||||
<p>As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 =
|
||||
(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you
|
||||
have only a single batch with all data points and on the other extreme,
|
||||
you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e
|
||||
\( B_k = \mathbf{x}_k \).
|
||||
</p>
|
||||
|
||||
<p>The idea is now to approximate the gradient by replacing the sum over
|
||||
all data points with a sum over the data points in one the minibatches
|
||||
picked at random in each gradient descent step
|
||||
</p>
|
||||
$$
|
||||
\nabla_{\beta}
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta
|
||||
c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="the-gradient-step">The gradient step </h2>
|
||||
|
||||
<p>Thus a gradient descent step now looks like </p>
|
||||
$$
|
||||
\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta})
|
||||
$$
|
||||
|
||||
<p>where \( k \) is picked at random with equal
|
||||
probability from \( [1,n/M] \). An iteration over the number of
|
||||
minibathces (n/M) is commonly referred to as an epoch. Thus it is
|
||||
typical to choose a number of epochs and for each epoch iterate over
|
||||
the number of minibatches, as exemplified in the code below.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="simple-example-code">Simple example code </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
|
||||
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
|
||||
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each mini-batche</span>
|
||||
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
||||
n_epochs = <span style="color: #B452CD">10</span> <span style="color: #228B22">#number of epochs</span>
|
||||
|
||||
j = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n_epochs+<span style="color: #B452CD">1</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
k = np.random.randint(m) <span style="color: #228B22">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #228B22">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #228B22">#Compute new suggestion for </span>
|
||||
j += <span style="color: #B452CD">1</span>
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Taking the gradient only on a subset of the data has two important
|
||||
benefits. First, it introduces randomness which decreases the chance
|
||||
that our opmization scheme gets stuck in a local minima. Second, if
|
||||
the size of the minibatches are small relative to the number of
|
||||
datapoints (\( M < n \)), the computation of the gradient is much
|
||||
cheaper since we sum over the datapoints in the \( k-th \) minibatch and not
|
||||
all \( n \) datapoints.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="when-do-we-stop">When do we stop? </h2>
|
||||
|
||||
<p>A natural question is when do we stop the search for a new minimum?
|
||||
One possibility is to compute the full gradient after a given number
|
||||
of epochs and check if the norm of the gradient is smaller than some
|
||||
threshold and stop if true. However, the condition that the gradient
|
||||
is zero is valid also for local minima, so this would only tell us
|
||||
that we are close to a local/global minimum. However, we could also
|
||||
evaluate the cost function at this point, store the result and
|
||||
continue the search. If the test kicks in at a later stage we can
|
||||
compare the values of the cost function and keep the \( \beta \) that
|
||||
gave the lowest value.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="slightly-different-approach">Slightly different approach </h2>
|
||||
|
||||
<p>Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
</p>
|
||||
|
||||
<p>As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).</p>
|
||||
|
||||
<p>In this way we can fix the number of epochs, compute \( \beta \) and
|
||||
evaluate the cost function at the end. Repeating the computation will
|
||||
give a different result since the scheme is random by design. Then we
|
||||
pick the final \( \beta \) that gives the lowest value of the cost
|
||||
function.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">step_length</span>(t,t0,t1):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
||||
|
||||
n = <span style="color: #B452CD">100</span> <span style="color: #228B22">#100 datapoints </span>
|
||||
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
||||
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
||||
n_epochs = <span style="color: #B452CD">500</span> <span style="color: #228B22">#number of epochs</span>
|
||||
t0 = <span style="color: #B452CD">1.0</span>
|
||||
t1 = <span style="color: #B452CD">10</span>
|
||||
|
||||
gamma_j = t0/t1
|
||||
j = <span style="color: #B452CD">0</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,n_epochs+<span style="color: #B452CD">1</span>):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
k = np.random.randint(m) <span style="color: #228B22">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #228B22">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #228B22">#Compute new suggestion for beta</span>
|
||||
t = epoch*m+i
|
||||
gamma_j = step_length(t,t0,t1)
|
||||
j += <span style="color: #B452CD">1</span>
|
||||
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"gamma_j after %d epochs: %g"</span> % (n_epochs,gamma_j))
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters \( t_0 \) and \( t_1 \).</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="program-for-stochastic-gradient">Program for stochastic gradient </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #228B22"># Importing various packages</span>
|
||||
<span style="color: #228B22"># Importing various packages</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">math</span> <span style="color: #8B008B; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">random</span> <span style="color: #8B008B; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
|
||||
n = <span style="color: #B452CD">100</span>
|
||||
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
|
||||
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
|
||||
|
||||
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
|
||||
XT_X = X.T @ X
|
||||
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"Own inversion"</span>)
|
||||
<span style="color: #658b00">print</span>(theta_linreg)
|
||||
<span style="color: #228B22"># Hessian matrix</span>
|
||||
H = (<span style="color: #B452CD">2.0</span>/n)* XT_X
|
||||
EigValues, EigVectors = np.linalg.eig(H)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f"Eigenvalues of Hessian Matrix:{</span>EigValues<span style="color: #CD5555">}"</span>)
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
eta = <span style="color: #B452CD">1.0</span>/np.max(EigValues)
|
||||
Niterations = <span style="color: #B452CD">1000</span>
|
||||
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
||||
gradients = <span style="color: #B452CD">2.0</span>/n*X.T @ ((X @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own gd"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
|
||||
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
|
||||
ypredict = Xnew.dot(theta)
|
||||
ypredict2 = Xnew.dot(theta_linreg)
|
||||
|
||||
n_epochs = <span style="color: #B452CD">50</span>
|
||||
M = <span style="color: #B452CD">5</span> <span style="color: #228B22">#size of each minibatch</span>
|
||||
m = <span style="color: #658b00">int</span>(n/M) <span style="color: #228B22">#number of minibatches</span>
|
||||
t0, t1 = <span style="color: #B452CD">5</span>, <span style="color: #B452CD">50</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">learning_schedule</span>(t):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> t0/(t+t1)
|
||||
|
||||
theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
|
||||
<span style="color: #228B22"># Can you figure out a better way of setting up the contributions to each batch?</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
|
||||
random_index = M*np.random.randint(m)
|
||||
xi = X[random_index:random_index+M]
|
||||
yi = y[random_index:random_index+M]
|
||||
gradients = (<span style="color: #B452CD">2.0</span>/M)* xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">"theta from own sdg"</span>)
|
||||
<span style="color: #658b00">print</span>(theta)
|
||||
|
||||
|
||||
|
||||
|
||||
plt.plot(xnew, ypredict, <span style="color: #CD5555">"r-"</span>)
|
||||
plt.plot(xnew, ypredict2, <span style="color: #CD5555">"b-"</span>)
|
||||
plt.plot(x, y ,<span style="color: #CD5555">'ro'</span>)
|
||||
plt.axis([<span style="color: #B452CD">0</span>,<span style="color: #B452CD">2.0</span>,<span style="color: #B452CD">0</span>, <span style="color: #B452CD">15.0</span>])
|
||||
plt.xlabel(<span style="color: #CD5555">r'$x$'</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">r'$y$'</span>)
|
||||
plt.title(<span style="color: #CD5555">r'Random numbers '</span>)
|
||||
plt.show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="replace-or-not">Replace or not </h2>
|
||||
|
||||
@@ -3438,6 +3035,49 @@ weights = np.array([<span style="color: #B452CD">0.0</span>, <span style="color:
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="introducing-jax-https-jax-readthedocs-io-en-latest">Introducing <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a> </h2>
|
||||
|
||||
<p>Presently, instead of using <b>autograd</b>, we recommend using <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a></p>
|
||||
|
||||
<p><b>JAX</b> is Autograd and <a href="https://www.tensorflow.org/xla" target="_blank">XLA (Accelerated Linear Algebra))</a>,
|
||||
brought together for high-performance numerical computing and machine learning research.
|
||||
It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more.
|
||||
</p>
|
||||
|
||||
<p>Here's a simple example on how you can use <b>JAX</b> to compute the derivate of the logistic function.</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #eeeedd">
|
||||
<pre style="line-height: 125%;"><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">jax.numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">jnp</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">jax</span> <span style="color: #8B008B; font-weight: bold">import</span> grad, jit, vmap
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">sum_logistic</span>(x):
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> jnp.sum(<span style="color: #B452CD">1.0</span> / (<span style="color: #B452CD">1.0</span> + jnp.exp(-x)))
|
||||
|
||||
x_small = jnp.arange(<span style="color: #B452CD">3.</span>)
|
||||
derivative_fn = grad(sum_logistic)
|
||||
<span style="color: #658b00">print</span>(derivative_fn(x_small))
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright --> © 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
||||
|
||||
+61
-421
@@ -141,7 +141,6 @@ div.toc p,a {
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plan for week 39', 2, None, 'plan-for-week-39'),
|
||||
('Thursday September 29', 2, None, 'thursday-september-29'),
|
||||
('Optimization, the central part of any Machine Learning '
|
||||
'algortithm',
|
||||
2,
|
||||
@@ -271,11 +270,6 @@ div.toc p,a {
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('Stochastic Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'stochastic-gradient-descent'),
|
||||
('Computation of gradients', 2, None, 'computation-of-gradients'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
@@ -284,26 +278,11 @@ div.toc p,a {
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Time decay rate', 2, None, 'time-decay-rate'),
|
||||
('Code with a Number of Minibatches which varies',
|
||||
2,
|
||||
None,
|
||||
'code-with-a-number-of-minibatches-which-varies'),
|
||||
('SGD example', 2, None, 'sgd-example'),
|
||||
('The gradient step', 2, None, 'the-gradient-step'),
|
||||
('Simple example code', 2, None, 'simple-example-code'),
|
||||
('When do we stop?', 2, None, 'when-do-we-stop'),
|
||||
('Slightly different approach',
|
||||
2,
|
||||
None,
|
||||
'slightly-different-approach'),
|
||||
('Program for stochastic gradient',
|
||||
2,
|
||||
None,
|
||||
'program-for-stochastic-gradient'),
|
||||
('Replace or not', 2, None, 'replace-or-not'),
|
||||
('Momentum based GD', 2, None, 'momentum-based-gd'),
|
||||
('More on momentum based approaches',
|
||||
@@ -350,7 +329,11 @@ div.toc p,a {
|
||||
2,
|
||||
None,
|
||||
'including-stochastic-gradient-descent-with-autograd'),
|
||||
('And Logistic Regression', 2, None, 'and-logistic-regression')]}
|
||||
('And Logistic Regression', 2, None, 'and-logistic-regression'),
|
||||
('Introducing "JAX":"https://jax.readthedocs.io/en/latest/"',
|
||||
2,
|
||||
None,
|
||||
'introducing-jax-https-jax-readthedocs-io-en-latest')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -396,7 +379,7 @@ MathJax.Hub.Config({
|
||||
<h2 id="plan-for-week-39">Plan for week 39 </h2>
|
||||
|
||||
<ul>
|
||||
<li> Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression</li>
|
||||
<li> Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression and discussion of stochastic gradient descent</li>
|
||||
<li> Friday: Stochastic Gradient descent with examples and automatic differentiation</li>
|
||||
<li> Reading recommendations:</li>
|
||||
</ul>
|
||||
@@ -409,11 +392,6 @@ For a good discussion on gradient methods, we would like to recommend Goodfellow
|
||||
<b>For more discussions of project 1, chapter 5 of Goodfellow et al is a good read, in particular sections 5.1-5.5 and 5.7-5.11</b>.
|
||||
<p>These sections summarize neatly what we have done till now and point to what is coming with respect to deep learning. </p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="thursday-september-29">Thursday September 29 </h2>
|
||||
|
||||
<a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/OverarchingAimsWeek39.mp4?vrtx=view-as-webpage" target="_blank">Overview Video, why do we care about gradient methods?</a>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="optimization-the-central-part-of-any-machine-learning-algortithm">Optimization, the central part of any Machine Learning algortithm </h2>
|
||||
|
||||
@@ -1740,41 +1718,6 @@ C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="computation-of-gradients">Computation of gradients </h2>
|
||||
|
||||
<p>This in turn means that the gradient can be
|
||||
computed as a sum over \( i \)-gradients
|
||||
</p>
|
||||
$$
|
||||
\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
|
||||
<p>Stochasticity/randomness is introduced by only taking the
|
||||
gradient on a subset of the data called minibatches. If there are \( n \)
|
||||
data points and the size of each minibatch is \( M \), there will be \( n/M \)
|
||||
minibatches. We denote these minibatches by \( B_k \) where
|
||||
\( k=1,\cdots,n/M \).
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="stochastic-gradient-descent">Stochastic Gradient Descent </h2>
|
||||
|
||||
<p>Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
the shortcomings of the Gradient descent method discussed above.
|
||||
</p>
|
||||
|
||||
<p>The underlying idea of SGD comes from the observation that the cost
|
||||
function, which we want to minimize, can almost always be written as a
|
||||
sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \),
|
||||
</p>
|
||||
$$
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="computation-of-gradients">Computation of gradients </h2>
|
||||
|
||||
@@ -1901,9 +1844,18 @@ gave the lowest value.
|
||||
|
||||
<p>Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
reasonable time such that we do not move at all. Such approaches are
|
||||
also called scaling. There are many such ways to <a href="https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1" target="_blank">scale the learning
|
||||
rate</a>
|
||||
and <a href="https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf" target="_blank">discussions here</a>. See
|
||||
also
|
||||
<a href="https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1" target="_blank"><tt>https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1</tt></a>
|
||||
for a discussion of different scaling functions for the learning rate.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="time-decay-rate">Time decay rate </h2>
|
||||
|
||||
<p>As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).</p>
|
||||
|
||||
<p>In this way we can fix the number of epochs, compute \( \beta \) and
|
||||
@@ -1960,98 +1912,6 @@ j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="program-for-stochastic-gradient">Program for stochastic gradient </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
|
||||
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
|
||||
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
|
||||
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Own inversion"</span>)
|
||||
<span style="color: #008000">print</span>(theta_linreg)
|
||||
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, eta0<span style="color: #666666">=0.1</span>)
|
||||
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"sgdreg from scikit"</span>)
|
||||
<span style="color: #008000">print</span>(sgdreg<span style="color: #666666">.</span>intercept_, sgdreg<span style="color: #666666">.</span>coef_)
|
||||
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
|
||||
Niterations <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
|
||||
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"theta from own gd"</span>)
|
||||
<span style="color: #008000">print</span>(theta)
|
||||
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
||||
Xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
|
||||
ypredict <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta)
|
||||
ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta_linreg)
|
||||
|
||||
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</span>, <span style="color: #666666">50</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">learning_schedule</span>(t):
|
||||
<span style="color: #008000; font-weight: bold">return</span> t0<span style="color: #666666">/</span>(t<span style="color: #666666">+</span>t1)
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># note: here the number of minibatches is equal to the number of points!!</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
|
||||
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
|
||||
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
|
||||
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"theta from own sdg"</span>)
|
||||
<span style="color: #008000">print</span>(theta)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict2, <span style="color: #BA2121">"b-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Random numbers '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p><b>Challenge</b>: try to write a similar code for a Logistic Regression case.</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="code-with-a-number-of-minibatches-which-varies">Code with a Number of Minibatches which varies </h2>
|
||||
|
||||
@@ -2103,6 +1963,7 @@ n_epochs = 50
|
||||
M = 5 #size of each minibatch
|
||||
m = int(n/M) #number of minibatches
|
||||
t0, t1 = 5, 50
|
||||
|
||||
def learning_schedule(t):
|
||||
return t0/(t+t1)
|
||||
|
||||
@@ -2144,270 +2005,6 @@ plt.show()
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="sgd-example">SGD example </h2>
|
||||
|
||||
<p>As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 =
|
||||
(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you
|
||||
have only a single batch with all data points and on the other extreme,
|
||||
you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e
|
||||
\( B_k = \mathbf{x}_k \).
|
||||
</p>
|
||||
|
||||
<p>The idea is now to approximate the gradient by replacing the sum over
|
||||
all data points with a sum over the data points in one the minibatches
|
||||
picked at random in each gradient descent step
|
||||
</p>
|
||||
$$
|
||||
\nabla_{\beta}
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta
|
||||
c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
$$
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="the-gradient-step">The gradient step </h2>
|
||||
|
||||
<p>Thus a gradient descent step now looks like </p>
|
||||
$$
|
||||
\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta})
|
||||
$$
|
||||
|
||||
<p>where \( k \) is picked at random with equal
|
||||
probability from \( [1,n/M] \). An iteration over the number of
|
||||
minibathces (n/M) is commonly referred to as an epoch. Thus it is
|
||||
typical to choose a number of epochs and for each epoch iterate over
|
||||
the number of minibatches, as exemplified in the code below.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="simple-example-code">Simple example code </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
|
||||
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each mini-batche</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">10</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
|
||||
|
||||
j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,n_epochs<span style="color: #666666">+1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
k <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m) <span style="color: #408080; font-style: italic">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute new suggestion for </span>
|
||||
j <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>Taking the gradient only on a subset of the data has two important
|
||||
benefits. First, it introduces randomness which decreases the chance
|
||||
that our opmization scheme gets stuck in a local minima. Second, if
|
||||
the size of the minibatches are small relative to the number of
|
||||
datapoints (\( M < n \)), the computation of the gradient is much
|
||||
cheaper since we sum over the datapoints in the \( k-th \) minibatch and not
|
||||
all \( n \) datapoints.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="when-do-we-stop">When do we stop? </h2>
|
||||
|
||||
<p>A natural question is when do we stop the search for a new minimum?
|
||||
One possibility is to compute the full gradient after a given number
|
||||
of epochs and check if the norm of the gradient is smaller than some
|
||||
threshold and stop if true. However, the condition that the gradient
|
||||
is zero is valid also for local minima, so this would only tell us
|
||||
that we are close to a local/global minimum. However, we could also
|
||||
evaluate the cost function at this point, store the result and
|
||||
continue the search. If the test kicks in at a later stage we can
|
||||
compare the values of the cost function and keep the \( \beta \) that
|
||||
gave the lowest value.
|
||||
</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="slightly-different-approach">Slightly different approach </h2>
|
||||
|
||||
<p>Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
</p>
|
||||
|
||||
<p>As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).</p>
|
||||
|
||||
<p>In this way we can fix the number of epochs, compute \( \beta \) and
|
||||
evaluate the cost function at the end. Repeating the computation will
|
||||
give a different result since the scheme is random by design. Then we
|
||||
pick the final \( \beta \) that gives the lowest value of the cost
|
||||
function.
|
||||
</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">step_length</span>(t,t0,t1):
|
||||
<span style="color: #008000; font-weight: bold">return</span> t0<span style="color: #666666">/</span>(t<span style="color: #666666">+</span>t1)
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
|
||||
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">500</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
|
||||
t0 <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
|
||||
t1 <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
|
||||
gamma_j <span style="color: #666666">=</span> t0<span style="color: #666666">/</span>t1
|
||||
j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,n_epochs<span style="color: #666666">+1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
k <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m) <span style="color: #408080; font-style: italic">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute new suggestion for beta</span>
|
||||
t <span style="color: #666666">=</span> epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i
|
||||
gamma_j <span style="color: #666666">=</span> step_length(t,t0,t1)
|
||||
j <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"gamma_j after </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> epochs: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> (n_epochs,gamma_j))
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p>We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters \( t_0 \) and \( t_1 \).</p>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="program-for-stochastic-gradient">Program for stochastic gradient </h2>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
|
||||
|
||||
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x]
|
||||
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
|
||||
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Own inversion"</span>)
|
||||
<span style="color: #008000">print</span>(theta_linreg)
|
||||
<span style="color: #408080; font-style: italic"># Hessian matrix</span>
|
||||
H <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>n)<span style="color: #666666">*</span> XT_X
|
||||
EigValues, EigVectors <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>eig(H)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Eigenvalues of Hessian Matrix:</span><span style="color: #BB6688; font-weight: bold">{</span>EigValues<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
eta <span style="color: #666666">=</span> <span style="color: #666666">1.0/</span>np<span style="color: #666666">.</span>max(EigValues)
|
||||
Niterations <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
|
||||
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>n<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
|
||||
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"theta from own gd"</span>)
|
||||
<span style="color: #008000">print</span>(theta)
|
||||
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
||||
Xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
|
||||
ypredict <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta)
|
||||
ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta_linreg)
|
||||
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
|
||||
t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</span>, <span style="color: #666666">50</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">learning_schedule</span>(t):
|
||||
<span style="color: #008000; font-weight: bold">return</span> t0<span style="color: #666666">/</span>(t<span style="color: #666666">+</span>t1)
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
|
||||
<span style="color: #408080; font-style: italic"># Can you figure out a better way of setting up the contributions to each batch?</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
random_index <span style="color: #666666">=</span> M<span style="color: #666666">*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
|
||||
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+</span>M]
|
||||
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+</span>M]
|
||||
gradients <span style="color: #666666">=</span> (<span style="color: #666666">2.0/</span>M)<span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
|
||||
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
|
||||
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"theta from own sdg"</span>)
|
||||
<span style="color: #008000">print</span>(theta)
|
||||
|
||||
|
||||
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict2, <span style="color: #BA2121">"b-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Random numbers '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="replace-or-not">Replace or not </h2>
|
||||
|
||||
@@ -3515,6 +3112,49 @@ weights <span style="color: #666666">=</span> np<span style="color: #666666">.</
|
||||
</div>
|
||||
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
<h2 id="introducing-jax-https-jax-readthedocs-io-en-latest">Introducing <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a> </h2>
|
||||
|
||||
<p>Presently, instead of using <b>autograd</b>, we recommend using <a href="https://jax.readthedocs.io/en/latest/" target="_blank">JAX</a></p>
|
||||
|
||||
<p><b>JAX</b> is Autograd and <a href="https://www.tensorflow.org/xla" target="_blank">XLA (Accelerated Linear Algebra))</a>,
|
||||
brought together for high-performance numerical computing and machine learning research.
|
||||
It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more.
|
||||
</p>
|
||||
|
||||
<p>Here's a simple example on how you can use <b>JAX</b> to compute the derivate of the logistic function.</p>
|
||||
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight" style="background: #f8f8f8">
|
||||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">jax.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">jnp</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">jax</span> <span style="color: #008000; font-weight: bold">import</span> grad, jit, vmap
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sum_logistic</span>(x):
|
||||
<span style="color: #008000; font-weight: bold">return</span> jnp<span style="color: #666666">.</span>sum(<span style="color: #666666">1.0</span> <span style="color: #666666">/</span> (<span style="color: #666666">1.0</span> <span style="color: #666666">+</span> jnp<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x)))
|
||||
|
||||
x_small <span style="color: #666666">=</span> jnp<span style="color: #666666">.</span>arange(<span style="color: #666666">3.</span>)
|
||||
derivative_fn <span style="color: #666666">=</span> grad(sum_logistic)
|
||||
<span style="color: #008000">print</span>(derivative_fn(x_small))
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="output_wrapper">
|
||||
<div class="output">
|
||||
<div class="output_area">
|
||||
<div class="output_subarea output_stream output_stdout output_text">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
<center style="font-size:80%">
|
||||
<!-- copyright --> © 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
||||
|
||||
Binary file not shown.
File diff suppressed because one or more lines are too long
+35
-323
@@ -5,7 +5,7 @@ DATE: today
|
||||
!split
|
||||
===== Plan for week 39 =====
|
||||
|
||||
* Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression
|
||||
* Thursday: Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression and discussion of stochastic gradient descent
|
||||
* Friday: Stochastic Gradient descent with examples and automatic differentiation
|
||||
|
||||
* Reading recommendations:
|
||||
@@ -18,11 +18,6 @@ For Stochastic Gradient Descent, we recommend chapter 4 of Geron's text.
|
||||
_For more discussions of project 1, chapter 5 of Goodfellow et al is a good read, in particular sections 5.1-5.5 and 5.7-5.11_.
|
||||
These sections summarize neatly what we have done till now and point to what is coming with respect to deep learning.
|
||||
|
||||
!split
|
||||
===== Thursday September 29 =====
|
||||
|
||||
"Overview Video, why do we care about gradient methods?":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/OverarchingAimsWeek39.mp4?vrtx=view-as-webpage"
|
||||
|
||||
|
||||
|
||||
!split
|
||||
@@ -1152,39 +1147,6 @@ minibatches. We denote these minibatches by $B_k$ where
|
||||
$k=1,\cdots,n/M$.
|
||||
|
||||
|
||||
!split
|
||||
===== Stochastic Gradient Descent =====
|
||||
|
||||
Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
the shortcomings of the Gradient descent method discussed above.
|
||||
|
||||
The underlying idea of SGD comes from the observation that the cost
|
||||
function, which we want to minimize, can almost always be written as a
|
||||
sum over $n$ data points $\{\mathbf{x}_i\}_{i=1}^n$,
|
||||
!bt
|
||||
\[
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
\]
|
||||
!et
|
||||
|
||||
!split
|
||||
===== Computation of gradients =====
|
||||
|
||||
This in turn means that the gradient can be
|
||||
computed as a sum over $i$-gradients
|
||||
!bt
|
||||
\[
|
||||
\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
\]
|
||||
!et
|
||||
|
||||
Stochasticity/randomness is introduced by only taking the
|
||||
gradient on a subset of the data called minibatches. If there are $n$
|
||||
data points and the size of each minibatch is $M$, there will be $n/M$
|
||||
minibatches. We denote these minibatches by $B_k$ where
|
||||
$k=1,\cdots,n/M$.
|
||||
|
||||
!split
|
||||
===== SGD example =====
|
||||
@@ -1273,7 +1235,16 @@ gave the lowest value.
|
||||
|
||||
Another approach is to let the step length $\gamma_j$ depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
reasonable time such that we do not move at all. Such approaches are
|
||||
also called scaling. There are many such ways to "scale the learning
|
||||
rate":"https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1"
|
||||
and "discussions here":"https://www.jmlr.org/papers/volume23/20-1258/20-1258.pdf". See
|
||||
also
|
||||
URL:"https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1"
|
||||
for a discussion of different scaling functions for the learning rate.
|
||||
|
||||
!split
|
||||
===== Time decay rate =====
|
||||
|
||||
As an example, let $e = 0,1,2,3,\cdots$ denote the current epoch and let $t_0, t_1 > 0$ be two fixed numbers. Furthermore, let $t = e \cdot m + i$ where $m$ is the number of minibatches and $i=0,\cdots,m-1$. Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length $\gamma_j (0; t_0, t_1) = t_0/t_1$ which decays in *time* $t$.
|
||||
|
||||
@@ -1314,81 +1285,6 @@ print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Program for stochastic gradient =====
|
||||
|
||||
!bc pycod
|
||||
# Importing various packages
|
||||
from math import exp, sqrt
|
||||
from random import random, seed
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.linear_model import SGDRegressor
|
||||
|
||||
m = 100
|
||||
x = 2*np.random.rand(m,1)
|
||||
y = 4+3*x+np.random.randn(m,1)
|
||||
|
||||
X = np.c_[np.ones((m,1)), x]
|
||||
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
|
||||
print("Own inversion")
|
||||
print(theta_linreg)
|
||||
sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)
|
||||
sgdreg.fit(x,y.ravel())
|
||||
print("sgdreg from scikit")
|
||||
print(sgdreg.intercept_, sgdreg.coef_)
|
||||
|
||||
|
||||
theta = np.random.randn(2,1)
|
||||
eta = 0.1
|
||||
Niterations = 1000
|
||||
|
||||
|
||||
for iter in range(Niterations):
|
||||
gradients = 2.0/m*X.T @ ((X @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
print("theta from own gd")
|
||||
print(theta)
|
||||
|
||||
xnew = np.array([[0],[2]])
|
||||
Xnew = np.c_[np.ones((2,1)), xnew]
|
||||
ypredict = Xnew.dot(theta)
|
||||
ypredict2 = Xnew.dot(theta_linreg)
|
||||
|
||||
|
||||
n_epochs = 50
|
||||
t0, t1 = 5, 50
|
||||
def learning_schedule(t):
|
||||
return t0/(t+t1)
|
||||
|
||||
theta = np.random.randn(2,1)
|
||||
|
||||
# note: here the number of minibatches is equal to the number of points!!
|
||||
for epoch in range(n_epochs):
|
||||
for i in range(m):
|
||||
random_index = np.random.randint(m)
|
||||
xi = X[random_index:random_index+1]
|
||||
yi = y[random_index:random_index+1]
|
||||
gradients = 2 * xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
print("theta from own sdg")
|
||||
print(theta)
|
||||
|
||||
plt.plot(xnew, ypredict, "r-")
|
||||
plt.plot(xnew, ypredict2, "b-")
|
||||
plt.plot(x, y ,'ro')
|
||||
plt.axis([0,2.0,0, 15.0])
|
||||
plt.xlabel(r'$x$')
|
||||
plt.ylabel(r'$y$')
|
||||
plt.title(r'Random numbers ')
|
||||
plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
_Challenge_: try to write a similar code for a Logistic Regression case.
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Code with a Number of Minibatches which varies =====
|
||||
@@ -1435,6 +1331,7 @@ n_epochs = 50
|
||||
M = 5 #size of each minibatch
|
||||
m = int(n/M) #number of minibatches
|
||||
t0, t1 = 5, 50
|
||||
|
||||
def learning_schedule(t):
|
||||
return t0/(t+t1)
|
||||
|
||||
@@ -1465,214 +1362,6 @@ plt.show()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== SGD example =====
|
||||
|
||||
As an example, suppose we have $10$ data points $(\mathbf{x}_1,\cdots, \mathbf{x}_{10})$
|
||||
and we choose to have $M=5$ minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
$B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 =
|
||||
(\mathbf{x}_9,\mathbf{x}_{10})$. Note that if you choose $M=1$ you
|
||||
have only a single batch with all data points and on the other extreme,
|
||||
you may choose $M=n$ resulting in a minibatch for each datapoint, i.e
|
||||
$B_k = \mathbf{x}_k$.
|
||||
|
||||
The idea is now to approximate the gradient by replacing the sum over
|
||||
all data points with a sum over the data points in one the minibatches
|
||||
picked at random in each gradient descent step
|
||||
!bt
|
||||
\[
|
||||
\nabla_{\beta}
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta
|
||||
c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
\]
|
||||
!et
|
||||
|
||||
!split
|
||||
===== The gradient step =====
|
||||
|
||||
Thus a gradient descent step now looks like
|
||||
!bt
|
||||
\[
|
||||
\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta})
|
||||
\]
|
||||
!et
|
||||
|
||||
where $k$ is picked at random with equal
|
||||
probability from $[1,n/M]$. An iteration over the number of
|
||||
minibathces (n/M) is commonly referred to as an epoch. Thus it is
|
||||
typical to choose a number of epochs and for each epoch iterate over
|
||||
the number of minibatches, as exemplified in the code below.
|
||||
|
||||
!split
|
||||
===== Simple example code =====
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
|
||||
n = 100 #100 datapoints
|
||||
M = 5 #size of each mini-batche
|
||||
m = int(n/M) #number of minibatches
|
||||
n_epochs = 10 #number of epochs
|
||||
|
||||
j = 0
|
||||
for epoch in range(1,n_epochs+1):
|
||||
for i in range(m):
|
||||
k = np.random.randint(m) #Pick the k-th minibatch at random
|
||||
#Compute the gradient using the data in minibatch Bk
|
||||
#Compute new suggestion for
|
||||
j += 1
|
||||
!ec
|
||||
|
||||
Taking the gradient only on a subset of the data has two important
|
||||
benefits. First, it introduces randomness which decreases the chance
|
||||
that our opmization scheme gets stuck in a local minima. Second, if
|
||||
the size of the minibatches are small relative to the number of
|
||||
datapoints ($M < n$), the computation of the gradient is much
|
||||
cheaper since we sum over the datapoints in the $k-th$ minibatch and not
|
||||
all $n$ datapoints.
|
||||
|
||||
!split
|
||||
===== When do we stop? =====
|
||||
|
||||
A natural question is when do we stop the search for a new minimum?
|
||||
One possibility is to compute the full gradient after a given number
|
||||
of epochs and check if the norm of the gradient is smaller than some
|
||||
threshold and stop if true. However, the condition that the gradient
|
||||
is zero is valid also for local minima, so this would only tell us
|
||||
that we are close to a local/global minimum. However, we could also
|
||||
evaluate the cost function at this point, store the result and
|
||||
continue the search. If the test kicks in at a later stage we can
|
||||
compare the values of the cost function and keep the $\beta$ that
|
||||
gave the lowest value.
|
||||
|
||||
!split
|
||||
===== Slightly different approach =====
|
||||
|
||||
Another approach is to let the step length $\gamma_j$ depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
|
||||
As an example, let $e = 0,1,2,3,\cdots$ denote the current epoch and let $t_0, t_1 > 0$ be two fixed numbers. Furthermore, let $t = e \cdot m + i$ where $m$ is the number of minibatches and $i=0,\cdots,m-1$. Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length $\gamma_j (0; t_0, t_1) = t_0/t_1$ which decays in *time* $t$.
|
||||
|
||||
In this way we can fix the number of epochs, compute $\beta$ and
|
||||
evaluate the cost function at the end. Repeating the computation will
|
||||
give a different result since the scheme is random by design. Then we
|
||||
pick the final $\beta$ that gives the lowest value of the cost
|
||||
function.
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
|
||||
def step_length(t,t0,t1):
|
||||
return t0/(t+t1)
|
||||
|
||||
n = 100 #100 datapoints
|
||||
M = 5 #size of each minibatch
|
||||
m = int(n/M) #number of minibatches
|
||||
n_epochs = 500 #number of epochs
|
||||
t0 = 1.0
|
||||
t1 = 10
|
||||
|
||||
gamma_j = t0/t1
|
||||
j = 0
|
||||
for epoch in range(1,n_epochs+1):
|
||||
for i in range(m):
|
||||
k = np.random.randint(m) #Pick the k-th minibatch at random
|
||||
#Compute the gradient using the data in minibatch Bk
|
||||
#Compute new suggestion for beta
|
||||
t = epoch*m+i
|
||||
gamma_j = step_length(t,t0,t1)
|
||||
j += 1
|
||||
|
||||
print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))
|
||||
!ec
|
||||
|
||||
|
||||
We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters $t_0$ and $t_1$.
|
||||
|
||||
|
||||
!split
|
||||
===== Program for stochastic gradient =====
|
||||
|
||||
!bc pycod
|
||||
# Importing various packages
|
||||
# Importing various packages
|
||||
from math import exp, sqrt
|
||||
from random import random, seed
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
n = 100
|
||||
x = 2*np.random.rand(n,1)
|
||||
y = 4+3*x+np.random.randn(n,1)
|
||||
|
||||
X = np.c_[np.ones((n,1)), x]
|
||||
XT_X = X.T @ X
|
||||
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
|
||||
print("Own inversion")
|
||||
print(theta_linreg)
|
||||
# Hessian matrix
|
||||
H = (2.0/n)* XT_X
|
||||
EigValues, EigVectors = np.linalg.eig(H)
|
||||
print(f"Eigenvalues of Hessian Matrix:{EigValues}")
|
||||
|
||||
theta = np.random.randn(2,1)
|
||||
eta = 1.0/np.max(EigValues)
|
||||
Niterations = 1000
|
||||
|
||||
|
||||
for iter in range(Niterations):
|
||||
gradients = 2.0/n*X.T @ ((X @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
print("theta from own gd")
|
||||
print(theta)
|
||||
|
||||
xnew = np.array([[0],[2]])
|
||||
Xnew = np.c_[np.ones((2,1)), xnew]
|
||||
ypredict = Xnew.dot(theta)
|
||||
ypredict2 = Xnew.dot(theta_linreg)
|
||||
|
||||
n_epochs = 50
|
||||
M = 5 #size of each minibatch
|
||||
m = int(n/M) #number of minibatches
|
||||
t0, t1 = 5, 50
|
||||
def learning_schedule(t):
|
||||
return t0/(t+t1)
|
||||
|
||||
theta = np.random.randn(2,1)
|
||||
|
||||
for epoch in range(n_epochs):
|
||||
# Can you figure out a better way of setting up the contributions to each batch?
|
||||
for i in range(m):
|
||||
random_index = M*np.random.randint(m)
|
||||
xi = X[random_index:random_index+M]
|
||||
yi = y[random_index:random_index+M]
|
||||
gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
print("theta from own sdg")
|
||||
print(theta)
|
||||
|
||||
|
||||
|
||||
|
||||
plt.plot(xnew, ypredict, "r-")
|
||||
plt.plot(xnew, ypredict2, "b-")
|
||||
plt.plot(x, y ,'ro')
|
||||
plt.axis([0,2.0,0, 15.0])
|
||||
plt.xlabel(r'$x$')
|
||||
plt.ylabel(r'$y$')
|
||||
plt.title(r'Random numbers ')
|
||||
plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Replace or not =====
|
||||
|
||||
@@ -2468,3 +2157,26 @@ print("Trained loss:", training_loss(weights))
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Introducing "JAX":"https://jax.readthedocs.io/en/latest/" =====
|
||||
|
||||
Presently, instead of using _autograd_, we recommend using "JAX":"https://jax.readthedocs.io/en/latest/"
|
||||
|
||||
_JAX_ is Autograd and "XLA (Accelerated Linear Algebra))":"https://www.tensorflow.org/xla",
|
||||
brought together for high-performance numerical computing and machine learning research.
|
||||
It provides composable transformations of Python+NumPy programs: differentiate, vectorize, parallelize, Just-In-Time compile to GPU/TPU, and more.
|
||||
|
||||
Here's a simple example on how you can use _JAX_ to compute the derivate of the logistic function.
|
||||
|
||||
!bc pycod
|
||||
import jax.numpy as jnp
|
||||
from jax import grad, jit, vmap
|
||||
|
||||
def sum_logistic(x):
|
||||
return jnp.sum(1.0 / (1.0 + jnp.exp(-x)))
|
||||
|
||||
x_small = jnp.arange(3.)
|
||||
derivative_fn = grad(sum_logistic)
|
||||
print(derivative_fn(x_small))
|
||||
|
||||
!ec
|
||||
|
||||
Reference in New Issue
Block a user