diff --git a/doc/pub/week37/html/._week37-bs000.html b/doc/pub/week37/html/._week37-bs000.html index 541ecc814..cfe72140a 100644 --- a/doc/pub/week37/html/._week37-bs000.html +++ b/doc/pub/week37/html/._week37-bs000.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -300,7 +319,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs001.html b/doc/pub/week37/html/._week37-bs001.html index 3edf27444..8ce33a94c 100644 --- a/doc/pub/week37/html/._week37-bs001.html +++ b/doc/pub/week37/html/._week37-bs001.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -291,7 +310,7 @@ MathJax.Hub.Config({
  • 10
  • 11
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs002.html b/doc/pub/week37/html/._week37-bs002.html index f673291cf..56c23018e 100644 --- a/doc/pub/week37/html/._week37-bs002.html +++ b/doc/pub/week37/html/._week37-bs002.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -263,7 +282,7 @@ MathJax.Hub.Config({
      -
    1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at URL::https://www.deeplearningbook.org/contents/optimization.html"
      2. +
    2. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at https://www.deeplearningbook.org/contents/optimization.html
    3. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.
    4. Video on gradient descent at https://www.youtube.com/watch?v=sDv4f4s2SB8
    5. Video on Stochastic gradient descent at https://www.youtube.com/watch?v=vMh0zPT0tLI
      6. @@ -289,7 +308,7 @@ MathJax.Hub.Config({
    6. 11
    7. 12
    8. ...
      9. -
    9. 46
      10. +
    10. 48
    11. »
      12. diff --git a/doc/pub/week37/html/._week37-bs003.html b/doc/pub/week37/html/._week37-bs003.html index ccb14f2ac..fca95acdf 100644 --- a/doc/pub/week37/html/._week37-bs003.html +++ b/doc/pub/week37/html/._week37-bs003.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    12. Program example for gradient descent with Ridge Regression
    13. Using gradient descent methods, limitations
    14. Improving gradient descent with momentum
      15. -
    15. Same code but now with momentum gradient descent
      16. -
    16. Overview video on Stochastic Gradient Descent
      17. +
    17. Same code but now with momentum gradient descent
      18. +
    18. Overview video on Stochastic Gradient Descent (SGD)
    19. Batches and mini-batches
      20. -
    20. Stochastic Gradient Descent (SGD)
      21. -
    21. Stochastic Gradient Descent
      22. -
    22. Computation of gradients
      23. -
    23. SGD example
      24. -
    24. The gradient step
      25. -
    25. Simple example code
      26. -
    26. When do we stop?
      27. -
    27. Slightly different approach
      28. -
    28. Time decay rate
      29. -
    29. Code with a Number of Minibatches which varies
      30. -
    30. Replace or not
      31. -
    31. Momentum based GD
      32. -
    32. More on momentum based approaches
      33. -
    33. Momentum parameter
      34. -
    34. Second moment of the gradient
      35. -
    35. RMS prop
      36. -
    36. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      37. -
    37. Algorithms and codes for Adagrad, RMSprop and Adam
      38. -
    38. Practical tips
      39. -
    39. Sneaking in automatic differentiation using Autograd
      40. -
    40. Same code but now with momentum gradient descent
      41. -
    41. But none of these can compete with Newton's method
      42. -
    42. Including Stochastic Gradient Descent with Autograd
      43. -
    43. Same code but now with momentum gradient descent
      44. -
    44. Similar (second order function now) problem but now with AdaGrad
      45. -
    45. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      46. -
    46. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      47. -
    47. Material for the lab sessions
      48. +
    48. Pros and cons
      49. +
    49. Convergence rates
      50. +
    50. Accuracy
      51. +
    51. Stochastic Gradient Descent (SGD)
      52. +
    52. Stochastic Gradient Descent
      53. +
    53. Computation of gradients
      54. +
    54. SGD example
      55. +
    55. The gradient step
      56. +
    56. Simple example code
      57. +
    57. When do we stop?
      58. +
    58. Slightly different approach
      59. +
    59. Time decay rate
      60. +
    60. Code with a Number of Minibatches which varies
      61. +
    61. Replace or not
      62. +
    62. Momentum based GD
      63. +
    63. More on momentum based approaches
      64. +
    64. Momentum parameter
      65. +
    65. Second moment of the gradient
      66. +
    66. RMS prop
      67. +
    67. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      68. +
    68. Algorithms and codes for Adagrad, RMSprop and Adam
      69. +
    69. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      70. +
    70. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      71. +
    71. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      72. +
    72. Practical tips
      73. +
    73. Sneaking in auotmatic differentiation using Autograd
      74. +
    74. Same code but now with momentum gradient descent
      75. +
    75. Including Stochastic Gradient Descent with Autograd
      76. +
    76. Same code but now with momentum gradient descent
      77. +
    77. Similar (second order function now) problem but now with AdaGrad
      78. +
    78. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      79. +
    79. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      80. +
    80. Material for the lab sessions
      81. @@ -278,7 +297,7 @@ MathJax.Hub.Config({
    81. 12
    82. 13
    83. ...
      84. -
    84. 46
      85. +
    85. 48
    86. »
      87. diff --git a/doc/pub/week37/html/._week37-bs004.html b/doc/pub/week37/html/._week37-bs004.html index 5136d1a63..db8020187 100644 --- a/doc/pub/week37/html/._week37-bs004.html +++ b/doc/pub/week37/html/._week37-bs004.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    87. Program example for gradient descent with Ridge Regression
    88. Using gradient descent methods, limitations
    89. Improving gradient descent with momentum
      90. -
    90. Same code but now with momentum gradient descent
      91. -
    91. Overview video on Stochastic Gradient Descent
      92. +
    92. Same code but now with momentum gradient descent
      93. +
    93. Overview video on Stochastic Gradient Descent (SGD)
    94. Batches and mini-batches
      95. -
    95. Stochastic Gradient Descent (SGD)
      96. -
    96. Stochastic Gradient Descent
      97. -
    97. Computation of gradients
      98. -
    98. SGD example
      99. -
    99. The gradient step
      100. -
    100. Simple example code
      101. -
    101. When do we stop?
      102. -
    102. Slightly different approach
      103. -
    103. Time decay rate
      104. -
    104. Code with a Number of Minibatches which varies
      105. -
    105. Replace or not
      106. -
    106. Momentum based GD
      107. -
    107. More on momentum based approaches
      108. -
    108. Momentum parameter
      109. -
    109. Second moment of the gradient
      110. -
    110. RMS prop
      111. -
    111. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      112. -
    112. Algorithms and codes for Adagrad, RMSprop and Adam
      113. -
    113. Practical tips
      114. -
    114. Sneaking in automatic differentiation using Autograd
      115. -
    115. Same code but now with momentum gradient descent
      116. -
    116. But none of these can compete with Newton's method
      117. -
    117. Including Stochastic Gradient Descent with Autograd
      118. -
    118. Same code but now with momentum gradient descent
      119. -
    119. Similar (second order function now) problem but now with AdaGrad
      120. -
    120. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      121. -
    121. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      122. -
    122. Material for the lab sessions
      123. +
    123. Pros and cons
      124. +
    124. Convergence rates
      125. +
    125. Accuracy
      126. +
    126. Stochastic Gradient Descent (SGD)
      127. +
    127. Stochastic Gradient Descent
      128. +
    128. Computation of gradients
      129. +
    129. SGD example
      130. +
    130. The gradient step
      131. +
    131. Simple example code
      132. +
    132. When do we stop?
      133. +
    133. Slightly different approach
      134. +
    134. Time decay rate
      135. +
    135. Code with a Number of Minibatches which varies
      136. +
    136. Replace or not
      137. +
    137. Momentum based GD
      138. +
    138. More on momentum based approaches
      139. +
    139. Momentum parameter
      140. +
    140. Second moment of the gradient
      141. +
    141. RMS prop
      142. +
    142. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      143. +
    143. Algorithms and codes for Adagrad, RMSprop and Adam
      144. +
    144. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      145. +
    145. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      146. +
    146. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      147. +
    147. Practical tips
      148. +
    148. Sneaking in auotmatic differentiation using Autograd
      149. +
    149. Same code but now with momentum gradient descent
      150. +
    150. Including Stochastic Gradient Descent with Autograd
      151. +
    151. Same code but now with momentum gradient descent
      152. +
    152. Similar (second order function now) problem but now with AdaGrad
      153. +
    153. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      154. +
    154. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      155. +
    155. Material for the lab sessions
      156. @@ -329,7 +348,7 @@ $$
    156. 13
    157. 14
    158. ...
      159. -
    159. 46
      160. +
    160. 48
    161. »
      162. diff --git a/doc/pub/week37/html/._week37-bs005.html b/doc/pub/week37/html/._week37-bs005.html index a2b3478ab..beec70c4e 100644 --- a/doc/pub/week37/html/._week37-bs005.html +++ b/doc/pub/week37/html/._week37-bs005.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    162. Program example for gradient descent with Ridge Regression
    163. Using gradient descent methods, limitations
    164. Improving gradient descent with momentum
      165. -
    165. Same code but now with momentum gradient descent
      166. -
    166. Overview video on Stochastic Gradient Descent
      167. +
    167. Same code but now with momentum gradient descent
      168. +
    168. Overview video on Stochastic Gradient Descent (SGD)
    169. Batches and mini-batches
      170. -
    170. Stochastic Gradient Descent (SGD)
      171. -
    171. Stochastic Gradient Descent
      172. -
    172. Computation of gradients
      173. -
    173. SGD example
      174. -
    174. The gradient step
      175. -
    175. Simple example code
      176. -
    176. When do we stop?
      177. -
    177. Slightly different approach
      178. -
    178. Time decay rate
      179. -
    179. Code with a Number of Minibatches which varies
      180. -
    180. Replace or not
      181. -
    181. Momentum based GD
      182. -
    182. More on momentum based approaches
      183. -
    183. Momentum parameter
      184. -
    184. Second moment of the gradient
      185. -
    185. RMS prop
      186. -
    186. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      187. -
    187. Algorithms and codes for Adagrad, RMSprop and Adam
      188. -
    188. Practical tips
      189. -
    189. Sneaking in automatic differentiation using Autograd
      190. -
    190. Same code but now with momentum gradient descent
      191. -
    191. But none of these can compete with Newton's method
      192. -
    192. Including Stochastic Gradient Descent with Autograd
      193. -
    193. Same code but now with momentum gradient descent
      194. -
    194. Similar (second order function now) problem but now with AdaGrad
      195. -
    195. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      196. -
    196. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      197. -
    197. Material for the lab sessions
      198. +
    198. Pros and cons
      199. +
    199. Convergence rates
      200. +
    200. Accuracy
      201. +
    201. Stochastic Gradient Descent (SGD)
      202. +
    202. Stochastic Gradient Descent
      203. +
    203. Computation of gradients
      204. +
    204. SGD example
      205. +
    205. The gradient step
      206. +
    206. Simple example code
      207. +
    207. When do we stop?
      208. +
    208. Slightly different approach
      209. +
    209. Time decay rate
      210. +
    210. Code with a Number of Minibatches which varies
      211. +
    211. Replace or not
      212. +
    212. Momentum based GD
      213. +
    213. More on momentum based approaches
      214. +
    214. Momentum parameter
      215. +
    215. Second moment of the gradient
      216. +
    216. RMS prop
      217. +
    217. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      218. +
    218. Algorithms and codes for Adagrad, RMSprop and Adam
      219. +
    219. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      220. +
    220. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      221. +
    221. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      222. +
    222. Practical tips
      223. +
    223. Sneaking in auotmatic differentiation using Autograd
      224. +
    224. Same code but now with momentum gradient descent
      225. +
    225. Including Stochastic Gradient Descent with Autograd
      226. +
    226. Same code but now with momentum gradient descent
      227. +
    227. Similar (second order function now) problem but now with AdaGrad
      228. +
    228. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      229. +
    229. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      230. +
    230. Material for the lab sessions
      231. @@ -298,7 +317,7 @@ $$
    231. 14
    232. 15
    233. ...
      234. -
    234. 46
      235. +
    235. 48
    236. »
      237. diff --git a/doc/pub/week37/html/._week37-bs006.html b/doc/pub/week37/html/._week37-bs006.html index f42693ebf..e8de7baf5 100644 --- a/doc/pub/week37/html/._week37-bs006.html +++ b/doc/pub/week37/html/._week37-bs006.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    237. Program example for gradient descent with Ridge Regression
    238. Using gradient descent methods, limitations
    239. Improving gradient descent with momentum
      240. -
    240. Same code but now with momentum gradient descent
      241. -
    241. Overview video on Stochastic Gradient Descent
      242. +
    242. Same code but now with momentum gradient descent
      243. +
    243. Overview video on Stochastic Gradient Descent (SGD)
    244. Batches and mini-batches
      245. -
    245. Stochastic Gradient Descent (SGD)
      246. -
    246. Stochastic Gradient Descent
      247. -
    247. Computation of gradients
      248. -
    248. SGD example
      249. -
    249. The gradient step
      250. -
    250. Simple example code
      251. -
    251. When do we stop?
      252. -
    252. Slightly different approach
      253. -
    253. Time decay rate
      254. -
    254. Code with a Number of Minibatches which varies
      255. -
    255. Replace or not
      256. -
    256. Momentum based GD
      257. -
    257. More on momentum based approaches
      258. -
    258. Momentum parameter
      259. -
    259. Second moment of the gradient
      260. -
    260. RMS prop
      261. -
    261. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      262. -
    262. Algorithms and codes for Adagrad, RMSprop and Adam
      263. -
    263. Practical tips
      264. -
    264. Sneaking in automatic differentiation using Autograd
      265. -
    265. Same code but now with momentum gradient descent
      266. -
    266. But none of these can compete with Newton's method
      267. -
    267. Including Stochastic Gradient Descent with Autograd
      268. -
    268. Same code but now with momentum gradient descent
      269. -
    269. Similar (second order function now) problem but now with AdaGrad
      270. -
    270. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      271. -
    271. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      272. -
    272. Material for the lab sessions
      273. +
    273. Pros and cons
      274. +
    274. Convergence rates
      275. +
    275. Accuracy
      276. +
    276. Stochastic Gradient Descent (SGD)
      277. +
    277. Stochastic Gradient Descent
      278. +
    278. Computation of gradients
      279. +
    279. SGD example
      280. +
    280. The gradient step
      281. +
    281. Simple example code
      282. +
    282. When do we stop?
      283. +
    283. Slightly different approach
      284. +
    284. Time decay rate
      285. +
    285. Code with a Number of Minibatches which varies
      286. +
    286. Replace or not
      287. +
    287. Momentum based GD
      288. +
    288. More on momentum based approaches
      289. +
    289. Momentum parameter
      290. +
    290. Second moment of the gradient
      291. +
    291. RMS prop
      292. +
    292. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      293. +
    293. Algorithms and codes for Adagrad, RMSprop and Adam
      294. +
    294. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      295. +
    295. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      296. +
    296. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      297. +
    297. Practical tips
      298. +
    298. Sneaking in auotmatic differentiation using Autograd
      299. +
    299. Same code but now with momentum gradient descent
      300. +
    300. Including Stochastic Gradient Descent with Autograd
      301. +
    301. Same code but now with momentum gradient descent
      302. +
    302. Similar (second order function now) problem but now with AdaGrad
      303. +
    303. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      304. +
    304. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      305. +
    305. Material for the lab sessions
      306. @@ -290,7 +309,7 @@ $$
    306. 15
    307. 16
    308. ...
      309. -
    309. 46
      310. +
    310. 48
    311. »
      312. diff --git a/doc/pub/week37/html/._week37-bs007.html b/doc/pub/week37/html/._week37-bs007.html index 71994bda6..b6b3a1230 100644 --- a/doc/pub/week37/html/._week37-bs007.html +++ b/doc/pub/week37/html/._week37-bs007.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    312. Program example for gradient descent with Ridge Regression
    313. Using gradient descent methods, limitations
    314. Improving gradient descent with momentum
      315. -
    315. Same code but now with momentum gradient descent
      316. -
    316. Overview video on Stochastic Gradient Descent
      317. +
    317. Same code but now with momentum gradient descent
      318. +
    318. Overview video on Stochastic Gradient Descent (SGD)
    319. Batches and mini-batches
      320. -
    320. Stochastic Gradient Descent (SGD)
      321. -
    321. Stochastic Gradient Descent
      322. -
    322. Computation of gradients
      323. -
    323. SGD example
      324. -
    324. The gradient step
      325. -
    325. Simple example code
      326. -
    326. When do we stop?
      327. -
    327. Slightly different approach
      328. -
    328. Time decay rate
      329. -
    329. Code with a Number of Minibatches which varies
      330. -
    330. Replace or not
      331. -
    331. Momentum based GD
      332. -
    332. More on momentum based approaches
      333. -
    333. Momentum parameter
      334. -
    334. Second moment of the gradient
      335. -
    335. RMS prop
      336. -
    336. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      337. -
    337. Algorithms and codes for Adagrad, RMSprop and Adam
      338. -
    338. Practical tips
      339. -
    339. Sneaking in automatic differentiation using Autograd
      340. -
    340. Same code but now with momentum gradient descent
      341. -
    341. But none of these can compete with Newton's method
      342. -
    342. Including Stochastic Gradient Descent with Autograd
      343. -
    343. Same code but now with momentum gradient descent
      344. -
    344. Similar (second order function now) problem but now with AdaGrad
      345. -
    345. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      346. -
    346. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      347. -
    347. Material for the lab sessions
      348. +
    348. Pros and cons
      349. +
    349. Convergence rates
      350. +
    350. Accuracy
      351. +
    351. Stochastic Gradient Descent (SGD)
      352. +
    352. Stochastic Gradient Descent
      353. +
    353. Computation of gradients
      354. +
    354. SGD example
      355. +
    355. The gradient step
      356. +
    356. Simple example code
      357. +
    357. When do we stop?
      358. +
    358. Slightly different approach
      359. +
    359. Time decay rate
      360. +
    360. Code with a Number of Minibatches which varies
      361. +
    361. Replace or not
      362. +
    362. Momentum based GD
      363. +
    363. More on momentum based approaches
      364. +
    364. Momentum parameter
      365. +
    365. Second moment of the gradient
      366. +
    366. RMS prop
      367. +
    367. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      368. +
    368. Algorithms and codes for Adagrad, RMSprop and Adam
      369. +
    369. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      370. +
    370. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      371. +
    371. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      372. +
    372. Practical tips
      373. +
    373. Sneaking in auotmatic differentiation using Autograd
      374. +
    374. Same code but now with momentum gradient descent
      375. +
    375. Including Stochastic Gradient Descent with Autograd
      376. +
    376. Same code but now with momentum gradient descent
      377. +
    377. Similar (second order function now) problem but now with AdaGrad
      378. +
    378. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      379. +
    379. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      380. +
    380. Material for the lab sessions
      381. @@ -291,7 +310,7 @@ $$
    381. 16
    382. 17
    383. ...
      384. -
    384. 46
      385. +
    385. 48
    386. »
      387. diff --git a/doc/pub/week37/html/._week37-bs008.html b/doc/pub/week37/html/._week37-bs008.html index 4cc405564..67e684b22 100644 --- a/doc/pub/week37/html/._week37-bs008.html +++ b/doc/pub/week37/html/._week37-bs008.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    387. Program example for gradient descent with Ridge Regression
    388. Using gradient descent methods, limitations
    389. Improving gradient descent with momentum
      390. -
    390. Same code but now with momentum gradient descent
      391. -
    391. Overview video on Stochastic Gradient Descent
      392. +
    392. Same code but now with momentum gradient descent
      393. +
    393. Overview video on Stochastic Gradient Descent (SGD)
    394. Batches and mini-batches
      395. -
    395. Stochastic Gradient Descent (SGD)
      396. -
    396. Stochastic Gradient Descent
      397. -
    397. Computation of gradients
      398. -
    398. SGD example
      399. -
    399. The gradient step
      400. -
    400. Simple example code
      401. -
    401. When do we stop?
      402. -
    402. Slightly different approach
      403. -
    403. Time decay rate
      404. -
    404. Code with a Number of Minibatches which varies
      405. -
    405. Replace or not
      406. -
    406. Momentum based GD
      407. -
    407. More on momentum based approaches
      408. -
    408. Momentum parameter
      409. -
    409. Second moment of the gradient
      410. -
    410. RMS prop
      411. -
    411. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      412. -
    412. Algorithms and codes for Adagrad, RMSprop and Adam
      413. -
    413. Practical tips
      414. -
    414. Sneaking in automatic differentiation using Autograd
      415. -
    415. Same code but now with momentum gradient descent
      416. -
    416. But none of these can compete with Newton's method
      417. -
    417. Including Stochastic Gradient Descent with Autograd
      418. -
    418. Same code but now with momentum gradient descent
      419. -
    419. Similar (second order function now) problem but now with AdaGrad
      420. -
    420. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      421. -
    421. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      422. -
    422. Material for the lab sessions
      423. +
    423. Pros and cons
      424. +
    424. Convergence rates
      425. +
    425. Accuracy
      426. +
    426. Stochastic Gradient Descent (SGD)
      427. +
    427. Stochastic Gradient Descent
      428. +
    428. Computation of gradients
      429. +
    429. SGD example
      430. +
    430. The gradient step
      431. +
    431. Simple example code
      432. +
    432. When do we stop?
      433. +
    433. Slightly different approach
      434. +
    434. Time decay rate
      435. +
    435. Code with a Number of Minibatches which varies
      436. +
    436. Replace or not
      437. +
    437. Momentum based GD
      438. +
    438. More on momentum based approaches
      439. +
    439. Momentum parameter
      440. +
    440. Second moment of the gradient
      441. +
    441. RMS prop
      442. +
    442. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      443. +
    443. Algorithms and codes for Adagrad, RMSprop and Adam
      444. +
    444. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      445. +
    445. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      446. +
    446. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      447. +
    447. Practical tips
      448. +
    448. Sneaking in auotmatic differentiation using Autograd
      449. +
    449. Same code but now with momentum gradient descent
      450. +
    450. Including Stochastic Gradient Descent with Autograd
      451. +
    451. Same code but now with momentum gradient descent
      452. +
    452. Similar (second order function now) problem but now with AdaGrad
      453. +
    453. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      454. +
    454. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      455. +
    455. Material for the lab sessions
      456. @@ -297,7 +316,7 @@ when \( ||\nabla_\theta C(\theta_k) || \leq \epsilon = 10^{-8} \). Note that
    456. 17
    457. 18
    458. ...
      459. -
    459. 46
      460. +
    460. 48
    461. »
      462. diff --git a/doc/pub/week37/html/._week37-bs009.html b/doc/pub/week37/html/._week37-bs009.html index df9f8f43e..65ab5bed9 100644 --- a/doc/pub/week37/html/._week37-bs009.html +++ b/doc/pub/week37/html/._week37-bs009.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    462. Program example for gradient descent with Ridge Regression
    463. Using gradient descent methods, limitations
    464. Improving gradient descent with momentum
      465. -
    465. Same code but now with momentum gradient descent
      466. -
    466. Overview video on Stochastic Gradient Descent
      467. +
    467. Same code but now with momentum gradient descent
      468. +
    468. Overview video on Stochastic Gradient Descent (SGD)
    469. Batches and mini-batches
      470. -
    470. Stochastic Gradient Descent (SGD)
      471. -
    471. Stochastic Gradient Descent
      472. -
    472. Computation of gradients
      473. -
    473. SGD example
      474. -
    474. The gradient step
      475. -
    475. Simple example code
      476. -
    476. When do we stop?
      477. -
    477. Slightly different approach
      478. -
    478. Time decay rate
      479. -
    479. Code with a Number of Minibatches which varies
      480. -
    480. Replace or not
      481. -
    481. Momentum based GD
      482. -
    482. More on momentum based approaches
      483. -
    483. Momentum parameter
      484. -
    484. Second moment of the gradient
      485. -
    485. RMS prop
      486. -
    486. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      487. -
    487. Algorithms and codes for Adagrad, RMSprop and Adam
      488. -
    488. Practical tips
      489. -
    489. Sneaking in automatic differentiation using Autograd
      490. -
    490. Same code but now with momentum gradient descent
      491. -
    491. But none of these can compete with Newton's method
      492. -
    492. Including Stochastic Gradient Descent with Autograd
      493. -
    493. Same code but now with momentum gradient descent
      494. -
    494. Similar (second order function now) problem but now with AdaGrad
      495. -
    495. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      496. -
    496. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      497. -
    497. Material for the lab sessions
      498. +
    498. Pros and cons
      499. +
    499. Convergence rates
      500. +
    500. Accuracy
      501. +
    501. Stochastic Gradient Descent (SGD)
      502. +
    502. Stochastic Gradient Descent
      503. +
    503. Computation of gradients
      504. +
    504. SGD example
      505. +
    505. The gradient step
      506. +
    506. Simple example code
      507. +
    507. When do we stop?
      508. +
    508. Slightly different approach
      509. +
    509. Time decay rate
      510. +
    510. Code with a Number of Minibatches which varies
      511. +
    511. Replace or not
      512. +
    512. Momentum based GD
      513. +
    513. More on momentum based approaches
      514. +
    514. Momentum parameter
      515. +
    515. Second moment of the gradient
      516. +
    516. RMS prop
      517. +
    517. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      518. +
    518. Algorithms and codes for Adagrad, RMSprop and Adam
      519. +
    519. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      520. +
    520. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      521. +
    521. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      522. +
    522. Practical tips
      523. +
    523. Sneaking in auotmatic differentiation using Autograd
      524. +
    524. Same code but now with momentum gradient descent
      525. +
    525. Including Stochastic Gradient Descent with Autograd
      526. +
    526. Same code but now with momentum gradient descent
      527. +
    527. Similar (second order function now) problem but now with AdaGrad
      528. +
    528. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      529. +
    529. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      530. +
    530. Material for the lab sessions
      531. @@ -353,7 +372,7 @@ plt.show()
    531. 18
    532. 19
    533. ...
      534. -
    534. 46
      535. +
    535. 48
    536. »
      537. diff --git a/doc/pub/week37/html/._week37-bs010.html b/doc/pub/week37/html/._week37-bs010.html index 075aea3b0..26ec2439a 100644 --- a/doc/pub/week37/html/._week37-bs010.html +++ b/doc/pub/week37/html/._week37-bs010.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    537. Program example for gradient descent with Ridge Regression
    538. Using gradient descent methods, limitations
    539. Improving gradient descent with momentum
      540. -
    540. Same code but now with momentum gradient descent
      541. -
    541. Overview video on Stochastic Gradient Descent
      542. +
    542. Same code but now with momentum gradient descent
      543. +
    543. Overview video on Stochastic Gradient Descent (SGD)
    544. Batches and mini-batches
      545. -
    545. Stochastic Gradient Descent (SGD)
      546. -
    546. Stochastic Gradient Descent
      547. -
    547. Computation of gradients
      548. -
    548. SGD example
      549. -
    549. The gradient step
      550. -
    550. Simple example code
      551. -
    551. When do we stop?
      552. -
    552. Slightly different approach
      553. -
    553. Time decay rate
      554. -
    554. Code with a Number of Minibatches which varies
      555. -
    555. Replace or not
      556. -
    556. Momentum based GD
      557. -
    557. More on momentum based approaches
      558. -
    558. Momentum parameter
      559. -
    559. Second moment of the gradient
      560. -
    560. RMS prop
      561. -
    561. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      562. -
    562. Algorithms and codes for Adagrad, RMSprop and Adam
      563. -
    563. Practical tips
      564. -
    564. Sneaking in automatic differentiation using Autograd
      565. -
    565. Same code but now with momentum gradient descent
      566. -
    566. But none of these can compete with Newton's method
      567. -
    567. Including Stochastic Gradient Descent with Autograd
      568. -
    568. Same code but now with momentum gradient descent
      569. -
    569. Similar (second order function now) problem but now with AdaGrad
      570. -
    570. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      571. -
    571. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      572. -
    572. Material for the lab sessions
      573. +
    573. Pros and cons
      574. +
    574. Convergence rates
      575. +
    575. Accuracy
      576. +
    576. Stochastic Gradient Descent (SGD)
      577. +
    577. Stochastic Gradient Descent
      578. +
    578. Computation of gradients
      579. +
    579. SGD example
      580. +
    580. The gradient step
      581. +
    581. Simple example code
      582. +
    582. When do we stop?
      583. +
    583. Slightly different approach
      584. +
    584. Time decay rate
      585. +
    585. Code with a Number of Minibatches which varies
      586. +
    586. Replace or not
      587. +
    587. Momentum based GD
      588. +
    588. More on momentum based approaches
      589. +
    589. Momentum parameter
      590. +
    590. Second moment of the gradient
      591. +
    591. RMS prop
      592. +
    592. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      593. +
    593. Algorithms and codes for Adagrad, RMSprop and Adam
      594. +
    594. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      595. +
    595. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      596. +
    596. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      597. +
    597. Practical tips
      598. +
    598. Sneaking in auotmatic differentiation using Autograd
      599. +
    599. Same code but now with momentum gradient descent
      600. +
    600. Including Stochastic Gradient Descent with Autograd
      601. +
    601. Same code but now with momentum gradient descent
      602. +
    602. Similar (second order function now) problem but now with AdaGrad
      603. +
    603. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      604. +
    604. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      605. +
    605. Material for the lab sessions
      606. @@ -303,7 +322,7 @@ $$
    606. 19
    607. 20
    608. ...
      609. -
    609. 46
      610. +
    610. 48
    611. »
      612. diff --git a/doc/pub/week37/html/._week37-bs011.html b/doc/pub/week37/html/._week37-bs011.html index 87f9fdafa..fb806a5f7 100644 --- a/doc/pub/week37/html/._week37-bs011.html +++ b/doc/pub/week37/html/._week37-bs011.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    612. Program example for gradient descent with Ridge Regression
    613. Using gradient descent methods, limitations
    614. Improving gradient descent with momentum
      615. -
    615. Same code but now with momentum gradient descent
      616. -
    616. Overview video on Stochastic Gradient Descent
      617. +
    617. Same code but now with momentum gradient descent
      618. +
    618. Overview video on Stochastic Gradient Descent (SGD)
    619. Batches and mini-batches
      620. -
    620. Stochastic Gradient Descent (SGD)
      621. -
    621. Stochastic Gradient Descent
      622. -
    622. Computation of gradients
      623. -
    623. SGD example
      624. -
    624. The gradient step
      625. -
    625. Simple example code
      626. -
    626. When do we stop?
      627. -
    627. Slightly different approach
      628. -
    628. Time decay rate
      629. -
    629. Code with a Number of Minibatches which varies
      630. -
    630. Replace or not
      631. -
    631. Momentum based GD
      632. -
    632. More on momentum based approaches
      633. -
    633. Momentum parameter
      634. -
    634. Second moment of the gradient
      635. -
    635. RMS prop
      636. -
    636. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      637. -
    637. Algorithms and codes for Adagrad, RMSprop and Adam
      638. -
    638. Practical tips
      639. -
    639. Sneaking in automatic differentiation using Autograd
      640. -
    640. Same code but now with momentum gradient descent
      641. -
    641. But none of these can compete with Newton's method
      642. -
    642. Including Stochastic Gradient Descent with Autograd
      643. -
    643. Same code but now with momentum gradient descent
      644. -
    644. Similar (second order function now) problem but now with AdaGrad
      645. -
    645. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      646. -
    646. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      647. -
    647. Material for the lab sessions
      648. +
    648. Pros and cons
      649. +
    649. Convergence rates
      650. +
    650. Accuracy
      651. +
    651. Stochastic Gradient Descent (SGD)
      652. +
    652. Stochastic Gradient Descent
      653. +
    653. Computation of gradients
      654. +
    654. SGD example
      655. +
    655. The gradient step
      656. +
    656. Simple example code
      657. +
    657. When do we stop?
      658. +
    658. Slightly different approach
      659. +
    659. Time decay rate
      660. +
    660. Code with a Number of Minibatches which varies
      661. +
    661. Replace or not
      662. +
    662. Momentum based GD
      663. +
    663. More on momentum based approaches
      664. +
    664. Momentum parameter
      665. +
    665. Second moment of the gradient
      666. +
    666. RMS prop
      667. +
    667. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      668. +
    668. Algorithms and codes for Adagrad, RMSprop and Adam
      669. +
    669. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      670. +
    670. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      671. +
    671. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      672. +
    672. Practical tips
      673. +
    673. Sneaking in auotmatic differentiation using Autograd
      674. +
    674. Same code but now with momentum gradient descent
      675. +
    675. Including Stochastic Gradient Descent with Autograd
      676. +
    676. Same code but now with momentum gradient descent
      677. +
    677. Similar (second order function now) problem but now with AdaGrad
      678. +
    678. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      679. +
    679. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      680. +
    680. Material for the lab sessions
      681. @@ -299,7 +318,7 @@ minimum of this function.
    681. 20
    682. 21
    683. ...
      684. -
    684. 46
      685. +
    685. 48
    686. »
      687. diff --git a/doc/pub/week37/html/._week37-bs012.html b/doc/pub/week37/html/._week37-bs012.html index 90213a2d2..7a5311501 100644 --- a/doc/pub/week37/html/._week37-bs012.html +++ b/doc/pub/week37/html/._week37-bs012.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    687. Program example for gradient descent with Ridge Regression
    688. Using gradient descent methods, limitations
    689. Improving gradient descent with momentum
      690. -
    690. Same code but now with momentum gradient descent
      691. -
    691. Overview video on Stochastic Gradient Descent
      692. +
    692. Same code but now with momentum gradient descent
      693. +
    693. Overview video on Stochastic Gradient Descent (SGD)
    694. Batches and mini-batches
      695. -
    695. Stochastic Gradient Descent (SGD)
      696. -
    696. Stochastic Gradient Descent
      697. -
    697. Computation of gradients
      698. -
    698. SGD example
      699. -
    699. The gradient step
      700. -
    700. Simple example code
      701. -
    701. When do we stop?
      702. -
    702. Slightly different approach
      703. -
    703. Time decay rate
      704. -
    704. Code with a Number of Minibatches which varies
      705. -
    705. Replace or not
      706. -
    706. Momentum based GD
      707. -
    707. More on momentum based approaches
      708. -
    708. Momentum parameter
      709. -
    709. Second moment of the gradient
      710. -
    710. RMS prop
      711. -
    711. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      712. -
    712. Algorithms and codes for Adagrad, RMSprop and Adam
      713. -
    713. Practical tips
      714. -
    714. Sneaking in automatic differentiation using Autograd
      715. -
    715. Same code but now with momentum gradient descent
      716. -
    716. But none of these can compete with Newton's method
      717. -
    717. Including Stochastic Gradient Descent with Autograd
      718. -
    718. Same code but now with momentum gradient descent
      719. -
    719. Similar (second order function now) problem but now with AdaGrad
      720. -
    720. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      721. -
    721. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      722. -
    722. Material for the lab sessions
      723. +
    723. Pros and cons
      724. +
    724. Convergence rates
      725. +
    725. Accuracy
      726. +
    726. Stochastic Gradient Descent (SGD)
      727. +
    727. Stochastic Gradient Descent
      728. +
    728. Computation of gradients
      729. +
    729. SGD example
      730. +
    730. The gradient step
      731. +
    731. Simple example code
      732. +
    732. When do we stop?
      733. +
    733. Slightly different approach
      734. +
    734. Time decay rate
      735. +
    735. Code with a Number of Minibatches which varies
      736. +
    736. Replace or not
      737. +
    737. Momentum based GD
      738. +
    738. More on momentum based approaches
      739. +
    739. Momentum parameter
      740. +
    740. Second moment of the gradient
      741. +
    741. RMS prop
      742. +
    742. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      743. +
    743. Algorithms and codes for Adagrad, RMSprop and Adam
      744. +
    744. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      745. +
    745. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      746. +
    746. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      747. +
    747. Practical tips
      748. +
    748. Sneaking in auotmatic differentiation using Autograd
      749. +
    749. Same code but now with momentum gradient descent
      750. +
    750. Including Stochastic Gradient Descent with Autograd
      751. +
    751. Same code but now with momentum gradient descent
      752. +
    752. Similar (second order function now) problem but now with AdaGrad
      753. +
    753. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      754. +
    754. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      755. +
    755. Material for the lab sessions
      756. @@ -357,7 +376,7 @@ plt.show()
    756. 21
    757. 22
    758. ...
      759. -
    759. 46
      760. +
    760. 48
    761. »
      762. diff --git a/doc/pub/week37/html/._week37-bs013.html b/doc/pub/week37/html/._week37-bs013.html index 0a3af037d..e4ea98d20 100644 --- a/doc/pub/week37/html/._week37-bs013.html +++ b/doc/pub/week37/html/._week37-bs013.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    762. Program example for gradient descent with Ridge Regression
    763. Using gradient descent methods, limitations
    764. Improving gradient descent with momentum
      765. -
    765. Same code but now with momentum gradient descent
      766. -
    766. Overview video on Stochastic Gradient Descent
      767. +
    767. Same code but now with momentum gradient descent
      768. +
    768. Overview video on Stochastic Gradient Descent (SGD)
    769. Batches and mini-batches
      770. -
    770. Stochastic Gradient Descent (SGD)
      771. -
    771. Stochastic Gradient Descent
      772. -
    772. Computation of gradients
      773. -
    773. SGD example
      774. -
    774. The gradient step
      775. -
    775. Simple example code
      776. -
    776. When do we stop?
      777. -
    777. Slightly different approach
      778. -
    778. Time decay rate
      779. -
    779. Code with a Number of Minibatches which varies
      780. -
    780. Replace or not
      781. -
    781. Momentum based GD
      782. -
    782. More on momentum based approaches
      783. -
    783. Momentum parameter
      784. -
    784. Second moment of the gradient
      785. -
    785. RMS prop
      786. -
    786. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      787. -
    787. Algorithms and codes for Adagrad, RMSprop and Adam
      788. -
    788. Practical tips
      789. -
    789. Sneaking in automatic differentiation using Autograd
      790. -
    790. Same code but now with momentum gradient descent
      791. -
    791. But none of these can compete with Newton's method
      792. -
    792. Including Stochastic Gradient Descent with Autograd
      793. -
    793. Same code but now with momentum gradient descent
      794. -
    794. Similar (second order function now) problem but now with AdaGrad
      795. -
    795. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      796. -
    796. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      797. -
    797. Material for the lab sessions
      798. +
    798. Pros and cons
      799. +
    799. Convergence rates
      800. +
    800. Accuracy
      801. +
    801. Stochastic Gradient Descent (SGD)
      802. +
    802. Stochastic Gradient Descent
      803. +
    803. Computation of gradients
      804. +
    804. SGD example
      805. +
    805. The gradient step
      806. +
    806. Simple example code
      807. +
    807. When do we stop?
      808. +
    808. Slightly different approach
      809. +
    809. Time decay rate
      810. +
    810. Code with a Number of Minibatches which varies
      811. +
    811. Replace or not
      812. +
    812. Momentum based GD
      813. +
    813. More on momentum based approaches
      814. +
    814. Momentum parameter
      815. +
    815. Second moment of the gradient
      816. +
    816. RMS prop
      817. +
    817. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      818. +
    818. Algorithms and codes for Adagrad, RMSprop and Adam
      819. +
    819. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      820. +
    820. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      821. +
    821. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      822. +
    822. Practical tips
      823. +
    823. Sneaking in auotmatic differentiation using Autograd
      824. +
    824. Same code but now with momentum gradient descent
      825. +
    825. Including Stochastic Gradient Descent with Autograd
      826. +
    826. Same code but now with momentum gradient descent
      827. +
    827. Similar (second order function now) problem but now with AdaGrad
      828. +
    828. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      829. +
    829. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      830. +
    830. Material for the lab sessions
      831. @@ -293,7 +312,7 @@ MathJax.Hub.Config({
    831. 22
    832. 23
    833. ...
      834. -
    834. 46
      835. +
    835. 48
    836. »
      837. diff --git a/doc/pub/week37/html/._week37-bs014.html b/doc/pub/week37/html/._week37-bs014.html index 2d1cd02f8..19d9ba2cd 100644 --- a/doc/pub/week37/html/._week37-bs014.html +++ b/doc/pub/week37/html/._week37-bs014.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    837. Program example for gradient descent with Ridge Regression
    838. Using gradient descent methods, limitations
    839. Improving gradient descent with momentum
      840. -
    840. Same code but now with momentum gradient descent
      841. -
    841. Overview video on Stochastic Gradient Descent
      842. +
    842. Same code but now with momentum gradient descent
      843. +
    843. Overview video on Stochastic Gradient Descent (SGD)
    844. Batches and mini-batches
      845. -
    845. Stochastic Gradient Descent (SGD)
      846. -
    846. Stochastic Gradient Descent
      847. -
    847. Computation of gradients
      848. -
    848. SGD example
      849. -
    849. The gradient step
      850. -
    850. Simple example code
      851. -
    851. When do we stop?
      852. -
    852. Slightly different approach
      853. -
    853. Time decay rate
      854. -
    854. Code with a Number of Minibatches which varies
      855. -
    855. Replace or not
      856. -
    856. Momentum based GD
      857. -
    857. More on momentum based approaches
      858. -
    858. Momentum parameter
      859. -
    859. Second moment of the gradient
      860. -
    860. RMS prop
      861. -
    861. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      862. -
    862. Algorithms and codes for Adagrad, RMSprop and Adam
      863. -
    863. Practical tips
      864. -
    864. Sneaking in automatic differentiation using Autograd
      865. -
    865. Same code but now with momentum gradient descent
      866. -
    866. But none of these can compete with Newton's method
      867. -
    867. Including Stochastic Gradient Descent with Autograd
      868. -
    868. Same code but now with momentum gradient descent
      869. -
    869. Similar (second order function now) problem but now with AdaGrad
      870. -
    870. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      871. -
    871. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      872. -
    872. Material for the lab sessions
      873. +
    873. Pros and cons
      874. +
    874. Convergence rates
      875. +
    875. Accuracy
      876. +
    876. Stochastic Gradient Descent (SGD)
      877. +
    877. Stochastic Gradient Descent
      878. +
    878. Computation of gradients
      879. +
    879. SGD example
      880. +
    880. The gradient step
      881. +
    881. Simple example code
      882. +
    882. When do we stop?
      883. +
    883. Slightly different approach
      884. +
    884. Time decay rate
      885. +
    885. Code with a Number of Minibatches which varies
      886. +
    886. Replace or not
      887. +
    887. Momentum based GD
      888. +
    888. More on momentum based approaches
      889. +
    889. Momentum parameter
      890. +
    890. Second moment of the gradient
      891. +
    891. RMS prop
      892. +
    892. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      893. +
    893. Algorithms and codes for Adagrad, RMSprop and Adam
      894. +
    894. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      895. +
    895. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      896. +
    896. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      897. +
    897. Practical tips
      898. +
    898. Sneaking in auotmatic differentiation using Autograd
      899. +
    899. Same code but now with momentum gradient descent
      900. +
    900. Including Stochastic Gradient Descent with Autograd
      901. +
    901. Same code but now with momentum gradient descent
      902. +
    902. Similar (second order function now) problem but now with AdaGrad
      903. +
    903. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      904. +
    904. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      905. +
    905. Material for the lab sessions
      906. @@ -365,7 +384,7 @@ pyplot.show()
    906. 23
    907. 24
    908. ...
      909. -
    909. 46
      910. +
    910. 48
    911. »
      912. diff --git a/doc/pub/week37/html/._week37-bs015.html b/doc/pub/week37/html/._week37-bs015.html index 39be69329..9df5139ad 100644 --- a/doc/pub/week37/html/._week37-bs015.html +++ b/doc/pub/week37/html/._week37-bs015.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    912. Program example for gradient descent with Ridge Regression
    913. Using gradient descent methods, limitations
    914. Improving gradient descent with momentum
      915. -
    915. Same code but now with momentum gradient descent
      916. -
    916. Overview video on Stochastic Gradient Descent
      917. +
    917. Same code but now with momentum gradient descent
      918. +
    918. Overview video on Stochastic Gradient Descent (SGD)
    919. Batches and mini-batches
      920. -
    920. Stochastic Gradient Descent (SGD)
      921. -
    921. Stochastic Gradient Descent
      922. -
    922. Computation of gradients
      923. -
    923. SGD example
      924. -
    924. The gradient step
      925. -
    925. Simple example code
      926. -
    926. When do we stop?
      927. -
    927. Slightly different approach
      928. -
    928. Time decay rate
      929. -
    929. Code with a Number of Minibatches which varies
      930. -
    930. Replace or not
      931. -
    931. Momentum based GD
      932. -
    932. More on momentum based approaches
      933. -
    933. Momentum parameter
      934. -
    934. Second moment of the gradient
      935. -
    935. RMS prop
      936. -
    936. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      937. -
    937. Algorithms and codes for Adagrad, RMSprop and Adam
      938. -
    938. Practical tips
      939. -
    939. Sneaking in automatic differentiation using Autograd
      940. -
    940. Same code but now with momentum gradient descent
      941. -
    941. But none of these can compete with Newton's method
      942. -
    942. Including Stochastic Gradient Descent with Autograd
      943. -
    943. Same code but now with momentum gradient descent
      944. -
    944. Similar (second order function now) problem but now with AdaGrad
      945. -
    945. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      946. -
    946. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      947. -
    947. Material for the lab sessions
      948. +
    948. Pros and cons
      949. +
    949. Convergence rates
      950. +
    950. Accuracy
      951. +
    951. Stochastic Gradient Descent (SGD)
      952. +
    952. Stochastic Gradient Descent
      953. +
    953. Computation of gradients
      954. +
    954. SGD example
      955. +
    955. The gradient step
      956. +
    956. Simple example code
      957. +
    957. When do we stop?
      958. +
    958. Slightly different approach
      959. +
    959. Time decay rate
      960. +
    960. Code with a Number of Minibatches which varies
      961. +
    961. Replace or not
      962. +
    962. Momentum based GD
      963. +
    963. More on momentum based approaches
      964. +
    964. Momentum parameter
      965. +
    965. Second moment of the gradient
      966. +
    966. RMS prop
      967. +
    967. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      968. +
    968. Algorithms and codes for Adagrad, RMSprop and Adam
      969. +
    969. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      970. +
    970. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      971. +
    971. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      972. +
    972. Practical tips
      973. +
    973. Sneaking in auotmatic differentiation using Autograd
      974. +
    974. Same code but now with momentum gradient descent
      975. +
    975. Including Stochastic Gradient Descent with Autograd
      976. +
    976. Same code but now with momentum gradient descent
      977. +
    977. Similar (second order function now) problem but now with AdaGrad
      978. +
    978. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      979. +
    979. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      980. +
    980. Material for the lab sessions
      981. @@ -371,7 +390,7 @@ pyplot.show()
    981. 24
    982. 25
    983. ...
      984. -
    984. 46
      985. +
    985. 48
    986. »
      987. diff --git a/doc/pub/week37/html/._week37-bs016.html b/doc/pub/week37/html/._week37-bs016.html index 1b3515b56..13617ec8b 100644 --- a/doc/pub/week37/html/._week37-bs016.html +++ b/doc/pub/week37/html/._week37-bs016.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    987. Program example for gradient descent with Ridge Regression
    988. Using gradient descent methods, limitations
    989. Improving gradient descent with momentum
      990. -
    990. Same code but now with momentum gradient descent
      991. -
    991. Overview video on Stochastic Gradient Descent
      992. +
    992. Same code but now with momentum gradient descent
      993. +
    993. Overview video on Stochastic Gradient Descent (SGD)
    994. Batches and mini-batches
      995. -
    995. Stochastic Gradient Descent (SGD)
      996. -
    996. Stochastic Gradient Descent
      997. -
    997. Computation of gradients
      998. -
    998. SGD example
      999. -
    999. The gradient step
      1000. -
    1000. Simple example code
      1001. -
    1001. When do we stop?
      1002. -
    1002. Slightly different approach
      1003. -
    1003. Time decay rate
      1004. -
    1004. Code with a Number of Minibatches which varies
      1005. -
    1005. Replace or not
      1006. -
    1006. Momentum based GD
      1007. -
    1007. More on momentum based approaches
      1008. -
    1008. Momentum parameter
      1009. -
    1009. Second moment of the gradient
      1010. -
    1010. RMS prop
      1011. -
    1011. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1012. -
    1012. Algorithms and codes for Adagrad, RMSprop and Adam
      1013. -
    1013. Practical tips
      1014. -
    1014. Sneaking in automatic differentiation using Autograd
      1015. -
    1015. Same code but now with momentum gradient descent
      1016. -
    1016. But none of these can compete with Newton's method
      1017. -
    1017. Including Stochastic Gradient Descent with Autograd
      1018. -
    1018. Same code but now with momentum gradient descent
      1019. -
    1019. Similar (second order function now) problem but now with AdaGrad
      1020. -
    1020. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1021. -
    1021. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1022. -
    1022. Material for the lab sessions
      1023. +
    1023. Pros and cons
      1024. +
    1024. Convergence rates
      1025. +
    1025. Accuracy
      1026. +
    1026. Stochastic Gradient Descent (SGD)
      1027. +
    1027. Stochastic Gradient Descent
      1028. +
    1028. Computation of gradients
      1029. +
    1029. SGD example
      1030. +
    1030. The gradient step
      1031. +
    1031. Simple example code
      1032. +
    1032. When do we stop?
      1033. +
    1033. Slightly different approach
      1034. +
    1034. Time decay rate
      1035. +
    1035. Code with a Number of Minibatches which varies
      1036. +
    1036. Replace or not
      1037. +
    1037. Momentum based GD
      1038. +
    1038. More on momentum based approaches
      1039. +
    1039. Momentum parameter
      1040. +
    1040. Second moment of the gradient
      1041. +
    1041. RMS prop
      1042. +
    1042. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1043. +
    1043. Algorithms and codes for Adagrad, RMSprop and Adam
      1044. +
    1044. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1045. +
    1045. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1046. +
    1046. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1047. +
    1047. Practical tips
      1048. +
    1048. Sneaking in auotmatic differentiation using Autograd
      1049. +
    1049. Same code but now with momentum gradient descent
      1050. +
    1050. Including Stochastic Gradient Descent with Autograd
      1051. +
    1051. Same code but now with momentum gradient descent
      1052. +
    1052. Similar (second order function now) problem but now with AdaGrad
      1053. +
    1053. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1054. +
    1054. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1055. +
    1055. Material for the lab sessions
      1056. @@ -258,10 +277,16 @@ MathJax.Hub.Config({

       

       

       

      -

      Overview video on Stochastic Gradient Descent

      +

      Overview video on Stochastic Gradient Descent (SGD)

      What is Stochastic Gradient Descent +

      There are several reasons for using stochastic gradient descent. Some of these are:

      +
        +
      1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.
        2. +
      2. Hopefully avoid Local Minima
        3. +
      3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset.
        4. +

        @@ -287,7 +312,7 @@ MathJax.Hub.Config({
      • 25
      • 26
      • ...
        • -
      • 46
        • +
      • 48
      • »
      diff --git a/doc/pub/week37/html/._week37-bs017.html b/doc/pub/week37/html/._week37-bs017.html index e1882aad3..8dde5e1e5 100644 --- a/doc/pub/week37/html/._week37-bs017.html +++ b/doc/pub/week37/html/._week37-bs017.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1056. Program example for gradient descent with Ridge Regression
    1057. Using gradient descent methods, limitations
    1058. Improving gradient descent with momentum
      1059. -
    1059. Same code but now with momentum gradient descent
      1060. -
    1060. Overview video on Stochastic Gradient Descent
      1061. +
    1061. Same code but now with momentum gradient descent
      1062. +
    1062. Overview video on Stochastic Gradient Descent (SGD)
    1063. Batches and mini-batches
      1064. -
    1064. Stochastic Gradient Descent (SGD)
      1065. -
    1065. Stochastic Gradient Descent
      1066. -
    1066. Computation of gradients
      1067. -
    1067. SGD example
      1068. -
    1068. The gradient step
      1069. -
    1069. Simple example code
      1070. -
    1070. When do we stop?
      1071. -
    1071. Slightly different approach
      1072. -
    1072. Time decay rate
      1073. -
    1073. Code with a Number of Minibatches which varies
      1074. -
    1074. Replace or not
      1075. -
    1075. Momentum based GD
      1076. -
    1076. More on momentum based approaches
      1077. -
    1077. Momentum parameter
      1078. -
    1078. Second moment of the gradient
      1079. -
    1079. RMS prop
      1080. -
    1080. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1081. -
    1081. Algorithms and codes for Adagrad, RMSprop and Adam
      1082. -
    1082. Practical tips
      1083. -
    1083. Sneaking in automatic differentiation using Autograd
      1084. -
    1084. Same code but now with momentum gradient descent
      1085. -
    1085. But none of these can compete with Newton's method
      1086. -
    1086. Including Stochastic Gradient Descent with Autograd
      1087. -
    1087. Same code but now with momentum gradient descent
      1088. -
    1088. Similar (second order function now) problem but now with AdaGrad
      1089. -
    1089. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1090. -
    1090. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1091. -
    1091. Material for the lab sessions
      1092. +
    1092. Pros and cons
      1093. +
    1093. Convergence rates
      1094. +
    1094. Accuracy
      1095. +
    1095. Stochastic Gradient Descent (SGD)
      1096. +
    1096. Stochastic Gradient Descent
      1097. +
    1097. Computation of gradients
      1098. +
    1098. SGD example
      1099. +
    1099. The gradient step
      1100. +
    1100. Simple example code
      1101. +
    1101. When do we stop?
      1102. +
    1102. Slightly different approach
      1103. +
    1103. Time decay rate
      1104. +
    1104. Code with a Number of Minibatches which varies
      1105. +
    1105. Replace or not
      1106. +
    1106. Momentum based GD
      1107. +
    1107. More on momentum based approaches
      1108. +
    1108. Momentum parameter
      1109. +
    1109. Second moment of the gradient
      1110. +
    1110. RMS prop
      1111. +
    1111. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1112. +
    1112. Algorithms and codes for Adagrad, RMSprop and Adam
      1113. +
    1113. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1114. +
    1114. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1115. +
    1115. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1116. +
    1116. Practical tips
      1117. +
    1117. Sneaking in auotmatic differentiation using Autograd
      1118. +
    1118. Same code but now with momentum gradient descent
      1119. +
    1119. Including Stochastic Gradient Descent with Autograd
      1120. +
    1120. Same code but now with momentum gradient descent
      1121. +
    1121. Similar (second order function now) problem but now with AdaGrad
      1122. +
    1122. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1123. +
    1123. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1124. +
    1124. Material for the lab sessions
      1125. @@ -297,7 +316,7 @@ perform a parameter update.
    1125. 26
    1126. 27
    1127. ...
      1128. -
    1128. 46
      1129. +
    1129. 48
    1130. »
      1131. diff --git a/doc/pub/week37/html/._week37-bs018.html b/doc/pub/week37/html/._week37-bs018.html index a4b654a0b..86155fa0d 100644 --- a/doc/pub/week37/html/._week37-bs018.html +++ b/doc/pub/week37/html/._week37-bs018.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1131. Program example for gradient descent with Ridge Regression
    1132. Using gradient descent methods, limitations
    1133. Improving gradient descent with momentum
      1134. -
    1134. Same code but now with momentum gradient descent
      1135. -
    1135. Overview video on Stochastic Gradient Descent
      1136. +
    1136. Same code but now with momentum gradient descent
      1137. +
    1137. Overview video on Stochastic Gradient Descent (SGD)
    1138. Batches and mini-batches
      1139. -
    1139. Stochastic Gradient Descent (SGD)
      1140. -
    1140. Stochastic Gradient Descent
      1141. -
    1141. Computation of gradients
      1142. -
    1142. SGD example
      1143. -
    1143. The gradient step
      1144. -
    1144. Simple example code
      1145. -
    1145. When do we stop?
      1146. -
    1146. Slightly different approach
      1147. -
    1147. Time decay rate
      1148. -
    1148. Code with a Number of Minibatches which varies
      1149. -
    1149. Replace or not
      1150. -
    1150. Momentum based GD
      1151. -
    1151. More on momentum based approaches
      1152. -
    1152. Momentum parameter
      1153. -
    1153. Second moment of the gradient
      1154. -
    1154. RMS prop
      1155. -
    1155. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1156. -
    1156. Algorithms and codes for Adagrad, RMSprop and Adam
      1157. -
    1157. Practical tips
      1158. -
    1158. Sneaking in automatic differentiation using Autograd
      1159. -
    1159. Same code but now with momentum gradient descent
      1160. -
    1160. But none of these can compete with Newton's method
      1161. -
    1161. Including Stochastic Gradient Descent with Autograd
      1162. -
    1162. Same code but now with momentum gradient descent
      1163. -
    1163. Similar (second order function now) problem but now with AdaGrad
      1164. -
    1164. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1165. -
    1165. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1166. -
    1166. Material for the lab sessions
      1167. +
    1167. Pros and cons
      1168. +
    1168. Convergence rates
      1169. +
    1169. Accuracy
      1170. +
    1170. Stochastic Gradient Descent (SGD)
      1171. +
    1171. Stochastic Gradient Descent
      1172. +
    1172. Computation of gradients
      1173. +
    1173. SGD example
      1174. +
    1174. The gradient step
      1175. +
    1175. Simple example code
      1176. +
    1176. When do we stop?
      1177. +
    1177. Slightly different approach
      1178. +
    1178. Time decay rate
      1179. +
    1179. Code with a Number of Minibatches which varies
      1180. +
    1180. Replace or not
      1181. +
    1181. Momentum based GD
      1182. +
    1182. More on momentum based approaches
      1183. +
    1183. Momentum parameter
      1184. +
    1184. Second moment of the gradient
      1185. +
    1185. RMS prop
      1186. +
    1186. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1187. +
    1187. Algorithms and codes for Adagrad, RMSprop and Adam
      1188. +
    1188. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1189. +
    1189. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1190. +
    1190. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1191. +
    1191. Practical tips
      1192. +
    1192. Sneaking in auotmatic differentiation using Autograd
      1193. +
    1193. Same code but now with momentum gradient descent
      1194. +
    1194. Including Stochastic Gradient Descent with Autograd
      1195. +
    1195. Same code but now with momentum gradient descent
      1196. +
    1196. Similar (second order function now) problem but now with AdaGrad
      1197. +
    1197. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1198. +
    1198. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1199. +
    1199. Material for the lab sessions
      1200. @@ -258,32 +277,13 @@ MathJax.Hub.Config({

       

       

       

      -

      Stochastic Gradient Descent (SGD)

      - -

      In stochastic gradient descent, the extreme case is the case where we -have only one batch, that is we include the whole data set. -

      - -

      This process is called Stochastic Gradient -Descent (SGD) (or also sometimes on-line gradient descent). This is -relatively less common to see because in practice due to vectorized -code optimizations it can be computationally much more efficient to -evaluate the gradient for 100 examples, than the gradient for one -example 100 times. Even though SGD technically refers to using a -single example at a time to evaluate the gradient, you will hear -people use the term SGD even when referring to mini-batch gradient -descent (i.e. mentions of MGD for “Minibatch Gradient Descent”, or BGD -for “Batch gradient descent” are rare to see), where it is usually -assumed that mini-batches are used. The size of the mini-batch is a -hyperparameter but it is not very common to cross-validate or bootstrap it. It is -usually based on memory constraints (if any), or set to some value, -e.g. 32, 64 or 128. We use powers of 2 in practice because many -vectorized operation implementations work faster when their inputs are -sized in powers of 2. -

      - -

      In our notes with SGD we mean stochastic gradient descent with mini-batches.

      +

      Pros and cons

      +
        +
      1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.
        2. +
      2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.
        3. +
      3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient.
        4. +

        @@ -309,7 +309,7 @@ sized in powers of 2.
      • 27
      • 28
      • ...
        • -
      • 46
        • +
      • 48
      • »
      diff --git a/doc/pub/week37/html/._week37-bs019.html b/doc/pub/week37/html/._week37-bs019.html index ceb6547d6..bd928e866 100644 --- a/doc/pub/week37/html/._week37-bs019.html +++ b/doc/pub/week37/html/._week37-bs019.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1200. Program example for gradient descent with Ridge Regression
    1201. Using gradient descent methods, limitations
    1202. Improving gradient descent with momentum
      1203. -
    1203. Same code but now with momentum gradient descent
      1204. -
    1204. Overview video on Stochastic Gradient Descent
      1205. +
    1205. Same code but now with momentum gradient descent
      1206. +
    1206. Overview video on Stochastic Gradient Descent (SGD)
    1207. Batches and mini-batches
      1208. -
    1208. Stochastic Gradient Descent (SGD)
      1209. -
    1209. Stochastic Gradient Descent
      1210. -
    1210. Computation of gradients
      1211. -
    1211. SGD example
      1212. -
    1212. The gradient step
      1213. -
    1213. Simple example code
      1214. -
    1214. When do we stop?
      1215. -
    1215. Slightly different approach
      1216. -
    1216. Time decay rate
      1217. -
    1217. Code with a Number of Minibatches which varies
      1218. -
    1218. Replace or not
      1219. -
    1219. Momentum based GD
      1220. -
    1220. More on momentum based approaches
      1221. -
    1221. Momentum parameter
      1222. -
    1222. Second moment of the gradient
      1223. -
    1223. RMS prop
      1224. -
    1224. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1225. -
    1225. Algorithms and codes for Adagrad, RMSprop and Adam
      1226. -
    1226. Practical tips
      1227. -
    1227. Sneaking in automatic differentiation using Autograd
      1228. -
    1228. Same code but now with momentum gradient descent
      1229. -
    1229. But none of these can compete with Newton's method
      1230. -
    1230. Including Stochastic Gradient Descent with Autograd
      1231. -
    1231. Same code but now with momentum gradient descent
      1232. -
    1232. Similar (second order function now) problem but now with AdaGrad
      1233. -
    1233. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1234. -
    1234. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1235. -
    1235. Material for the lab sessions
      1236. +
    1236. Pros and cons
      1237. +
    1237. Convergence rates
      1238. +
    1238. Accuracy
      1239. +
    1239. Stochastic Gradient Descent (SGD)
      1240. +
    1240. Stochastic Gradient Descent
      1241. +
    1241. Computation of gradients
      1242. +
    1242. SGD example
      1243. +
    1243. The gradient step
      1244. +
    1244. Simple example code
      1245. +
    1245. When do we stop?
      1246. +
    1246. Slightly different approach
      1247. +
    1247. Time decay rate
      1248. +
    1248. Code with a Number of Minibatches which varies
      1249. +
    1249. Replace or not
      1250. +
    1250. Momentum based GD
      1251. +
    1251. More on momentum based approaches
      1252. +
    1252. Momentum parameter
      1253. +
    1253. Second moment of the gradient
      1254. +
    1254. RMS prop
      1255. +
    1255. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1256. +
    1256. Algorithms and codes for Adagrad, RMSprop and Adam
      1257. +
    1257. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1258. +
    1258. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1259. +
    1259. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1260. +
    1260. Practical tips
      1261. +
    1261. Sneaking in auotmatic differentiation using Autograd
      1262. +
    1262. Same code but now with momentum gradient descent
      1263. +
    1263. Including Stochastic Gradient Descent with Autograd
      1264. +
    1264. Same code but now with momentum gradient descent
      1265. +
    1265. Similar (second order function now) problem but now with AdaGrad
      1266. +
    1266. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1267. +
    1267. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1268. +
    1268. Material for the lab sessions
      1269. @@ -258,22 +277,12 @@ MathJax.Hub.Config({

       

       

       

      -

      Stochastic Gradient Descent

      - -

      Stochastic gradient descent (SGD) and variants thereof address some of -the shortcomings of the Gradient descent method discussed above. -

      - -

      The underlying idea of SGD comes from the observation that the cost -function, which we want to minimize, can almost always be written as a -sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \), -

      -$$ -C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i, -\mathbf{\beta}). -$$ - +

      Convergence rates

      +
        +
      1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.
        2. +
      2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration.
        3. +

      diff --git a/doc/pub/week37/html/._week37-bs020.html b/doc/pub/week37/html/._week37-bs020.html index 75db11f4d..ed9d1f077 100644 --- a/doc/pub/week37/html/._week37-bs020.html +++ b/doc/pub/week37/html/._week37-bs020.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1269. Program example for gradient descent with Ridge Regression
    1270. Using gradient descent methods, limitations
    1271. Improving gradient descent with momentum
      1272. -
    1272. Same code but now with momentum gradient descent
      1273. -
    1273. Overview video on Stochastic Gradient Descent
      1274. +
    1274. Same code but now with momentum gradient descent
      1275. +
    1275. Overview video on Stochastic Gradient Descent (SGD)
    1276. Batches and mini-batches
      1277. -
    1277. Stochastic Gradient Descent (SGD)
      1278. -
    1278. Stochastic Gradient Descent
      1279. -
    1279. Computation of gradients
      1280. -
    1280. SGD example
      1281. -
    1281. The gradient step
      1282. -
    1282. Simple example code
      1283. -
    1283. When do we stop?
      1284. -
    1284. Slightly different approach
      1285. -
    1285. Time decay rate
      1286. -
    1286. Code with a Number of Minibatches which varies
      1287. -
    1287. Replace or not
      1288. -
    1288. Momentum based GD
      1289. -
    1289. More on momentum based approaches
      1290. -
    1290. Momentum parameter
      1291. -
    1291. Second moment of the gradient
      1292. -
    1292. RMS prop
      1293. -
    1293. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1294. -
    1294. Algorithms and codes for Adagrad, RMSprop and Adam
      1295. -
    1295. Practical tips
      1296. -
    1296. Sneaking in automatic differentiation using Autograd
      1297. -
    1297. Same code but now with momentum gradient descent
      1298. -
    1298. But none of these can compete with Newton's method
      1299. -
    1299. Including Stochastic Gradient Descent with Autograd
      1300. -
    1300. Same code but now with momentum gradient descent
      1301. -
    1301. Similar (second order function now) problem but now with AdaGrad
      1302. -
    1302. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1303. -
    1303. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1304. -
    1304. Material for the lab sessions
      1305. +
    1305. Pros and cons
      1306. +
    1306. Convergence rates
      1307. +
    1307. Accuracy
      1308. +
    1308. Stochastic Gradient Descent (SGD)
      1309. +
    1309. Stochastic Gradient Descent
      1310. +
    1310. Computation of gradients
      1311. +
    1311. SGD example
      1312. +
    1312. The gradient step
      1313. +
    1313. Simple example code
      1314. +
    1314. When do we stop?
      1315. +
    1315. Slightly different approach
      1316. +
    1316. Time decay rate
      1317. +
    1317. Code with a Number of Minibatches which varies
      1318. +
    1318. Replace or not
      1319. +
    1319. Momentum based GD
      1320. +
    1320. More on momentum based approaches
      1321. +
    1321. Momentum parameter
      1322. +
    1322. Second moment of the gradient
      1323. +
    1323. RMS prop
      1324. +
    1324. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1325. +
    1325. Algorithms and codes for Adagrad, RMSprop and Adam
      1326. +
    1326. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1327. +
    1327. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1328. +
    1328. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1329. +
    1329. Practical tips
      1330. +
    1330. Sneaking in auotmatic differentiation using Autograd
      1331. +
    1331. Same code but now with momentum gradient descent
      1332. +
    1332. Including Stochastic Gradient Descent with Autograd
      1333. +
    1333. Same code but now with momentum gradient descent
      1334. +
    1334. Similar (second order function now) problem but now with AdaGrad
      1335. +
    1335. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1336. +
    1336. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1337. +
    1337. Material for the lab sessions
      1338. @@ -258,21 +277,27 @@ MathJax.Hub.Config({

       

       

       

      -

      Computation of gradients

      +

      Accuracy

      -

      This in turn means that the gradient can be -computed as a sum over \( i \)-gradients +

      In general, stochastic Gradient Descent is Less accurate than gradient +descent, as it calculates the gradient on single examples, which may +not accurately represent the overall dataset. Gradient Descent is +more accurate because it uses the average gradient calculated over the +entire dataset.

      -$$ -\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i, -\mathbf{\beta}). -$$ -

      Stochasticity/randomness is introduced by only taking the -gradient on a subset of the data called minibatches. If there are \( n \) -data points and the size of each minibatch is \( M \), there will be \( n/M \) -minibatches. We denote these minibatches by \( B_k \) where -\( k=1,\cdots,n/M \). +

      There are other disadvantages to using SGD. The main drawback is that +its convergence behaviour can be more erratic due to the random +sampling of individual training examples. This can lead to less +accurate results, as the algorithm may not converge to the true +minimum of the cost function. Additionally, the learning rate, which +determines the step size of each update to the model’s parameters, +must be carefully chosen to ensure convergence. +

      + +

      It is however the method of choice in deep learning algorithms where +SGD is often used in combination with other optimization techniques, +such as momentum or adaptive learning rates

      @@ -300,7 +325,7 @@ minibatches. We denote these minibatches by \( B_k \) where

    1338. 29
    1339. 30
    1340. ...
      1341. -
    1341. 46
      1342. +
    1342. 48
    1343. »
      1344. diff --git a/doc/pub/week37/html/._week37-bs021.html b/doc/pub/week37/html/._week37-bs021.html index 4a4d3fe14..3533677e4 100644 --- a/doc/pub/week37/html/._week37-bs021.html +++ b/doc/pub/week37/html/._week37-bs021.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1344. Program example for gradient descent with Ridge Regression
    1345. Using gradient descent methods, limitations
    1346. Improving gradient descent with momentum
      1347. -
    1347. Same code but now with momentum gradient descent
      1348. -
    1348. Overview video on Stochastic Gradient Descent
      1349. +
    1349. Same code but now with momentum gradient descent
      1350. +
    1350. Overview video on Stochastic Gradient Descent (SGD)
    1351. Batches and mini-batches
      1352. -
    1352. Stochastic Gradient Descent (SGD)
      1353. -
    1353. Stochastic Gradient Descent
      1354. -
    1354. Computation of gradients
      1355. -
    1355. SGD example
      1356. -
    1356. The gradient step
      1357. -
    1357. Simple example code
      1358. -
    1358. When do we stop?
      1359. -
    1359. Slightly different approach
      1360. -
    1360. Time decay rate
      1361. -
    1361. Code with a Number of Minibatches which varies
      1362. -
    1362. Replace or not
      1363. -
    1363. Momentum based GD
      1364. -
    1364. More on momentum based approaches
      1365. -
    1365. Momentum parameter
      1366. -
    1366. Second moment of the gradient
      1367. -
    1367. RMS prop
      1368. -
    1368. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1369. -
    1369. Algorithms and codes for Adagrad, RMSprop and Adam
      1370. -
    1370. Practical tips
      1371. -
    1371. Sneaking in automatic differentiation using Autograd
      1372. -
    1372. Same code but now with momentum gradient descent
      1373. -
    1373. But none of these can compete with Newton's method
      1374. -
    1374. Including Stochastic Gradient Descent with Autograd
      1375. -
    1375. Same code but now with momentum gradient descent
      1376. -
    1376. Similar (second order function now) problem but now with AdaGrad
      1377. -
    1377. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1378. -
    1378. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1379. -
    1379. Material for the lab sessions
      1380. +
    1380. Pros and cons
      1381. +
    1381. Convergence rates
      1382. +
    1382. Accuracy
      1383. +
    1383. Stochastic Gradient Descent (SGD)
      1384. +
    1384. Stochastic Gradient Descent
      1385. +
    1385. Computation of gradients
      1386. +
    1386. SGD example
      1387. +
    1387. The gradient step
      1388. +
    1388. Simple example code
      1389. +
    1389. When do we stop?
      1390. +
    1390. Slightly different approach
      1391. +
    1391. Time decay rate
      1392. +
    1392. Code with a Number of Minibatches which varies
      1393. +
    1393. Replace or not
      1394. +
    1394. Momentum based GD
      1395. +
    1395. More on momentum based approaches
      1396. +
    1396. Momentum parameter
      1397. +
    1397. Second moment of the gradient
      1398. +
    1398. RMS prop
      1399. +
    1399. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1400. +
    1400. Algorithms and codes for Adagrad, RMSprop and Adam
      1401. +
    1401. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1402. +
    1402. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1403. +
    1403. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1404. +
    1404. Practical tips
      1405. +
    1405. Sneaking in auotmatic differentiation using Autograd
      1406. +
    1406. Same code but now with momentum gradient descent
      1407. +
    1407. Including Stochastic Gradient Descent with Autograd
      1408. +
    1408. Same code but now with momentum gradient descent
      1409. +
    1409. Similar (second order function now) problem but now with AdaGrad
      1410. +
    1410. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1411. +
    1411. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1412. +
    1412. Material for the lab sessions
      1413. @@ -258,28 +277,31 @@ MathJax.Hub.Config({

       

       

       

      -

      SGD example

      -

      As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \) -and we choose to have \( M=5 \) minibathces, -then each minibatch contains two data points. In particular we have -\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 = -(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you -have only a single batch with all data points and on the other extreme, -you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e -\( B_k = \mathbf{x}_k \). +

      Stochastic Gradient Descent (SGD)

      + +

      In stochastic gradient descent, the extreme case is the case where we +have only one batch, that is we include the whole data set.

      -

      The idea is now to approximate the gradient by replacing the sum over -all data points with a sum over the data points in one the minibatches -picked at random in each gradient descent step +

      This process is called Stochastic Gradient +Descent (SGD) (or also sometimes on-line gradient descent). This is +relatively less common to see because in practice due to vectorized +code optimizations it can be computationally much more efficient to +evaluate the gradient for 100 examples, than the gradient for one +example 100 times. Even though SGD technically refers to using a +single example at a time to evaluate the gradient, you will hear +people use the term SGD even when referring to mini-batch gradient +descent (i.e. mentions of MGD for “Minibatch Gradient Descent”, or BGD +for “Batch gradient descent” are rare to see), where it is usually +assumed that mini-batches are used. The size of the mini-batch is a +hyperparameter but it is not very common to cross-validate or bootstrap it. It is +usually based on memory constraints (if any), or set to some value, +e.g. 32, 64 or 128. We use powers of 2 in practice because many +vectorized operation implementations work faster when their inputs are +sized in powers of 2.

      -$$ -\nabla_{\beta} -C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i, -\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta -c_i(\mathbf{x}_i, \mathbf{\beta}). -$$ +

      In our notes with SGD we mean stochastic gradient descent with mini-batches.

      @@ -306,7 +328,7 @@ $$

    1413. 30
    1414. 31
    1415. ...
      1416. -
    1416. 46
      1417. +
    1417. 48
    1418. »
      1419. diff --git a/doc/pub/week37/html/._week37-bs022.html b/doc/pub/week37/html/._week37-bs022.html index 2921b04c2..e0f11d8cd 100644 --- a/doc/pub/week37/html/._week37-bs022.html +++ b/doc/pub/week37/html/._week37-bs022.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1419. Program example for gradient descent with Ridge Regression
    1420. Using gradient descent methods, limitations
    1421. Improving gradient descent with momentum
      1422. -
    1422. Same code but now with momentum gradient descent
      1423. -
    1423. Overview video on Stochastic Gradient Descent
      1424. +
    1424. Same code but now with momentum gradient descent
      1425. +
    1425. Overview video on Stochastic Gradient Descent (SGD)
    1426. Batches and mini-batches
      1427. -
    1427. Stochastic Gradient Descent (SGD)
      1428. -
    1428. Stochastic Gradient Descent
      1429. -
    1429. Computation of gradients
      1430. -
    1430. SGD example
      1431. -
    1431. The gradient step
      1432. -
    1432. Simple example code
      1433. -
    1433. When do we stop?
      1434. -
    1434. Slightly different approach
      1435. -
    1435. Time decay rate
      1436. -
    1436. Code with a Number of Minibatches which varies
      1437. -
    1437. Replace or not
      1438. -
    1438. Momentum based GD
      1439. -
    1439. More on momentum based approaches
      1440. -
    1440. Momentum parameter
      1441. -
    1441. Second moment of the gradient
      1442. -
    1442. RMS prop
      1443. -
    1443. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1444. -
    1444. Algorithms and codes for Adagrad, RMSprop and Adam
      1445. -
    1445. Practical tips
      1446. -
    1446. Sneaking in automatic differentiation using Autograd
      1447. -
    1447. Same code but now with momentum gradient descent
      1448. -
    1448. But none of these can compete with Newton's method
      1449. -
    1449. Including Stochastic Gradient Descent with Autograd
      1450. -
    1450. Same code but now with momentum gradient descent
      1451. -
    1451. Similar (second order function now) problem but now with AdaGrad
      1452. -
    1452. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1453. -
    1453. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1454. -
    1454. Material for the lab sessions
      1455. +
    1455. Pros and cons
      1456. +
    1456. Convergence rates
      1457. +
    1457. Accuracy
      1458. +
    1458. Stochastic Gradient Descent (SGD)
      1459. +
    1459. Stochastic Gradient Descent
      1460. +
    1460. Computation of gradients
      1461. +
    1461. SGD example
      1462. +
    1462. The gradient step
      1463. +
    1463. Simple example code
      1464. +
    1464. When do we stop?
      1465. +
    1465. Slightly different approach
      1466. +
    1466. Time decay rate
      1467. +
    1467. Code with a Number of Minibatches which varies
      1468. +
    1468. Replace or not
      1469. +
    1469. Momentum based GD
      1470. +
    1470. More on momentum based approaches
      1471. +
    1471. Momentum parameter
      1472. +
    1472. Second moment of the gradient
      1473. +
    1473. RMS prop
      1474. +
    1474. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1475. +
    1475. Algorithms and codes for Adagrad, RMSprop and Adam
      1476. +
    1476. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1477. +
    1477. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1478. +
    1478. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1479. +
    1479. Practical tips
      1480. +
    1480. Sneaking in auotmatic differentiation using Autograd
      1481. +
    1481. Same code but now with momentum gradient descent
      1482. +
    1482. Including Stochastic Gradient Descent with Autograd
      1483. +
    1483. Same code but now with momentum gradient descent
      1484. +
    1484. Similar (second order function now) problem but now with AdaGrad
      1485. +
    1485. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1486. +
    1486. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1487. +
    1487. Material for the lab sessions
      1488. @@ -258,21 +277,22 @@ MathJax.Hub.Config({

       

       

       

      -

      The gradient step

      +

      Stochastic Gradient Descent

      -

      Thus a gradient descent step now looks like

      -$$ -\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i, -\mathbf{\beta}) -$$ - -

      where \( k \) is picked at random with equal -probability from \( [1,n/M] \). An iteration over the number of -minibathces (n/M) is commonly referred to as an epoch. Thus it is -typical to choose a number of epochs and for each epoch iterate over -the number of minibatches, as exemplified in the code below. +

      Stochastic gradient descent (SGD) and variants thereof address some of +the shortcomings of the Gradient descent method discussed above.

      +

      The underlying idea of SGD comes from the observation that the cost +function, which we want to minimize, can almost always be written as a +sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \), +

      +$$ +C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i, +\mathbf{\beta}). +$$ + +

        @@ -298,7 +318,7 @@ the number of minibatches, as exemplified in the code below.
      • 31
      • 32
      • ...
        • -
      • 46
        • +
      • 48
      • »
      diff --git a/doc/pub/week37/html/._week37-bs023.html b/doc/pub/week37/html/._week37-bs023.html index c65574214..9bbefc25d 100644 --- a/doc/pub/week37/html/._week37-bs023.html +++ b/doc/pub/week37/html/._week37-bs023.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1488. Program example for gradient descent with Ridge Regression
    1489. Using gradient descent methods, limitations
    1490. Improving gradient descent with momentum
      1491. -
    1491. Same code but now with momentum gradient descent
      1492. -
    1492. Overview video on Stochastic Gradient Descent
      1493. +
    1493. Same code but now with momentum gradient descent
      1494. +
    1494. Overview video on Stochastic Gradient Descent (SGD)
    1495. Batches and mini-batches
      1496. -
    1496. Stochastic Gradient Descent (SGD)
      1497. -
    1497. Stochastic Gradient Descent
      1498. -
    1498. Computation of gradients
      1499. -
    1499. SGD example
      1500. -
    1500. The gradient step
      1501. -
    1501. Simple example code
      1502. -
    1502. When do we stop?
      1503. -
    1503. Slightly different approach
      1504. -
    1504. Time decay rate
      1505. -
    1505. Code with a Number of Minibatches which varies
      1506. -
    1506. Replace or not
      1507. -
    1507. Momentum based GD
      1508. -
    1508. More on momentum based approaches
      1509. -
    1509. Momentum parameter
      1510. -
    1510. Second moment of the gradient
      1511. -
    1511. RMS prop
      1512. -
    1512. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1513. -
    1513. Algorithms and codes for Adagrad, RMSprop and Adam
      1514. -
    1514. Practical tips
      1515. -
    1515. Sneaking in automatic differentiation using Autograd
      1516. -
    1516. Same code but now with momentum gradient descent
      1517. -
    1517. But none of these can compete with Newton's method
      1518. -
    1518. Including Stochastic Gradient Descent with Autograd
      1519. -
    1519. Same code but now with momentum gradient descent
      1520. -
    1520. Similar (second order function now) problem but now with AdaGrad
      1521. -
    1521. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1522. -
    1522. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1523. -
    1523. Material for the lab sessions
      1524. +
    1524. Pros and cons
      1525. +
    1525. Convergence rates
      1526. +
    1526. Accuracy
      1527. +
    1527. Stochastic Gradient Descent (SGD)
      1528. +
    1528. Stochastic Gradient Descent
      1529. +
    1529. Computation of gradients
      1530. +
    1530. SGD example
      1531. +
    1531. The gradient step
      1532. +
    1532. Simple example code
      1533. +
    1533. When do we stop?
      1534. +
    1534. Slightly different approach
      1535. +
    1535. Time decay rate
      1536. +
    1536. Code with a Number of Minibatches which varies
      1537. +
    1537. Replace or not
      1538. +
    1538. Momentum based GD
      1539. +
    1539. More on momentum based approaches
      1540. +
    1540. Momentum parameter
      1541. +
    1541. Second moment of the gradient
      1542. +
    1542. RMS prop
      1543. +
    1543. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1544. +
    1544. Algorithms and codes for Adagrad, RMSprop and Adam
      1545. +
    1545. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1546. +
    1546. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1547. +
    1547. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1548. +
    1548. Practical tips
      1549. +
    1549. Sneaking in auotmatic differentiation using Autograd
      1550. +
    1550. Same code but now with momentum gradient descent
      1551. +
    1551. Including Stochastic Gradient Descent with Autograd
      1552. +
    1552. Same code but now with momentum gradient descent
      1553. +
    1553. Similar (second order function now) problem but now with AdaGrad
      1554. +
    1554. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1555. +
    1555. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1556. +
    1556. Material for the lab sessions
      1557. @@ -258,51 +277,21 @@ MathJax.Hub.Config({

       

       

       

      -

      Simple example code

      +

      Computation of gradients

      +

      This in turn means that the gradient can be +computed as a sum over \( i \)-gradients +

      +$$ +\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}). +$$ - -
      -
      -
      -
      -
      - 
      import numpy as np 
-
-n = 100 #100 datapoints 
-M = 5   #size of each minibatch
-m = int(n/M) #number of minibatches
-n_epochs = 10 #number of epochs
-
-j = 0
-for epoch in range(1,n_epochs+1):
-    for i in range(m):
-        k = np.random.randint(m) #Pick the k-th minibatch at random
-        #Compute the gradient using the data in minibatch Bk
-        #Compute new suggestion for 
-        j += 1
-
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -

      Taking the gradient only on a subset of the data has two important -benefits. First, it introduces randomness which decreases the chance -that our opmization scheme gets stuck in a local minima. Second, if -the size of the minibatches are small relative to the number of -datapoints (\( M < n \)), the computation of the gradient is much -cheaper since we sum over the datapoints in the \( k-th \) minibatch and not -all \( n \) datapoints. +

      Stochasticity/randomness is introduced by only taking the +gradient on a subset of the data called minibatches. If there are \( n \) +data points and the size of each minibatch is \( M \), there will be \( n/M \) +minibatches. We denote these minibatches by \( B_k \) where +\( k=1,\cdots,n/M \).

      @@ -330,7 +319,7 @@ all \( n \) datapoints.

    1557. 32
    1558. 33
    1559. ...
      1560. -
    1560. 46
      1561. +
    1561. 48
    1562. »
      1563. diff --git a/doc/pub/week37/html/._week37-bs024.html b/doc/pub/week37/html/._week37-bs024.html index 90d0eed4e..8a2f022de 100644 --- a/doc/pub/week37/html/._week37-bs024.html +++ b/doc/pub/week37/html/._week37-bs024.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1563. Program example for gradient descent with Ridge Regression
    1564. Using gradient descent methods, limitations
    1565. Improving gradient descent with momentum
      1566. -
    1566. Same code but now with momentum gradient descent
      1567. -
    1567. Overview video on Stochastic Gradient Descent
      1568. +
    1568. Same code but now with momentum gradient descent
      1569. +
    1569. Overview video on Stochastic Gradient Descent (SGD)
    1570. Batches and mini-batches
      1571. -
    1571. Stochastic Gradient Descent (SGD)
      1572. -
    1572. Stochastic Gradient Descent
      1573. -
    1573. Computation of gradients
      1574. -
    1574. SGD example
      1575. -
    1575. The gradient step
      1576. -
    1576. Simple example code
      1577. -
    1577. When do we stop?
      1578. -
    1578. Slightly different approach
      1579. -
    1579. Time decay rate
      1580. -
    1580. Code with a Number of Minibatches which varies
      1581. -
    1581. Replace or not
      1582. -
    1582. Momentum based GD
      1583. -
    1583. More on momentum based approaches
      1584. -
    1584. Momentum parameter
      1585. -
    1585. Second moment of the gradient
      1586. -
    1586. RMS prop
      1587. -
    1587. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1588. -
    1588. Algorithms and codes for Adagrad, RMSprop and Adam
      1589. -
    1589. Practical tips
      1590. -
    1590. Sneaking in automatic differentiation using Autograd
      1591. -
    1591. Same code but now with momentum gradient descent
      1592. -
    1592. But none of these can compete with Newton's method
      1593. -
    1593. Including Stochastic Gradient Descent with Autograd
      1594. -
    1594. Same code but now with momentum gradient descent
      1595. -
    1595. Similar (second order function now) problem but now with AdaGrad
      1596. -
    1596. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1597. -
    1597. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1598. -
    1598. Material for the lab sessions
      1599. +
    1599. Pros and cons
      1600. +
    1600. Convergence rates
      1601. +
    1601. Accuracy
      1602. +
    1602. Stochastic Gradient Descent (SGD)
      1603. +
    1603. Stochastic Gradient Descent
      1604. +
    1604. Computation of gradients
      1605. +
    1605. SGD example
      1606. +
    1606. The gradient step
      1607. +
    1607. Simple example code
      1608. +
    1608. When do we stop?
      1609. +
    1609. Slightly different approach
      1610. +
    1610. Time decay rate
      1611. +
    1611. Code with a Number of Minibatches which varies
      1612. +
    1612. Replace or not
      1613. +
    1613. Momentum based GD
      1614. +
    1614. More on momentum based approaches
      1615. +
    1615. Momentum parameter
      1616. +
    1616. Second moment of the gradient
      1617. +
    1617. RMS prop
      1618. +
    1618. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1619. +
    1619. Algorithms and codes for Adagrad, RMSprop and Adam
      1620. +
    1620. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1621. +
    1621. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1622. +
    1622. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1623. +
    1623. Practical tips
      1624. +
    1624. Sneaking in auotmatic differentiation using Autograd
      1625. +
    1625. Same code but now with momentum gradient descent
      1626. +
    1626. Including Stochastic Gradient Descent with Autograd
      1627. +
    1627. Same code but now with momentum gradient descent
      1628. +
    1628. Similar (second order function now) problem but now with AdaGrad
      1629. +
    1629. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1630. +
    1630. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1631. +
    1631. Material for the lab sessions
      1632. @@ -258,20 +277,29 @@ MathJax.Hub.Config({

       

       

       

      -

      When do we stop?

      - -

      A natural question is when do we stop the search for a new minimum? -One possibility is to compute the full gradient after a given number -of epochs and check if the norm of the gradient is smaller than some -threshold and stop if true. However, the condition that the gradient -is zero is valid also for local minima, so this would only tell us -that we are close to a local/global minimum. However, we could also -evaluate the cost function at this point, store the result and -continue the search. If the test kicks in at a later stage we can -compare the values of the cost function and keep the \( \beta \) that -gave the lowest value. +

      SGD example

      +

      As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \) +and we choose to have \( M=5 \) minibathces, +then each minibatch contains two data points. In particular we have +\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 = +(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you +have only a single batch with all data points and on the other extreme, +you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e +\( B_k = \mathbf{x}_k \).

      +

      The idea is now to approximate the gradient by replacing the sum over +all data points with a sum over the data points in one the minibatches +picked at random in each gradient descent step +

      +$$ +\nabla_{\beta} +C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta +c_i(\mathbf{x}_i, \mathbf{\beta}). +$$ + +

        @@ -297,7 +325,7 @@ gave the lowest value.
      • 33
      • 34
      • ...
        • -
      • 46
        • +
      • 48
      • »
      diff --git a/doc/pub/week37/html/._week37-bs025.html b/doc/pub/week37/html/._week37-bs025.html index f248bb5e5..3d2c765b5 100644 --- a/doc/pub/week37/html/._week37-bs025.html +++ b/doc/pub/week37/html/._week37-bs025.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1632. Program example for gradient descent with Ridge Regression
    1633. Using gradient descent methods, limitations
    1634. Improving gradient descent with momentum
      1635. -
    1635. Same code but now with momentum gradient descent
      1636. -
    1636. Overview video on Stochastic Gradient Descent
      1637. +
    1637. Same code but now with momentum gradient descent
      1638. +
    1638. Overview video on Stochastic Gradient Descent (SGD)
    1639. Batches and mini-batches
      1640. -
    1640. Stochastic Gradient Descent (SGD)
      1641. -
    1641. Stochastic Gradient Descent
      1642. -
    1642. Computation of gradients
      1643. -
    1643. SGD example
      1644. -
    1644. The gradient step
      1645. -
    1645. Simple example code
      1646. -
    1646. When do we stop?
      1647. -
    1647. Slightly different approach
      1648. -
    1648. Time decay rate
      1649. -
    1649. Code with a Number of Minibatches which varies
      1650. -
    1650. Replace or not
      1651. -
    1651. Momentum based GD
      1652. -
    1652. More on momentum based approaches
      1653. -
    1653. Momentum parameter
      1654. -
    1654. Second moment of the gradient
      1655. -
    1655. RMS prop
      1656. -
    1656. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1657. -
    1657. Algorithms and codes for Adagrad, RMSprop and Adam
      1658. -
    1658. Practical tips
      1659. -
    1659. Sneaking in automatic differentiation using Autograd
      1660. -
    1660. Same code but now with momentum gradient descent
      1661. -
    1661. But none of these can compete with Newton's method
      1662. -
    1662. Including Stochastic Gradient Descent with Autograd
      1663. -
    1663. Same code but now with momentum gradient descent
      1664. -
    1664. Similar (second order function now) problem but now with AdaGrad
      1665. -
    1665. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1666. -
    1666. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1667. -
    1667. Material for the lab sessions
      1668. +
    1668. Pros and cons
      1669. +
    1669. Convergence rates
      1670. +
    1670. Accuracy
      1671. +
    1671. Stochastic Gradient Descent (SGD)
      1672. +
    1672. Stochastic Gradient Descent
      1673. +
    1673. Computation of gradients
      1674. +
    1674. SGD example
      1675. +
    1675. The gradient step
      1676. +
    1676. Simple example code
      1677. +
    1677. When do we stop?
      1678. +
    1678. Slightly different approach
      1679. +
    1679. Time decay rate
      1680. +
    1680. Code with a Number of Minibatches which varies
      1681. +
    1681. Replace or not
      1682. +
    1682. Momentum based GD
      1683. +
    1683. More on momentum based approaches
      1684. +
    1684. Momentum parameter
      1685. +
    1685. Second moment of the gradient
      1686. +
    1686. RMS prop
      1687. +
    1687. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1688. +
    1688. Algorithms and codes for Adagrad, RMSprop and Adam
      1689. +
    1689. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1690. +
    1690. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1691. +
    1691. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1692. +
    1692. Practical tips
      1693. +
    1693. Sneaking in auotmatic differentiation using Autograd
      1694. +
    1694. Same code but now with momentum gradient descent
      1695. +
    1695. Including Stochastic Gradient Descent with Autograd
      1696. +
    1696. Same code but now with momentum gradient descent
      1697. +
    1697. Similar (second order function now) problem but now with AdaGrad
      1698. +
    1698. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1699. +
    1699. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1700. +
    1700. Material for the lab sessions
      1701. @@ -258,17 +277,19 @@ MathJax.Hub.Config({

       

       

       

      -

      Slightly different approach

      +

      The gradient step

      -

      Another approach is to let the step length \( \gamma_j \) depend on the -number of epochs in such a way that it becomes very small after a -reasonable time such that we do not move at all. Such approaches are -also called scaling. There are many such ways to scale the learning -rate -and discussions here. See -also -https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1 -for a discussion of different scaling functions for the learning rate. +

      Thus a gradient descent step now looks like

      +$$ +\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}) +$$ + +

      where \( k \) is picked at random with equal +probability from \( [1,n/M] \). An iteration over the number of +minibathces (n/M) is commonly referred to as an epoch. Thus it is +typical to choose a number of epochs and for each epoch iterate over +the number of minibatches, as exemplified in the code below.

      @@ -296,7 +317,7 @@ for a discussion of different scaling functions for the learning rate.

    1701. 34
    1702. 35
    1703. ...
      1704. -
    1704. 46
      1705. +
    1705. 48
    1706. »
      1707. diff --git a/doc/pub/week37/html/._week37-bs026.html b/doc/pub/week37/html/._week37-bs026.html index af0e823ac..b7f2f789a 100644 --- a/doc/pub/week37/html/._week37-bs026.html +++ b/doc/pub/week37/html/._week37-bs026.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
    1707. Program example for gradient descent with Ridge Regression
    1708. Using gradient descent methods, limitations
    1709. Improving gradient descent with momentum
      1710. -
    1710. Same code but now with momentum gradient descent
      1711. -
    1711. Overview video on Stochastic Gradient Descent
      1712. +
    1712. Same code but now with momentum gradient descent
      1713. +
    1713. Overview video on Stochastic Gradient Descent (SGD)
    1714. Batches and mini-batches
      1715. -
    1715. Stochastic Gradient Descent (SGD)
      1716. -
    1716. Stochastic Gradient Descent
      1717. -
    1717. Computation of gradients
      1718. -
    1718. SGD example
      1719. -
    1719. The gradient step
      1720. -
    1720. Simple example code
      1721. -
    1721. When do we stop?
      1722. -
    1722. Slightly different approach
      1723. -
    1723. Time decay rate
      1724. -
    1724. Code with a Number of Minibatches which varies
      1725. -
    1725. Replace or not
      1726. -
    1726. Momentum based GD
      1727. -
    1727. More on momentum based approaches
      1728. -
    1728. Momentum parameter
      1729. -
    1729. Second moment of the gradient
      1730. -
    1730. RMS prop
      1731. -
    1731. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1732. -
    1732. Algorithms and codes for Adagrad, RMSprop and Adam
      1733. -
    1733. Practical tips
      1734. -
    1734. Sneaking in automatic differentiation using Autograd
      1735. -
    1735. Same code but now with momentum gradient descent
      1736. -
    1736. But none of these can compete with Newton's method
      1737. -
    1737. Including Stochastic Gradient Descent with Autograd
      1738. -
    1738. Same code but now with momentum gradient descent
      1739. -
    1739. Similar (second order function now) problem but now with AdaGrad
      1740. -
    1740. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1741. -
    1741. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1742. -
    1742. Material for the lab sessions
      1743. +
    1743. Pros and cons
      1744. +
    1744. Convergence rates
      1745. +
    1745. Accuracy
      1746. +
    1746. Stochastic Gradient Descent (SGD)
      1747. +
    1747. Stochastic Gradient Descent
      1748. +
    1748. Computation of gradients
      1749. +
    1749. SGD example
      1750. +
    1750. The gradient step
      1751. +
    1751. Simple example code
      1752. +
    1752. When do we stop?
      1753. +
    1753. Slightly different approach
      1754. +
    1754. Time decay rate
      1755. +
    1755. Code with a Number of Minibatches which varies
      1756. +
    1756. Replace or not
      1757. +
    1757. Momentum based GD
      1758. +
    1758. More on momentum based approaches
      1759. +
    1759. Momentum parameter
      1760. +
    1760. Second moment of the gradient
      1761. +
    1761. RMS prop
      1762. +
    1762. "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
      1763. +
    1763. Algorithms and codes for Adagrad, RMSprop and Adam
      1764. +
    1764. AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1765. +
    1765. RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1766. +
    1766. ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
      1767. +
    1767. Practical tips
      1768. +
    1768. Sneaking in auotmatic differentiation using Autograd
      1769. +
    1769. Same code but now with momentum gradient descent
      1770. +
    1770. Including Stochastic Gradient Descent with Autograd
      1771. +
    1771. Same code but now with momentum gradient descent
      1772. +
    1772. Similar (second order function now) problem but now with AdaGrad
      1773. +
    1773. RMSprop for adaptive learning rate with Stochastic Gradient Descent
      1774. +
    1774. And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
      1775. +
    1775. Material for the lab sessions
      1776. @@ -258,16 +277,7 @@ MathJax.Hub.Config({

       

       

       

      -

      Time decay rate

      - -

      As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in time \( t \).

      - -

      In this way we can fix the number of epochs, compute \( \beta \) and -evaluate the cost function at the end. Repeating the computation will -give a different result since the scheme is random by design. Then we -pick the final \( \beta \) that gives the lowest value of the cost -function. -

      +

      Simple example code

      @@ -278,28 +288,18 @@ function. 
      import numpy as np 
 
-def step_length(t,t0,t1):
-    return t0/(t+t1)
-
 n = 100 #100 datapoints 
 M = 5   #size of each minibatch
 m = int(n/M) #number of minibatches
-n_epochs = 500 #number of epochs
-t0 = 1.0
-t1 = 10
+n_epochs = 10 #number of epochs
 
-gamma_j = t0/t1
 j = 0
 for epoch in range(1,n_epochs+1):
     for i in range(m):
         k = np.random.randint(m) #Pick the k-th minibatch at random
         #Compute the gradient using the data in minibatch Bk
-        #Compute new suggestion for beta
-        t = epoch*m+i
-        gamma_j = step_length(t,t0,t1)
+        #Compute new suggestion for 
         j += 1
-
-print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))
    @@ -315,6 +315,14 @@ j = 0 +

    Taking the gradient only on a subset of the data has two important +benefits. First, it introduces randomness which decreases the chance +that our opmization scheme gets stuck in a local minima. Second, if +the size of the minibatches are small relative to the number of +datapoints (\( M < n \)), the computation of the gradient is much +cheaper since we sum over the datapoints in the \( k-th \) minibatch and not +all \( n \) datapoints. +

    @@ -341,7 +349,7 @@ j = 0

  • 35
  • 36
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs027.html b/doc/pub/week37/html/._week37-bs027.html index e59a6392a..9a1f03ee5 100644 --- a/doc/pub/week37/html/._week37-bs027.html +++ b/doc/pub/week37/html/._week37-bs027.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,97 +277,19 @@ MathJax.Hub.Config({

     

     

     

    -

    Code with a Number of Minibatches which varies

    - -

    In the code here we vary the number of mini-batches.

    - - -
    -
    -
    -
    -
    - 
    # Importing various packages
-from math import exp, sqrt
-from random import random, seed
-import numpy as np
-import matplotlib.pyplot as plt
-
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
-
-X = np.c_[np.ones((n,1)), x]
-XT_X = X.T @ X
-theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
-print("Own inversion")
-print(theta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
-
-theta = np.random.randn(2,1)
-eta = 1.0/np.max(EigValues)
-Niterations = 1000
-
-
-for iter in range(Niterations):
-    gradients = 2.0/n*X.T @ ((X @ theta)-y)
-    theta -= eta*gradients
-print("theta from own gd")
-print(theta)
-
-xnew = np.array([[0],[2]])
-Xnew = np.c_[np.ones((2,1)), xnew]
-ypredict = Xnew.dot(theta)
-ypredict2 = Xnew.dot(theta_linreg)
-
-n_epochs = 50
-M = 5   #size of each minibatch
-m = int(n/M) #number of minibatches
-t0, t1 = 5, 50
-
-def learning_schedule(t):
-    return t0/(t+t1)
-
-theta = np.random.randn(2,1)
-
-for epoch in range(n_epochs):
-# Can you figure out a better way of setting up the contributions to each batch?
-    for i in range(m):
-        random_index = M*np.random.randint(m)
-        xi = X[random_index:random_index+M]
-        yi = y[random_index:random_index+M]
-        gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)
-        eta = learning_schedule(epoch*m+i)
-        theta = theta - eta*gradients
-print("theta from own sdg")
-print(theta)
-
-plt.plot(xnew, ypredict, "r-")
-plt.plot(xnew, ypredict2, "b-")
-plt.plot(x, y ,'ro')
-plt.axis([0,2.0,0, 15.0])
-plt.xlabel(r'$x$')
-plt.ylabel(r'$y$')
-plt.title(r'Random numbers ')
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    When do we stop?

    +

    A natural question is when do we stop the search for a new minimum? +One possibility is to compute the full gradient after a given number +of epochs and check if the norm of the gradient is smaller than some +threshold and stop if true. However, the condition that the gradient +is zero is valid also for local minima, so this would only tell us +that we are close to a local/global minimum. However, we could also +evaluate the cost function at this point, store the result and +continue the search. If the test kicks in at a later stage we can +compare the values of the cost function and keep the \( \beta \) that +gave the lowest value. +

    @@ -375,7 +316,7 @@ plt.show()

  • 36
  • 37
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs028.html b/doc/pub/week37/html/._week37-bs028.html index abcec5972..e9e33bc93 100644 --- a/doc/pub/week37/html/._week37-bs028.html +++ b/doc/pub/week37/html/._week37-bs028.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,12 +277,17 @@ MathJax.Hub.Config({

     

     

     

    -

    Replace or not

    +

    Slightly different approach

    -

    In the above code, we have use replacement in setting up the -mini-batches. The discussion -here may be -useful. +

    Another approach is to let the step length \( \gamma_j \) depend on the +number of epochs in such a way that it becomes very small after a +reasonable time such that we do not move at all. Such approaches are +also called scaling. There are many such ways to scale the learning +rate +and discussions here. See +also +https://towardsdatascience.com/learning-rate-schedules-and-adaptive-learning-rate-methods-for-deep-learning-2c8f433990d1 +for a discussion of different scaling functions for the learning rate.

    @@ -291,7 +315,7 @@ useful.

  • 37
  • 38
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs029.html b/doc/pub/week37/html/._week37-bs029.html index e085e692b..202ea0fcc 100644 --- a/doc/pub/week37/html/._week37-bs029.html +++ b/doc/pub/week37/html/._week37-bs029.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,39 +277,63 @@ MathJax.Hub.Config({

     

     

     

    -

    Momentum based GD

    +

    Time decay rate

    -

    The stochastic gradient descent (SGD) is almost always used with a -momentum or inertia term that serves as a memory of the direction we -are moving in parameter space. This is typically implemented as -follows +

    As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in time \( t \).

    + +

    In this way we can fix the number of epochs, compute \( \beta \) and +evaluate the cost function at the end. Repeating the computation will +give a different result since the scheme is random by design. Then we +pick the final \( \beta \) that gives the lowest value of the cost +function.

    -$$ -\begin{align} -\mathbf{v}_{t}&=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t) \nonumber \\ -\boldsymbol{\theta}_{t+1}&= \boldsymbol{\theta}_t -\mathbf{v}_{t}, -\tag{1} -\end{align} -$$ -

    where we have introduced a momentum parameter \( \gamma \), with -\( 0\le\gamma\le 1 \), and for brevity we dropped the explicit notation to -indicate the gradient is to be taken over a different mini-batch at -each step. We call this algorithm gradient descent with momentum -(GDM). From these equations, it is clear that \( \mathbf{v}_t \) is a -running average of recently encountered gradients and -\( (1-\gamma)^{-1} \) sets the characteristic time scale for the memory -used in the averaging procedure. Consistent with this, when -\( \gamma=0 \), this just reduces down to ordinary SGD as discussed -earlier. An equivalent way of writing the updates is -

    + +
    +
    +
    +
    +
    + 
    import numpy as np 
 
-$$
-\Delta \boldsymbol{\theta}_{t+1} = \gamma \Delta \boldsymbol{\theta}_t -\ \eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t),
-$$
+def step_length(t,t0,t1):
+    return t0/(t+t1)
+
+n = 100 #100 datapoints 
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+n_epochs = 500 #number of epochs
+t0 = 1.0
+t1 = 10
+
+gamma_j = t0/t1
+j = 0
+for epoch in range(1,n_epochs+1):
+    for i in range(m):
+        k = np.random.randint(m) #Pick the k-th minibatch at random
+        #Compute the gradient using the data in minibatch Bk
+        #Compute new suggestion for beta
+        t = epoch*m+i
+        gamma_j = step_length(t,t0,t1)
+        j += 1
+
+print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    where we have defined \( \Delta \boldsymbol{\theta}_{t}= \boldsymbol{\theta}_t-\boldsymbol{\theta}_{t-1} \).

    @@ -317,7 +360,7 @@ $$

  • 38
  • 39
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs030.html b/doc/pub/week37/html/._week37-bs030.html index e149be2ff..4d810a1d2 100644 --- a/doc/pub/week37/html/._week37-bs030.html +++ b/doc/pub/week37/html/._week37-bs030.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,30 +277,96 @@ MathJax.Hub.Config({

     

     

     

    -

    More on momentum based approaches

    +

    Code with a Number of Minibatches which varies

    -

    Let us try to get more intuition from these equations. It is helpful -to consider a simple physical analogy with a particle of mass \( m \) -moving in a viscous medium with drag coefficient \( \mu \) and potential -\( E(\mathbf{w}) \). If we denote the particle's position by \( \mathbf{w} \), -then its motion is described by -

    +

    In the code here we vary the number of mini-batches.

    -$$ -m {d^2 \mathbf{w} \over dt^2} + \mu {d \mathbf{w} \over dt }= -\nabla_w E(\mathbf{w}). -$$ + +
    +
    +
    +
    +
    + 
    # Importing various packages
+from math import exp, sqrt
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
 
-
    We can discretize this equation in the usual way to get
    
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
 
-$$
-m { \mathbf{w}_{t+\Delta t}-2 \mathbf{w}_{t} +\mathbf{w}_{t-\Delta t} \over (\Delta t)^2}+\mu {\mathbf{w}_{t+\Delta t}- \mathbf{w}_{t} \over \Delta t} = -\nabla_w E(\mathbf{w}).
-$$
+X = np.c_[np.ones((n,1)), x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
 
-
    Rearranging this equation, we can rewrite this as
    
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 1000
 
-$$
-\Delta \mathbf{w}_{t +\Delta t}= - { (\Delta t)^2 \over m +\mu \Delta t} \nabla_w E(\mathbf{w})+ {m \over m +\mu \Delta t} \Delta \mathbf{w}_t.
-$$
+
+for iter in range(Niterations):
+    gradients = 2.0/n*X.T @ ((X @ theta)-y)
+    theta -= eta*gradients
+print("theta from own gd")
+print(theta)
+
+xnew = np.array([[0],[2]])
+Xnew = np.c_[np.ones((2,1)), xnew]
+ypredict = Xnew.dot(theta)
+ypredict2 = Xnew.dot(theta_linreg)
+
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+t0, t1 = 5, 50
+
+def learning_schedule(t):
+    return t0/(t+t1)
+
+theta = np.random.randn(2,1)
+
+for epoch in range(n_epochs):
+# Can you figure out a better way of setting up the contributions to each batch?
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (2.0/M)* xi.T @ ((xi @ theta)-yi)
+        eta = learning_schedule(epoch*m+i)
+        theta = theta - eta*gradients
+print("theta from own sdg")
+print(theta)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(xnew, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    @@ -309,7 +394,7 @@ $$

  • 39
  • 40
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs031.html b/doc/pub/week37/html/._week37-bs031.html index 1463b079c..58b507ddc 100644 --- a/doc/pub/week37/html/._week37-bs031.html +++ b/doc/pub/week37/html/._week37-bs031.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,58 +277,14 @@ MathJax.Hub.Config({

     

     

     

    -

    Momentum parameter

    +

    Replace or not

    -

    Notice that this equation is identical to previous one if we identify -the position of the particle, \( \mathbf{w} \), with the parameters -\( \boldsymbol{\theta} \). This allows us to identify the momentum -parameter and learning rate with the mass of the particle and the -viscous drag as: +

    In the above code, we have use replacement in setting up the +mini-batches. The discussion +here may be +useful.

    -$$ -\gamma= {m \over m +\mu \Delta t }, \qquad \eta = {(\Delta t)^2 \over m +\mu \Delta t}. -$$ - -

    Thus, as the name suggests, the momentum parameter is proportional to -the mass of the particle and effectively provides inertia. -Furthermore, in the large viscosity/small learning rate limit, our -memory time scales as \( (1-\gamma)^{-1} \approx m/(\mu \Delta t) \). -

    - -

    Why is momentum useful? SGD momentum helps the gradient descent -algorithm gain speed in directions with persistent but small gradients -even in the presence of stochasticity, while suppressing oscillations -in high-curvature directions. This becomes especially important in -situations where the landscape is shallow and flat in some directions -and narrow and steep in others. It has been argued that first-order -methods (with appropriate initial conditions) can perform comparable -to more expensive second order methods, especially in the context of -complex deep learning models. -

    - -

    These beneficial properties of momentum can sometimes become even more -pronounced by using a slight modification of the classical momentum -algorithm called Nesterov Accelerated Gradient (NAG). -

    - -

    In the NAG algorithm, rather than calculating the gradient at the -current parameters, \( \nabla_\theta E(\boldsymbol{\theta}_t) \), one -calculates the gradient at the expected value of the parameters given -our current momentum, \( \nabla_\theta E(\boldsymbol{\theta}_t +\gamma -\mathbf{v}_{t-1}) \). This yields the NAG update rule -

    - -$$ -\begin{align} -\mathbf{v}_{t}&=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t +\gamma \mathbf{v}_{t-1}) \nonumber \\ -\boldsymbol{\theta}_{t+1}&= \boldsymbol{\theta}_t -\mathbf{v}_{t}. -\tag{2} -\end{align} -$$ - -

    One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of \( \gamma \).

    -

    diff --git a/doc/pub/week37/html/._week37-bs032.html b/doc/pub/week37/html/._week37-bs032.html index 21fd568cf..c368a7e16 100644 --- a/doc/pub/week37/html/._week37-bs032.html +++ b/doc/pub/week37/html/._week37-bs032.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,31 +277,40 @@ MathJax.Hub.Config({

     

     

     

    -

    Second moment of the gradient

    +

    Momentum based GD

    -

    In stochastic gradient descent, with and without momentum, we still -have to specify a schedule for tuning the learning rates \( \eta_t \) -as a function of time. As discussed in the context of Newton's -method, this presents a number of dilemmas. The learning rate is -limited by the steepest direction which can change depending on the -current position in the landscape. To circumvent this problem, ideally -our algorithm would keep track of curvature and take large steps in -shallow, flat directions and small steps in steep, narrow directions. -Second-order methods accomplish this by calculating or approximating -the Hessian and normalizing the learning rate by the -curvature. However, this is very computationally expensive for -extremely large models. Ideally, we would like to be able to -adaptively change the step size to match the landscape without paying -the steep computational price of calculating or approximating -Hessians. +

    The stochastic gradient descent (SGD) is almost always used with a +momentum or inertia term that serves as a memory of the direction we +are moving in parameter space. This is typically implemented as +follows

    -

    Recently, a number of methods have been introduced that accomplish -this by tracking not only the gradient, but also the second moment of -the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and -ADAM. +$$ +\begin{align} +\mathbf{v}_{t}&=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t) \nonumber \\ +\boldsymbol{\theta}_{t+1}&= \boldsymbol{\theta}_t -\mathbf{v}_{t}, +\tag{1} +\end{align} +$$ + +

    where we have introduced a momentum parameter \( \gamma \), with +\( 0\le\gamma\le 1 \), and for brevity we dropped the explicit notation to +indicate the gradient is to be taken over a different mini-batch at +each step. We call this algorithm gradient descent with momentum +(GDM). From these equations, it is clear that \( \mathbf{v}_t \) is a +running average of recently encountered gradients and +\( (1-\gamma)^{-1} \) sets the characteristic time scale for the memory +used in the averaging procedure. Consistent with this, when +\( \gamma=0 \), this just reduces down to ordinary SGD as discussed +earlier. An equivalent way of writing the updates is

    +$$ +\Delta \boldsymbol{\theta}_{t+1} = \gamma \Delta \boldsymbol{\theta}_t -\ \eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t), +$$ + +

    where we have defined \( \Delta \boldsymbol{\theta}_{t}= \boldsymbol{\theta}_t-\boldsymbol{\theta}_{t-1} \).

    +

    diff --git a/doc/pub/week37/html/._week37-bs033.html b/doc/pub/week37/html/._week37-bs033.html index a375fe4c2..e18d330fb 100644 --- a/doc/pub/week37/html/._week37-bs033.html +++ b/doc/pub/week37/html/._week37-bs033.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,33 +277,31 @@ MathJax.Hub.Config({

     

     

     

    -

    RMS prop

    +

    More on momentum based approaches

    -

    In RMS prop, in addition to keeping a running average of the first -moment of the gradient, we also keep track of the second moment -denoted by \( \mathbf{s}_t=\mathbb{E}[\mathbf{g}_t^2] \). The update rule -for RMS prop is given by +

    Let us try to get more intuition from these equations. It is helpful +to consider a simple physical analogy with a particle of mass \( m \) +moving in a viscous medium with drag coefficient \( \mu \) and potential +\( E(\mathbf{w}) \). If we denote the particle's position by \( \mathbf{w} \), +then its motion is described by

    $$ -\begin{align} -\mathbf{g}_t &= \nabla_\theta E(\boldsymbol{\theta}) -\tag{3}\\ -\mathbf{s}_t &=\beta \mathbf{s}_{t-1} +(1-\beta)\mathbf{g}_t^2 \nonumber \\ -\boldsymbol{\theta}_{t+1}&=&\boldsymbol{\theta}_t - \eta_t { \mathbf{g}_t \over \sqrt{\mathbf{s}_t +\epsilon}}, \nonumber -\end{align} +m {d^2 \mathbf{w} \over dt^2} + \mu {d \mathbf{w} \over dt }= -\nabla_w E(\mathbf{w}). +$$ + +

    We can discretize this equation in the usual way to get

    + +$$ +m { \mathbf{w}_{t+\Delta t}-2 \mathbf{w}_{t} +\mathbf{w}_{t-\Delta t} \over (\Delta t)^2}+\mu {\mathbf{w}_{t+\Delta t}- \mathbf{w}_{t} \over \Delta t} = -\nabla_w E(\mathbf{w}). +$$ + +

    Rearranging this equation, we can rewrite this as

    + +$$ +\Delta \mathbf{w}_{t +\Delta t}= - { (\Delta t)^2 \over m +\mu \Delta t} \nabla_w E(\mathbf{w})+ {m \over m +\mu \Delta t} \Delta \mathbf{w}_t. $$ -

    where \( \beta \) controls the averaging time of the second moment and is -typically taken to be about \( \beta=0.9 \), \( \eta_t \) is a learning rate -typically chosen to be \( 10^{-3} \), and \( \epsilon\sim 10^{-8} \) is a -small regularization constant to prevent divergences. Multiplication -and division by vectors is understood as an element-wise operation. It -is clear from this formula that the learning rate is reduced in -directions where the norm of the gradient is consistently large. This -greatly speeds up the convergence by allowing us to use a larger -learning rate for flat directions. -

    @@ -311,7 +328,7 @@ learning rate for flat directions.

  • 42
  • 43
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs034.html b/doc/pub/week37/html/._week37-bs034.html index 1d4ebc309..c443a8f65 100644 --- a/doc/pub/week37/html/._week37-bs034.html +++ b/doc/pub/week37/html/._week37-bs034.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,60 +277,60 @@ MathJax.Hub.Config({

     

     

     

    -

    ADAM optimizer

    +

    Momentum parameter

    -

    A related algorithm is the ADAM optimizer. In -ADAM, we keep a running average of -both the first and second moment of the gradient and use this -information to adaptively change the learning rate for different -parameters. The method isefficient when working with large -problems involving lots data and/or parameters. It is a combination of the -gradient descent with momentum algorithm and the RMSprop algorithm -discussed above. +

    Notice that this equation is identical to previous one if we identify +the position of the particle, \( \mathbf{w} \), with the parameters +\( \boldsymbol{\theta} \). This allows us to identify the momentum +parameter and learning rate with the mass of the particle and the +viscous drag as:

    -

    In addition to keeping a running average of the first and -second moments of the gradient -(i.e. \( \mathbf{m}_t=\mathbb{E}[\mathbf{g}_t] \) and -\( \mathbf{s}_t=\mathbb{E}[\mathbf{g}^2_t] \), respectively), ADAM -performs an additional bias correction to account for the fact that we -are estimating the first two moments of the gradient using a running -average (denoted by the hats in the update rule below). The update -rule for ADAM is given by (where multiplication and division are once -again understood to be element-wise operations below) +$$ +\gamma= {m \over m +\mu \Delta t }, \qquad \eta = {(\Delta t)^2 \over m +\mu \Delta t}. +$$ + +

    Thus, as the name suggests, the momentum parameter is proportional to +the mass of the particle and effectively provides inertia. +Furthermore, in the large viscosity/small learning rate limit, our +memory time scales as \( (1-\gamma)^{-1} \approx m/(\mu \Delta t) \). +

    + +

    Why is momentum useful? SGD momentum helps the gradient descent +algorithm gain speed in directions with persistent but small gradients +even in the presence of stochasticity, while suppressing oscillations +in high-curvature directions. This becomes especially important in +situations where the landscape is shallow and flat in some directions +and narrow and steep in others. It has been argued that first-order +methods (with appropriate initial conditions) can perform comparable +to more expensive second order methods, especially in the context of +complex deep learning models. +

    + +

    These beneficial properties of momentum can sometimes become even more +pronounced by using a slight modification of the classical momentum +algorithm called Nesterov Accelerated Gradient (NAG). +

    + +

    In the NAG algorithm, rather than calculating the gradient at the +current parameters, \( \nabla_\theta E(\boldsymbol{\theta}_t) \), one +calculates the gradient at the expected value of the parameters given +our current momentum, \( \nabla_\theta E(\boldsymbol{\theta}_t +\gamma +\mathbf{v}_{t-1}) \). This yields the NAG update rule

    $$ \begin{align} -\mathbf{g}_t &= \nabla_\theta E(\boldsymbol{\theta}) -\tag{4}\\ -\mathbf{m}_t &= \beta_1 \mathbf{m}_{t-1} + (1-\beta_1) \mathbf{g}_t \nonumber \\ -\mathbf{s}_t &=\beta_2 \mathbf{s}_{t-1} +(1-\beta_2)\mathbf{g}_t^2 \nonumber \\ -\boldsymbol{\mathbf{m}}_t&={\mathbf{m}_t \over 1-\beta_1^t} \nonumber \\ -\boldsymbol{\mathbf{s}}_t &={\mathbf{s}_t \over1-\beta_2^t} \nonumber \\ -\boldsymbol{\theta}_{t+1}&=\boldsymbol{\theta}_t - \eta_t { \boldsymbol{\mathbf{m}}_t \over \sqrt{\boldsymbol{\mathbf{s}}_t} +\epsilon}, \nonumber \\ -\tag{5} +\mathbf{v}_{t}&=\gamma \mathbf{v}_{t-1}+\eta_{t}\nabla_\theta E(\boldsymbol{\theta}_t +\gamma \mathbf{v}_{t-1}) \nonumber \\ +\boldsymbol{\theta}_{t+1}&= \boldsymbol{\theta}_t -\mathbf{v}_{t}. +\tag{2} \end{align} $$ -

    where \( \beta_1 \) and \( \beta_2 \) set the memory lifetime of the first and -second moment and are typically taken to be \( 0.9 \) and \( 0.99 \) -respectively, and \( \eta \) and \( \epsilon \) are identical to RMSprop. +

    One of the major advantages of NAG is that it allows for the use of a +larger learning rate than GDM for the same choice of \( \gamma \).

    -

    Like in RMSprop, the effective step size of a parameter depends on the -magnitude of its gradient squared. To understand this better, let us -rewrite this expression in terms of the variance -\( \boldsymbol{\sigma}_t^2 = \boldsymbol{\mathbf{s}}_t - -(\boldsymbol{\mathbf{m}}_t)^2 \). Consider a single parameter \( \theta_t \). The -update rule for this parameter is given by -

    - -$$ -\Delta \theta_{t+1}= -\eta_t { \boldsymbol{m}_t \over \sqrt{\sigma_t^2 + m_t^2 }+\epsilon}. -$$ - -

    diff --git a/doc/pub/week37/html/._week37-bs035.html b/doc/pub/week37/html/._week37-bs035.html index 8d2b305ee..d835233f4 100644 --- a/doc/pub/week37/html/._week37-bs035.html +++ b/doc/pub/week37/html/._week37-bs035.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,11 +277,30 @@ MathJax.Hub.Config({

     

     

     

    -

    Algorithms and codes for Adagrad, RMSprop and Adam

    +

    Second moment of the gradient

    -

    The algorithms we have implemented are well described in the text by Goodfellow, Bengio and Courville, chapter 8.

    +

    In stochastic gradient descent, with and without momentum, we still +have to specify a schedule for tuning the learning rates \( \eta_t \) +as a function of time. As discussed in the context of Newton's +method, this presents a number of dilemmas. The learning rate is +limited by the steepest direction which can change depending on the +current position in the landscape. To circumvent this problem, ideally +our algorithm would keep track of curvature and take large steps in +shallow, flat directions and small steps in steep, narrow directions. +Second-order methods accomplish this by calculating or approximating +the Hessian and normalizing the learning rate by the +curvature. However, this is very computationally expensive for +extremely large models. Ideally, we would like to be able to +adaptively change the step size to match the landscape without paying +the steep computational price of calculating or approximating +Hessians. +

    -

    The codes which implement these algorithms are discussed below here.

    +

    During the last decade a number of methods have been introduced that accomplish +this by tracking not only the gradient, but also the second moment of +the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and +ADAM. +

    @@ -289,7 +327,7 @@ MathJax.Hub.Config({

  • 44
  • 45
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs036.html b/doc/pub/week37/html/._week37-bs036.html index e3d652f14..8f8250e24 100644 --- a/doc/pub/week37/html/._week37-bs036.html +++ b/doc/pub/week37/html/._week37-bs036.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,14 +277,34 @@ MathJax.Hub.Config({

     

     

     

    -

    Practical tips

    +

    RMS prop

    + +

    In RMS prop, in addition to keeping a running average of the first +moment of the gradient, we also keep track of the second moment +denoted by \( \mathbf{s}_t=\mathbb{E}[\mathbf{g}_t^2] \). The update rule +for RMS prop is given by +

    + +$$ +\begin{align} +\mathbf{g}_t &= \nabla_\theta E(\boldsymbol{\theta}) +\tag{3}\\ +\mathbf{s}_t &=\beta \mathbf{s}_{t-1} +(1-\beta)\mathbf{g}_t^2 \nonumber \\ +\boldsymbol{\theta}_{t+1}&=&\boldsymbol{\theta}_t - \eta_t { \mathbf{g}_t \over \sqrt{\mathbf{s}_t +\epsilon}}, \nonumber +\end{align} +$$ + +

    where \( \beta \) controls the averaging time of the second moment and is +typically taken to be about \( \beta=0.9 \), \( \eta_t \) is a learning rate +typically chosen to be \( 10^{-3} \), and \( \epsilon\sim 10^{-8} \) is a +small regularization constant to prevent divergences. Multiplication +and division by vectors is understood as an element-wise operation. It +is clear from this formula that the learning rate is reduced in +directions where the norm of the gradient is consistently large. This +greatly speeds up the convergence by allowing us to use a larger +learning rate for flat directions. +

    -

    diff --git a/doc/pub/week37/html/._week37-bs037.html b/doc/pub/week37/html/._week37-bs037.html index 32d254b75..95a6cb8b2 100644 --- a/doc/pub/week37/html/._week37-bs037.html +++ b/doc/pub/week37/html/._week37-bs037.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,82 +277,58 @@ MathJax.Hub.Config({

     

     

     

    -

    Sneaking in automatic differentiation using Autograd

    +

    ADAM optimizer

    -

    We anticipate our discussions to come in connection with neural networks and automatic differentiation -by showing how we can use autograd for the cases above. Later we will replace autograd with JAX. +

    A related algorithm is the ADAM optimizer. In +ADAM, we keep a running average of +both the first and second moment of the gradient and use this +information to adaptively change the learning rate for different +parameters. The method isefficient when working with large +problems involving lots data and/or parameters. It is a combination of the +gradient descent with momentum algorithm and the RMSprop algorithm +discussed above.

    +

    In addition to keeping a running average of the first and +second moments of the gradient +(i.e. \( \mathbf{m}_t=\mathbb{E}[\mathbf{g}_t] \) and +\( \mathbf{s}_t=\mathbb{E}[\mathbf{g}^2_t] \), respectively), ADAM +performs an additional bias correction to account for the fact that we +are estimating the first two moments of the gradient using a running +average (denoted by the hats in the update rule below). The update +rule for ADAM is given by (where multiplication and division are once +again understood to be element-wise operations below) +

    - -
    -
    -
    -
    -
    - 
    # Using Autograd to calculate gradients for OLS
-from random import random, seed
-import numpy as np
-import autograd.numpy as np
-import matplotlib.pyplot as plt
-from autograd import grad
+$$
+\begin{align}
+\mathbf{g}_t &= \nabla_\theta E(\boldsymbol{\theta}) 
+\tag{4}\\
+\mathbf{m}_t &= \beta_1 \mathbf{m}_{t-1} + (1-\beta_1) \mathbf{g}_t \nonumber \\
+\mathbf{s}_t &=\beta_2 \mathbf{s}_{t-1} +(1-\beta_2)\mathbf{g}_t^2 \nonumber \\
+\boldsymbol{\mathbf{m}}_t&={\mathbf{m}_t \over 1-\beta_1^t} \nonumber \\
+\boldsymbol{\mathbf{s}}_t &={\mathbf{s}_t \over1-\beta_2^t} \nonumber \\
+\boldsymbol{\theta}_{t+1}&=\boldsymbol{\theta}_t - \eta_t { \boldsymbol{\mathbf{m}}_t \over \sqrt{\boldsymbol{\mathbf{s}}_t} +\epsilon}, \nonumber \\
+\tag{5}
+\end{align}
+$$
 
-def CostOLS(beta):
-    return (1.0/n)*np.sum((y-X @ beta)**2)
+
    where \( \beta_1 \) and \( \beta_2 \) set the memory lifetime of the first and
+second moment and are typically taken to be \( 0.9 \) and \( 0.99 \)
+respectively, and \( \eta \) and \( \epsilon \) are identical to RMSprop.
+
    
 
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
+
    Like in RMSprop, the effective step size of a parameter depends on the
+magnitude of its gradient squared.  To understand this better, let us
+rewrite this expression in terms of the variance
+\( \boldsymbol{\sigma}_t^2 = \boldsymbol{\mathbf{s}}_t -
+(\boldsymbol{\mathbf{m}}_t)^2 \). Consider a single parameter \( \theta_t \). The
+update rule for this parameter is given by
+
    
 
-X = np.c_[np.ones((n,1)), x]
-XT_X = X.T @ X
-theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
-print("Own inversion")
-print(theta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
-
-theta = np.random.randn(2,1)
-eta = 1.0/np.max(EigValues)
-Niterations = 1000
-# define the gradient
-training_gradient = grad(CostOLS)
-
-for iter in range(Niterations):
-    gradients = training_gradient(theta)
-    theta -= eta*gradients
-print("theta from own gd")
-print(theta)
-
-xnew = np.array([[0],[2]])
-Xnew = np.c_[np.ones((2,1)), xnew]
-ypredict = Xnew.dot(theta)
-ypredict2 = Xnew.dot(theta_linreg)
-
-plt.plot(xnew, ypredict, "r-")
-plt.plot(xnew, ypredict2, "b-")
-plt.plot(x, y ,'ro')
-plt.axis([0,2.0,0, 15.0])
-plt.xlabel(r'$x$')
-plt.ylabel(r'$y$')
-plt.title(r'Random numbers ')
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +$$ +\Delta \theta_{t+1}= -\eta_t { \boldsymbol{m}_t \over \sqrt{\sigma_t^2 + m_t^2 }+\epsilon}. +$$

    @@ -359,6 +354,9 @@ plt.show()

  • 44
  • 45
  • 46
    • +
  • 47
    • +
  • ...
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs038.html b/doc/pub/week37/html/._week37-bs038.html index 0d2d20f03..706f006ab 100644 --- a/doc/pub/week37/html/._week37-bs038.html +++ b/doc/pub/week37/html/._week37-bs038.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,82 +277,32 @@ MathJax.Hub.Config({

     

     

     

    -

    Same code but now with momentum gradient descent

    +

    Algorithms and codes for Adagrad, RMSprop and Adam

    - -
    -
    -
    -
    -
    - 
    # Using Autograd to calculate gradients for OLS
-from random import random, seed
-import numpy as np
-import autograd.numpy as np
-import matplotlib.pyplot as plt
-from autograd import grad
+
    The algorithms we have implemented are well described in the text by Goodfellow, Bengio and Courville, chapter 8.
    
 
-def CostOLS(beta):
-    return (1.0/n)*np.sum((y-X @ beta)**2)
+
    The codes which implement these algorithms are discussed after our presentation of automatic differentiation.
    
+
    AdaGrad algorithm, taken from Goodfellow et al 
    
 
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x#+np.random.randn(n,1)
+

    
+
    
+
    
+
    
+

    
+
    RMSProp algorithm, taken from Goodfellow et al 
    
 
-X = np.c_[np.ones((n,1)), x]
-XT_X = X.T @ X
-theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
-print("Own inversion")
-print(theta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
-
-theta = np.random.randn(2,1)
-eta = 1.0/np.max(EigValues)
-Niterations = 30
-
-# define the gradient
-training_gradient = grad(CostOLS)
-
-for iter in range(Niterations):
-    gradients = training_gradient(theta)
-    theta -= eta*gradients
-    print(iter,gradients[0],gradients[1])
-print("theta from own gd")
-print(theta)
-
-# Now improve with momentum gradient descent
-change = 0.0
-delta_momentum = 0.3
-for iter in range(Niterations):
-    # calculate gradient
-    gradients = training_gradient(theta)
-    # calculate update
-    new_change = eta*gradients+delta_momentum*change
-    # take a step
-    theta -= new_change
-    # save the change
-    change = new_change
-    print(iter,gradients[0],gradients[1])
-print("theta from own gd wth momentum")
-print(theta)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    +
    +

    +
    +

    +

    ADAM algorithm, taken from Goodfellow et al

    +

    +
    +

    +
    +

    @@ -357,6 +326,8 @@ delta_momentum = 44

  • 45
  • 46
    • +
  • 47
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs039.html b/doc/pub/week37/html/._week37-bs039.html index 7aadc4545..dcc694333 100644 --- a/doc/pub/week37/html/._week37-bs039.html +++ b/doc/pub/week37/html/._week37-bs039.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,69 +277,14 @@ MathJax.Hub.Config({

     

     

     

    -

    But none of these can compete with Newton's method

    - - - -
    -
    -
    -
    -
    - 
    # Using Newton's method
-from random import random, seed
-import numpy as np
-import autograd.numpy as np
-import matplotlib.pyplot as plt
-from autograd import grad
-
-def CostOLS(beta):
-    return (1.0/n)*np.sum((y-X @ beta)**2)
-
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
-
-X = np.c_[np.ones((n,1)), x]
-XT_X = X.T @ X
-beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
-print("Own inversion")
-print(beta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-# Note that here the Hessian does not depend on the parameters beta
-invH = np.linalg.pinv(H)
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
-
-beta = np.random.randn(2,1)
-Niterations = 5
-
-# define the gradient
-training_gradient = grad(CostOLS)
-
-for iter in range(Niterations):
-    gradients = training_gradient(beta)
-    beta -= invH @ gradients
-    print(iter,gradients[0],gradients[1])
-print("beta from own Newton code")
-print(beta)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - +

    Practical tips

    +

    diff --git a/doc/pub/week37/html/._week37-bs040.html b/doc/pub/week37/html/._week37-bs040.html index e6c23f356..4d8cb7c5d 100644 --- a/doc/pub/week37/html/._week37-bs040.html +++ b/doc/pub/week37/html/._week37-bs040.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,8 +277,12 @@ MathJax.Hub.Config({

     

     

     

    -

    Including Stochastic Gradient Descent with Autograd

    -

    In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using autograd.

    +

    Sneaking in auotmatic differentiation using Autograd

    + +

    We conclude the part on optmization by showing how we can make codes +for linear regression and logistic regression using autograd. The +first example shows results with ordinary leats squares. +

    @@ -268,17 +291,15 @@ MathJax.Hub.Config({
    - 
    # Using Autograd to calculate gradients using SGD
-# OLS example
+  
    # Using Autograd to calculate gradients for OLS
 from random import random, seed
 import numpy as np
 import autograd.numpy as np
 import matplotlib.pyplot as plt
 from autograd import grad
 
-# Note change from previous example
-def CostOLS(y,X,theta):
-    return np.sum((y-X @ theta)**2)
+def CostOLS(beta):
+    return (1.0/n)*np.sum((y-X @ beta)**2)
 
 n = 100
 x = 2*np.random.rand(n,1)
@@ -297,12 +318,11 @@ EigValues, EigVectors = np= np.random.randn(2,1)
 eta = 1.0/np.max(EigValues)
 Niterations = 1000
-
-# Note that we request the derivative wrt third argument (theta, 2 here)
-training_gradient = grad(CostOLS,2)
+# define the gradient
+training_gradient = grad(CostOLS)
 
 for iter in range(Niterations):
-    gradients = (1.0/n)*training_gradient(y, X, theta)
+    gradients = training_gradient(theta)
     theta -= eta*gradients
 print("theta from own gd")
 print(theta)
@@ -320,27 +340,6 @@ plt.xlabel(r
 plt.ylabel(r'$y$')
 plt.title(r'Random numbers ')
 plt.show()
-
-n_epochs = 50
-M = 5   #size of each minibatch
-m = int(n/M) #number of minibatches
-t0, t1 = 5, 50
-def learning_schedule(t):
-    return t0/(t+t1)
-
-theta = np.random.randn(2,1)
-
-for epoch in range(n_epochs):
-# Can you figure out a better way of setting up the contributions to each batch?
-    for i in range(m):
-        random_index = M*np.random.randint(m)
-        xi = X[random_index:random_index+M]
-        yi = y[random_index:random_index+M]
-        gradients = (1.0/M)*training_gradient(yi, xi, theta)
-        eta = learning_schedule(epoch*m+i)
-        theta = theta - eta*gradients
-print("theta from own sdg")
-print(theta)
    @@ -377,6 +376,8 @@ theta = np.44
  • 45
  • 46
    • +
  • 47
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs041.html b/doc/pub/week37/html/._week37-bs041.html index ee18c3199..ca7cb828b 100644 --- a/doc/pub/week37/html/._week37-bs041.html +++ b/doc/pub/week37/html/._week37-bs041.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -266,21 +285,19 @@ MathJax.Hub.Config({
    - 
    # Using Autograd to calculate gradients using SGD
-# OLS example
+  
    # Using Autograd to calculate gradients for OLS
 from random import random, seed
 import numpy as np
 import autograd.numpy as np
 import matplotlib.pyplot as plt
 from autograd import grad
 
-# Note change from previous example
-def CostOLS(y,X,theta):
-    return np.sum((y-X @ theta)**2)
+def CostOLS(beta):
+    return (1.0/n)*np.sum((y-X @ beta)**2)
 
 n = 100
 x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
+y = 4+3*x#+np.random.randn(n,1)
 
 X = np.c_[np.ones((n,1)), x]
 XT_X = X.T @ X
@@ -294,44 +311,32 @@ EigValues, EigVectors = np= np.random.randn(2,1)
 eta = 1.0/np.max(EigValues)
-Niterations = 100
+Niterations = 30
 
-# Note that we request the derivative wrt third argument (theta, 2 here)
-training_gradient = grad(CostOLS,2)
+# define the gradient
+training_gradient = grad(CostOLS)
 
 for iter in range(Niterations):
-    gradients = (1.0/n)*training_gradient(y, X, theta)
+    gradients = training_gradient(theta)
     theta -= eta*gradients
+    print(iter,gradients[0],gradients[1])
 print("theta from own gd")
 print(theta)
 
-
-n_epochs = 50
-M = 5   #size of each minibatch
-m = int(n/M) #number of minibatches
-t0, t1 = 5, 50
-def learning_schedule(t):
-    return t0/(t+t1)
-
-theta = np.random.randn(2,1)
-
+# Now improve with momentum gradient descent
 change = 0.0
 delta_momentum = 0.3
-
-for epoch in range(n_epochs):
-    for i in range(m):
-        random_index = M*np.random.randint(m)
-        xi = X[random_index:random_index+M]
-        yi = y[random_index:random_index+M]
-        gradients = (1.0/M)*training_gradient(yi, xi, theta)
-        eta = learning_schedule(epoch*m+i)
-        # calculate update
-        new_change = eta*gradients+delta_momentum*change
-        # take a step
-        theta -= new_change
-        # save the change
-        change = new_change
-print("theta from own sdg with momentum")
+for iter in range(Niterations):
+    # calculate gradient
+    gradients = training_gradient(theta)
+    # calculate update
+    new_change = eta*gradients+delta_momentum*change
+    # take a step
+    theta -= new_change
+    # save the change
+    change = new_change
+    print(iter,gradients[0],gradients[1])
+print("theta from own gd wth momentum")
 print(theta)
    @@ -368,6 +373,8 @@ delta_momentum = 44
  • 45
  • 46
    • +
  • 47
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs042.html b/doc/pub/week37/html/._week37-bs042.html index ee24d5f87..d0a1066f6 100644 --- a/doc/pub/week37/html/._week37-bs042.html +++ b/doc/pub/week37/html/._week37-bs042.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,7 +277,9 @@ MathJax.Hub.Config({

     

     

     

    -

    Similar (second order function now) problem but now with AdaGrad

    +

    Including Stochastic Gradient Descent with Autograd

    +

    In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using autograd.

    +
    @@ -266,7 +287,7 @@ MathJax.Hub.Config({
    - 
    # Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
+  
    # Using Autograd to calculate gradients using SGD
 # OLS example
 from random import random, seed
 import numpy as np
@@ -278,41 +299,66 @@ MathJax.Hub.Config({
 def CostOLS(y,X,theta):
     return np.sum((y-X @ theta)**2)
 
-n = 1000
-x = np.random.rand(n,1)
-y = 2.0+3*x +4*x*x
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
 
-X = np.c_[np.ones((n,1)), x, x*x]
+X = np.c_[np.ones((n,1)), x]
 XT_X = X.T @ X
 theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
 print("Own inversion")
 print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
 
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 1000
 
 # Note that we request the derivative wrt third argument (theta, 2 here)
 training_gradient = grad(CostOLS,2)
-# Define parameters for Stochastic Gradient Descent
+
+for iter in range(Niterations):
+    gradients = (1.0/n)*training_gradient(y, X, theta)
+    theta -= eta*gradients
+print("theta from own gd")
+print(theta)
+
+xnew = np.array([[0],[2]])
+Xnew = np.c_[np.ones((2,1)), xnew]
+ypredict = Xnew.dot(theta)
+ypredict2 = Xnew.dot(theta_linreg)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(xnew, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
 n_epochs = 50
 M = 5   #size of each minibatch
 m = int(n/M) #number of minibatches
-# Guess for unknown parameters theta
-theta = np.random.randn(3,1)
+t0, t1 = 5, 50
+def learning_schedule(t):
+    return t0/(t+t1)
+
+theta = np.random.randn(2,1)
 
-# Value for learning rate
-eta = 0.01
-# Including AdaGrad parameter to avoid possible division by zero
-delta  = 1e-8
 for epoch in range(n_epochs):
-    Giter = 0.0
+# Can you figure out a better way of setting up the contributions to each batch?
     for i in range(m):
         random_index = M*np.random.randint(m)
         xi = X[random_index:random_index+M]
         yi = y[random_index:random_index+M]
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
-        Giter += gradients*gradients
-        update = gradients*eta/(delta+np.sqrt(Giter))
-        theta -= update
-print("theta from own AdaGrad")
+        eta = learning_schedule(epoch*m+i)
+        theta = theta - eta*gradients
+print("theta from own sdg")
 print(theta)
    @@ -329,7 +375,6 @@ delta = 1e-8
    -

    Running this code we note an almost perfect agreement with the results from matrix inversion.

    @@ -349,6 +394,8 @@ delta = 1e-844

  • 45
  • 46
    • +
  • 47
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs043.html b/doc/pub/week37/html/._week37-bs043.html index 95d62990c..ffedd3995 100644 --- a/doc/pub/week37/html/._week37-bs043.html +++ b/doc/pub/week37/html/._week37-bs043.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,7 +277,7 @@ MathJax.Hub.Config({

     

     

     

    -

    RMSprop for adaptive learning rate with Stochastic Gradient Descent

    +

    Same code but now with momentum gradient descent

    @@ -266,7 +285,7 @@ MathJax.Hub.Config({
    - 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+  
    # Using Autograd to calculate gradients using SGD
 # OLS example
 from random import random, seed
 import numpy as np
@@ -278,47 +297,60 @@ MathJax.Hub.Config({
 def CostOLS(y,X,theta):
     return np.sum((y-X @ theta)**2)
 
-n = 1000
-x = np.random.rand(n,1)
-y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x+np.random.randn(n,1)
 
-X = np.c_[np.ones((n,1)), x, x*x]
+X = np.c_[np.ones((n,1)), x]
 XT_X = X.T @ X
 theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
 print("Own inversion")
 print(theta_linreg)
+# Hessian matrix
+H = (2.0/n)* XT_X
+EigValues, EigVectors = np.linalg.eig(H)
+print(f"Eigenvalues of Hessian Matrix:{EigValues}")
 
+theta = np.random.randn(2,1)
+eta = 1.0/np.max(EigValues)
+Niterations = 100
 
 # Note that we request the derivative wrt third argument (theta, 2 here)
 training_gradient = grad(CostOLS,2)
-# Define parameters for Stochastic Gradient Descent
+
+for iter in range(Niterations):
+    gradients = (1.0/n)*training_gradient(y, X, theta)
+    theta -= eta*gradients
+print("theta from own gd")
+print(theta)
+
+
 n_epochs = 50
 M = 5   #size of each minibatch
 m = int(n/M) #number of minibatches
-# Guess for unknown parameters theta
-theta = np.random.randn(3,1)
+t0, t1 = 5, 50
+def learning_schedule(t):
+    return t0/(t+t1)
+
+theta = np.random.randn(2,1)
+
+change = 0.0
+delta_momentum = 0.3
 
-# Value for learning rate
-eta = 0.01
-# Value for parameter rho
-rho = 0.99
-# Including AdaGrad parameter to avoid possible division by zero
-delta  = 1e-8
 for epoch in range(n_epochs):
-    Giter = 0.0
     for i in range(m):
         random_index = M*np.random.randint(m)
         xi = X[random_index:random_index+M]
         yi = y[random_index:random_index+M]
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
-	# Accumulated gradient
-	# Scaling with rho the new and the previous results
-        Giter = (rho*Giter+(1-rho)*gradients*gradients)
-	# Taking the diagonal only and inverting
-        update = gradients*eta/(delta+np.sqrt(Giter))
-	# Hadamard product
-        theta -= update
-print("theta from own RMSprop")
+        eta = learning_schedule(epoch*m+i)
+        # calculate update
+        new_change = eta*gradients+delta_momentum*change
+        # take a step
+        theta -= new_change
+        # save the change
+        change = new_change
+print("theta from own sdg with momentum")
 print(theta)
    @@ -353,6 +385,8 @@ delta = 1e-844
  • 45
  • 46
    • +
  • 47
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs044.html b/doc/pub/week37/html/._week37-bs044.html index 8bebe3729..7324f3be7 100644 --- a/doc/pub/week37/html/._week37-bs044.html +++ b/doc/pub/week37/html/._week37-bs044.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,8 +277,7 @@ MathJax.Hub.Config({

     

     

     

    -

    And finally ADAM

    - +

    Similar (second order function now) problem but now with AdaGrad

    @@ -267,7 +285,7 @@ MathJax.Hub.Config({
    - 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+  
    # Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
 # OLS example
 from random import random, seed
 import numpy as np
@@ -281,7 +299,7 @@ MathJax.Hub.Config({
 
 n = 1000
 x = np.random.rand(n,1)
-y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+y = 2.0+3*x +4*x*x
 
 X = np.c_[np.ones((n,1)), x, x*x]
 XT_X = X.T @ X
@@ -301,30 +319,19 @@ theta = np.# Value for learning rate
 eta = 0.01
-# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
-beta1 = 0.9
-beta2 = 0.999
 # Including AdaGrad parameter to avoid possible division by zero
-delta  = 1e-7
-iter = 0
+delta  = 1e-8
 for epoch in range(n_epochs):
-    first_moment = 0.0
-    second_moment = 0.0
-    iter += 1
+    Giter = 0.0
     for i in range(m):
         random_index = M*np.random.randint(m)
         xi = X[random_index:random_index+M]
         yi = y[random_index:random_index+M]
         gradients = (1.0/M)*training_gradient(yi, xi, theta)
-        # Computing moments first
-        first_moment = beta1*first_moment + (1-beta1)*gradients
-        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
-        first_term = first_moment/(1.0-beta1**iter)
-        second_term = second_moment/(1.0-beta2**iter)
-	# Scaling with rho the new and the previous results
-        update = eta*first_term/(np.sqrt(second_term)+delta)
+        Giter += gradients*gradients
+        update = gradients*eta/(delta+np.sqrt(Giter))
         theta -= update
-print("theta from own ADAM")
+print("theta from own AdaGrad")
 print(theta)
    @@ -341,6 +348,7 @@ delta = 1e-7
    +

    Running this code we note an almost perfect agreement with the results from matrix inversion.

    @@ -358,6 +366,8 @@ delta = 1e-744

  • 45
  • 46
    • +
  • 47
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs045.html b/doc/pub/week37/html/._week37-bs045.html index 74fba6165..addba1319 100644 --- a/doc/pub/week37/html/._week37-bs045.html +++ b/doc/pub/week37/html/._week37-bs045.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -258,22 +277,84 @@ MathJax.Hub.Config({

     

     

     

    -

    Material for the lab sessions

    +

    RMSprop for adaptive learning rate with Stochastic Gradient Descent

    -
    -
    - -
      -
    1. Exercise set for week 37
      2. -
    2. Work on project 1 -
      3. -
        -
      • For more discussions of Ridge regression and calculation of averages, Wessel van Wieringen's article is highly recommended.
        • -
      -
    + +
    +
    +
    +
    +
    + 
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
+import matplotlib.pyplot as plt
+from autograd import grad
+
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
+
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
+
+
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
+
+# Value for learning rate
+eta = 0.01
+# Value for parameter rho
+rho = 0.99
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-8
+for epoch in range(n_epochs):
+    Giter = 0.0
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+	# Accumulated gradient
+	# Scaling with rho the new and the previous results
+        Giter = (rho*Giter+(1-rho)*gradients*gradients)
+	# Taking the diagonal only and inverting
+        update = gradients*eta/(delta+np.sqrt(Giter))
+	# Hadamard product
+        theta -= update
+print("theta from own RMSprop")
+print(theta)
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    - + +

      @@ -289,6 +370,9 @@ MathJax.Hub.Config({
    • 44
    • 45
    • 46
      • +
    • 47
      • +
    • 48
      • +
    • »
    diff --git a/doc/pub/week37/html/._week37-bs046.html b/doc/pub/week37/html/._week37-bs046.html index 9da2c6905..669e81735 100644 --- a/doc/pub/week37/html/._week37-bs046.html +++ b/doc/pub/week37/html/._week37-bs046.html @@ -40,159 +40,148 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'plans-for-week-37-lecture-monday'), - ('Plans for week 37, lab sessions', + ('Readings and Videos:', 2, None, 'readings-and-videos'), + ('Material for lecture Monday September 8', 2, None, - 'plans-for-week-37-lab-sessions'), - ('Material for lecture Monday September 9', + 'material-for-lecture-monday-september-8'), + ('Gradient descent and revisiting Ordinary Least Squares from ' + 'last week', 2, None, - 'material-for-lecture-monday-september-9'), - ('Deriving OLS from a probability distribution', + 'gradient-descent-and-revisiting-ordinary-least-squares-from-last-week'), + ('Gradient descent example', 2, None, 'gradient-descent-example'), + ('The derivative of the cost/loss function', 2, None, - 'deriving-ols-from-a-probability-distribution'), - ('Independent and Identically Distrubuted (iid)', + 'the-derivative-of-the-cost-loss-function'), + ('The Hessian matrix', 2, None, 'the-hessian-matrix'), + ('Simple program', 2, None, 'simple-program'), + ('Gradient Descent Example', 2, None, 'gradient-descent-example'), + ('Gradient descent and Ridge', 2, None, - 'independent-and-identically-distrubuted-iid'), - ('Maximum Likelihood Estimation (MLE)', + 'gradient-descent-and-ridge'), + ('The Hessian matrix for Ridge Regression', 2, None, - 'maximum-likelihood-estimation-mle'), - ('A new Cost Function', 2, None, 'a-new-cost-function'), - ("More basic Statistics and Bayes' theorem", + 'the-hessian-matrix-for-ridge-regression'), + ('Program example for gradient descent with Ridge Regression', 2, None, - 'more-basic-statistics-and-bayes-theorem'), - ('Marginal Probability', 2, None, 'marginal-probability'), - ('Conditional Probability', 2, None, 'conditional-probability'), - ("Bayes' Theorem", 2, None, 'bayes-theorem'), - ("Interpretations of Bayes' Theorem", + 'program-example-for-gradient-descent-with-ridge-regression'), + ('Using gradient descent methods, limitations', 2, None, - 'interpretations-of-bayes-theorem'), - ("Example of Usage of Bayes' theorem", + 'using-gradient-descent-methods-limitations'), + ('Improving gradient descent with momentum', 2, None, - 'example-of-usage-of-bayes-theorem'), - ('Doing it correctly', 2, None, 'doing-it-correctly'), - ("Bayes' Theorem and Ridge and Lasso Regression", + 'improving-gradient-descent-with-momentum'), + ('Same code but now with momentum gradient descent', 2, None, - 'bayes-theorem-and-ridge-and-lasso-regression'), - ('Ridge and Bayes', 2, None, 'ridge-and-bayes'), - ('Lasso and Bayes', 2, None, 'lasso-and-bayes'), - ('Why resampling methods', 2, None, 'why-resampling-methods'), - ('Resampling methods', 2, None, 'resampling-methods'), - ('Resampling approaches can be computationally expensive', + 'same-code-but-now-with-momentum-gradient-descent'), + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'resampling-approaches-can-be-computationally-expensive'), - ('Why resampling methods ?', 2, None, 'why-resampling-methods'), - ('Statistical analysis', 2, None, 'statistical-analysis'), - ('Resampling methods', 2, None, 'resampling-methods'), - ('Resampling methods: Bootstrap', + 'overview-video-on-stochastic-gradient-descent-sgd'), + ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), + ('Stochastic Gradient Descent (SGD)', 2, None, - 'resampling-methods-bootstrap'), - ('The Central Limit Theorem', + 'stochastic-gradient-descent-sgd'), + ('Stochastic Gradient Descent', 2, None, - 'the-central-limit-theorem'), - ('Finding the Limit', 2, None, 'finding-the-limit'), - ('Rewriting the $\\delta$-function', + 'stochastic-gradient-descent'), + ('Computation of gradients', 2, None, 'computation-of-gradients'), + ('SGD example', 2, None, 'sgd-example'), + ('The gradient step', 2, None, 'the-gradient-step'), + ('Simple example code', 2, None, 'simple-example-code'), + ('When do we stop?', 2, None, 'when-do-we-stop'), + ('Slightly different approach', 2, None, - 'rewriting-the-delta-function'), - ('Identifying Terms', 2, None, 'identifying-terms'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Confidence Intervals', 2, None, 'confidence-intervals'), - ('Standard Approach based on the Normal Distribution', + 'slightly-different-approach'), + ('Time decay rate', 2, None, 'time-decay-rate'), + ('Code with a Number of Minibatches which varies', 2, None, - 'standard-approach-based-on-the-normal-distribution'), - ('Resampling methods: Bootstrap background', + 'code-with-a-number-of-minibatches-which-varies'), + ('Replace or not', 2, None, 'replace-or-not'), + ('Momentum based GD', 2, None, 'momentum-based-gd'), + ('More on momentum based approaches', 2, None, - 'resampling-methods-bootstrap-background'), - ('Resampling methods: More Bootstrap background', + 'more-on-momentum-based-approaches'), + ('Momentum parameter', 2, None, 'momentum-parameter'), + ('Second moment of the gradient', 2, None, - 'resampling-methods-more-bootstrap-background'), - ('Resampling methods: Bootstrap approach', + 'second-moment-of-the-gradient'), + ('RMS prop', 2, None, 'rms-prop'), + ('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"', 2, None, - 'resampling-methods-bootstrap-approach'), - ('Resampling methods: Bootstrap steps', + 'adam-optimizer-https-arxiv-org-abs-1412-6980'), + ('Algorithms and codes for Adagrad, RMSprop and Adam', 2, None, - 'resampling-methods-bootstrap-steps'), - ('Code example for the Bootstrap method', + 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'code-example-for-the-bootstrap-method'), - ('Plotting the Histogram', 2, None, 'plotting-the-histogram'), - ('The bias-variance tradeoff', + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'the-bias-variance-tradeoff'), - ('A way to Read the Bias-Variance Tradeoff', + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'a-way-to-read-the-bias-variance-tradeoff'), - ('Example code for Bias-Variance tradeoff', + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', 2, None, - 'example-code-for-bias-variance-tradeoff'), - ('Understanding what happens', + 'sneaking-in-auotmatic-differentiation-using-autograd'), + ('Same code but now with momentum gradient descent', 2, None, - 'understanding-what-happens'), - ('Summing up', 2, None, 'summing-up'), - ("Another Example from Scikit-Learn's Repository", + 'same-code-but-now-with-momentum-gradient-descent'), + ('Including Stochastic Gradient Descent with Autograd', 2, None, - 'another-example-from-scikit-learn-s-repository'), - ('Various steps in cross-validation', + 'including-stochastic-gradient-descent-with-autograd'), + ('Same code but now with momentum gradient descent', 2, None, - 'various-steps-in-cross-validation'), - ('Cross-validation in brief', + 'same-code-but-now-with-momentum-gradient-descent'), + ('Similar (second order function now) problem but now with ' + 'AdaGrad', 2, None, - 'cross-validation-in-brief'), - ('Code Example for Cross-validation and $k$-fold ' - 'Cross-validation', + 'similar-second-order-function-now-problem-but-now-with-adagrad'), + ('RMSprop for adaptive learning rate with Stochastic Gradient ' + 'Descent', 2, None, - 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('More examples on bootstrap and cross-validation and errors', + 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), + ('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"', 2, None, - 'more-examples-on-bootstrap-and-cross-validation-and-errors'), - ('The same example but now with cross-validation', - 2, - None, - 'the-same-example-but-now-with-cross-validation'), + 'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'), ('Material for the lab sessions', 2, None, - 'material-for-the-lab-sessions'), - ('Linking the regression analysis with a statistical ' - 'interpretation', - 2, - None, - 'linking-the-regression-analysis-with-a-statistical-interpretation'), - ('Assumptions made', 2, None, 'assumptions-made'), - ('Expectation value and variance', - 2, - None, - 'expectation-value-and-variance'), - ('Expectation value and variance for $\\boldsymbol{\\beta}$', - 2, - None, - 'expectation-value-and-variance-for-boldsymbol-beta')]} + 'material-for-the-lab-sessions')]} end of tocinfo --> @@ -228,58 +217,55 @@ MathJax.Hub.Config({ Contents @@ -291,9 +277,8 @@ MathJax.Hub.Config({

     

     

     

    -

    Code Example for Cross-validation and \( k \)-fold Cross-validation

    +

    And finally ADAM

    -

    The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial.

    @@ -301,95 +286,65 @@ MathJax.Hub.Config({
    - 
    import numpy as np
+  
    # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
+# OLS example
+from random import random, seed
+import numpy as np
+import autograd.numpy as np
 import matplotlib.pyplot as plt
-from sklearn.model_selection import KFold
-from sklearn.linear_model import Ridge
-from sklearn.model_selection import cross_val_score
-from sklearn.preprocessing import PolynomialFeatures
+from autograd import grad
 
-# A seed just to ensure that the random numbers are the same for every run.
-# Useful for eventual debugging.
-np.random.seed(3155)
+# Note change from previous example
+def CostOLS(y,X,theta):
+    return np.sum((y-X @ theta)**2)
 
-# Generate the data.
-nsamples = 100
-x = np.random.randn(nsamples)
-y = 3*x**2 + np.random.randn(nsamples)
+n = 1000
+x = np.random.rand(n,1)
+y = 2.0+3*x +4*x*x# +np.random.randn(n,1)
 
-## Cross-validation on Ridge regression using KFold only
-
-# Decide degree on polynomial to fit
-poly = PolynomialFeatures(degree = 6)
-
-# Decide which values of lambda to use
-nlambdas = 500
-lambdas = np.logspace(-3, 5, nlambdas)
-
-# Initialize a KFold instance
-k = 5
-kfold = KFold(n_splits = k)
-
-# Perform the cross-validation to estimate MSE
-scores_KFold = np.zeros((nlambdas, k))
-
-i = 0
-for lmb in lambdas:
-    ridge = Ridge(alpha = lmb)
-    j = 0
-    for train_inds, test_inds in kfold.split(x):
-        xtrain = x[train_inds]
-        ytrain = y[train_inds]
-
-        xtest = x[test_inds]
-        ytest = y[test_inds]
-
-        Xtrain = poly.fit_transform(xtrain[:, np.newaxis])
-        ridge.fit(Xtrain, ytrain[:, np.newaxis])
-
-        Xtest = poly.fit_transform(xtest[:, np.newaxis])
-        ypred = ridge.predict(Xtest)
-
-        scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred)
-
-        j += 1
-    i += 1
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+theta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
+print("Own inversion")
+print(theta_linreg)
 
 
-estimated_mse_KFold = np.mean(scores_KFold, axis = 1)
+# Note that we request the derivative wrt third argument (theta, 2 here)
+training_gradient = grad(CostOLS,2)
+# Define parameters for Stochastic Gradient Descent
+n_epochs = 50
+M = 5   #size of each minibatch
+m = int(n/M) #number of minibatches
+# Guess for unknown parameters theta
+theta = np.random.randn(3,1)
 
-## Cross-validation using cross_val_score from sklearn along with KFold
-
-# kfold is an instance initialized above as:
-# kfold = KFold(n_splits = k)
-
-estimated_mse_sklearn = np.zeros(nlambdas)
-i = 0
-for lmb in lambdas:
-    ridge = Ridge(alpha = lmb)
-
-    X = poly.fit_transform(x[:, np.newaxis])
-    estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold)
-
-    # cross_val_score return an array containing the estimated negative mse for every fold.
-    # we have to the the mean of every array in order to get an estimate of the mse of the model
-    estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
-
-    i += 1
-
-## Plot and compare the slightly different ways to perform cross-validation
-
-plt.figure()
-
-plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
-plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold')
-
-plt.xlabel('log10(lambda)')
-plt.ylabel('mse')
-
-plt.legend()
-
-plt.show()
+# Value for learning rate
+eta = 0.01
+# Value for parameters beta1 and beta2, see https://arxiv.org/abs/1412.6980
+beta1 = 0.9
+beta2 = 0.999
+# Including AdaGrad parameter to avoid possible division by zero
+delta  = 1e-7
+iter = 0
+for epoch in range(n_epochs):
+    first_moment = 0.0
+    second_moment = 0.0
+    iter += 1
+    for i in range(m):
+        random_index = M*np.random.randint(m)
+        xi = X[random_index:random_index+M]
+        yi = y[random_index:random_index+M]
+        gradients = (1.0/M)*training_gradient(yi, xi, theta)
+        # Computing moments first
+        first_moment = beta1*first_moment + (1-beta1)*gradients
+        second_moment = beta2*second_moment+(1-beta2)*gradients*gradients
+        first_term = first_moment/(1.0-beta1**iter)
+        second_term = second_moment/(1.0-beta2**iter)
+	# Scaling with rho the new and the previous results
+        update = eta*first_term/(np.sqrt(second_term)+delta)
+        theta -= update
+print("theta from own ADAM")
+print(theta)
    @@ -422,12 +377,6 @@ plt.show()
  • 46
  • 47
  • 48
    • -
  • 49
    • -
  • 50
    • -
  • 51
    • -
  • 52
    • -
  • 53
    • -
  • 54
  • »
    • diff --git a/doc/pub/week37/html/._week37-bs047.html b/doc/pub/week37/html/._week37-bs047.html index a80e1fa8d..e2634b78a 100644 --- a/doc/pub/week37/html/._week37-bs047.html +++ b/doc/pub/week37/html/._week37-bs047.html @@ -40,159 +40,148 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'plans-for-week-37-lecture-monday'), - ('Plans for week 37, lab sessions', + ('Readings and Videos:', 2, None, 'readings-and-videos'), + ('Material for lecture Monday September 8', 2, None, - 'plans-for-week-37-lab-sessions'), - ('Material for lecture Monday September 9', + 'material-for-lecture-monday-september-8'), + ('Gradient descent and revisiting Ordinary Least Squares from ' + 'last week', 2, None, - 'material-for-lecture-monday-september-9'), - ('Deriving OLS from a probability distribution', + 'gradient-descent-and-revisiting-ordinary-least-squares-from-last-week'), + ('Gradient descent example', 2, None, 'gradient-descent-example'), + ('The derivative of the cost/loss function', 2, None, - 'deriving-ols-from-a-probability-distribution'), - ('Independent and Identically Distrubuted (iid)', + 'the-derivative-of-the-cost-loss-function'), + ('The Hessian matrix', 2, None, 'the-hessian-matrix'), + ('Simple program', 2, None, 'simple-program'), + ('Gradient Descent Example', 2, None, 'gradient-descent-example'), + ('Gradient descent and Ridge', 2, None, - 'independent-and-identically-distrubuted-iid'), - ('Maximum Likelihood Estimation (MLE)', + 'gradient-descent-and-ridge'), + ('The Hessian matrix for Ridge Regression', 2, None, - 'maximum-likelihood-estimation-mle'), - ('A new Cost Function', 2, None, 'a-new-cost-function'), - ("More basic Statistics and Bayes' theorem", + 'the-hessian-matrix-for-ridge-regression'), + ('Program example for gradient descent with Ridge Regression', 2, None, - 'more-basic-statistics-and-bayes-theorem'), - ('Marginal Probability', 2, None, 'marginal-probability'), - ('Conditional Probability', 2, None, 'conditional-probability'), - ("Bayes' Theorem", 2, None, 'bayes-theorem'), - ("Interpretations of Bayes' Theorem", + 'program-example-for-gradient-descent-with-ridge-regression'), + ('Using gradient descent methods, limitations', 2, None, - 'interpretations-of-bayes-theorem'), - ("Example of Usage of Bayes' theorem", + 'using-gradient-descent-methods-limitations'), + ('Improving gradient descent with momentum', 2, None, - 'example-of-usage-of-bayes-theorem'), - ('Doing it correctly', 2, None, 'doing-it-correctly'), - ("Bayes' Theorem and Ridge and Lasso Regression", + 'improving-gradient-descent-with-momentum'), + ('Same code but now with momentum gradient descent', 2, None, - 'bayes-theorem-and-ridge-and-lasso-regression'), - ('Ridge and Bayes', 2, None, 'ridge-and-bayes'), - ('Lasso and Bayes', 2, None, 'lasso-and-bayes'), - ('Why resampling methods', 2, None, 'why-resampling-methods'), - ('Resampling methods', 2, None, 'resampling-methods'), - ('Resampling approaches can be computationally expensive', + 'same-code-but-now-with-momentum-gradient-descent'), + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'resampling-approaches-can-be-computationally-expensive'), - ('Why resampling methods ?', 2, None, 'why-resampling-methods'), - ('Statistical analysis', 2, None, 'statistical-analysis'), - ('Resampling methods', 2, None, 'resampling-methods'), - ('Resampling methods: Bootstrap', + 'overview-video-on-stochastic-gradient-descent-sgd'), + ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), + ('Stochastic Gradient Descent (SGD)', 2, None, - 'resampling-methods-bootstrap'), - ('The Central Limit Theorem', + 'stochastic-gradient-descent-sgd'), + ('Stochastic Gradient Descent', 2, None, - 'the-central-limit-theorem'), - ('Finding the Limit', 2, None, 'finding-the-limit'), - ('Rewriting the $\\delta$-function', + 'stochastic-gradient-descent'), + ('Computation of gradients', 2, None, 'computation-of-gradients'), + ('SGD example', 2, None, 'sgd-example'), + ('The gradient step', 2, None, 'the-gradient-step'), + ('Simple example code', 2, None, 'simple-example-code'), + ('When do we stop?', 2, None, 'when-do-we-stop'), + ('Slightly different approach', 2, None, - 'rewriting-the-delta-function'), - ('Identifying Terms', 2, None, 'identifying-terms'), - ('Wrapping it up', 2, None, 'wrapping-it-up'), - ('Confidence Intervals', 2, None, 'confidence-intervals'), - ('Standard Approach based on the Normal Distribution', + 'slightly-different-approach'), + ('Time decay rate', 2, None, 'time-decay-rate'), + ('Code with a Number of Minibatches which varies', 2, None, - 'standard-approach-based-on-the-normal-distribution'), - ('Resampling methods: Bootstrap background', + 'code-with-a-number-of-minibatches-which-varies'), + ('Replace or not', 2, None, 'replace-or-not'), + ('Momentum based GD', 2, None, 'momentum-based-gd'), + ('More on momentum based approaches', 2, None, - 'resampling-methods-bootstrap-background'), - ('Resampling methods: More Bootstrap background', + 'more-on-momentum-based-approaches'), + ('Momentum parameter', 2, None, 'momentum-parameter'), + ('Second moment of the gradient', 2, None, - 'resampling-methods-more-bootstrap-background'), - ('Resampling methods: Bootstrap approach', + 'second-moment-of-the-gradient'), + ('RMS prop', 2, None, 'rms-prop'), + ('"ADAM optimizer":"https://arxiv.org/abs/1412.6980"', 2, None, - 'resampling-methods-bootstrap-approach'), - ('Resampling methods: Bootstrap steps', + 'adam-optimizer-https-arxiv-org-abs-1412-6980'), + ('Algorithms and codes for Adagrad, RMSprop and Adam', 2, None, - 'resampling-methods-bootstrap-steps'), - ('Code example for the Bootstrap method', + 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'code-example-for-the-bootstrap-method'), - ('Plotting the Histogram', 2, None, 'plotting-the-histogram'), - ('The bias-variance tradeoff', + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'the-bias-variance-tradeoff'), - ('A way to Read the Bias-Variance Tradeoff', + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'a-way-to-read-the-bias-variance-tradeoff'), - ('Example code for Bias-Variance tradeoff', + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', 2, None, - 'example-code-for-bias-variance-tradeoff'), - ('Understanding what happens', + 'sneaking-in-auotmatic-differentiation-using-autograd'), + ('Same code but now with momentum gradient descent', 2, None, - 'understanding-what-happens'), - ('Summing up', 2, None, 'summing-up'), - ("Another Example from Scikit-Learn's Repository", + 'same-code-but-now-with-momentum-gradient-descent'), + ('Including Stochastic Gradient Descent with Autograd', 2, None, - 'another-example-from-scikit-learn-s-repository'), - ('Various steps in cross-validation', + 'including-stochastic-gradient-descent-with-autograd'), + ('Same code but now with momentum gradient descent', 2, None, - 'various-steps-in-cross-validation'), - ('Cross-validation in brief', + 'same-code-but-now-with-momentum-gradient-descent'), + ('Similar (second order function now) problem but now with ' + 'AdaGrad', 2, None, - 'cross-validation-in-brief'), - ('Code Example for Cross-validation and $k$-fold ' - 'Cross-validation', + 'similar-second-order-function-now-problem-but-now-with-adagrad'), + ('RMSprop for adaptive learning rate with Stochastic Gradient ' + 'Descent', 2, None, - 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('More examples on bootstrap and cross-validation and errors', + 'rmsprop-for-adaptive-learning-rate-with-stochastic-gradient-descent'), + ('And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"', 2, None, - 'more-examples-on-bootstrap-and-cross-validation-and-errors'), - ('The same example but now with cross-validation', - 2, - None, - 'the-same-example-but-now-with-cross-validation'), + 'and-finally-adam-https-arxiv-org-pdf-1412-6980-pdf'), ('Material for the lab sessions', 2, None, - 'material-for-the-lab-sessions'), - ('Linking the regression analysis with a statistical ' - 'interpretation', - 2, - None, - 'linking-the-regression-analysis-with-a-statistical-interpretation'), - ('Assumptions made', 2, None, 'assumptions-made'), - ('Expectation value and variance', - 2, - None, - 'expectation-value-and-variance'), - ('Expectation value and variance for $\\boldsymbol{\\beta}$', - 2, - None, - 'expectation-value-and-variance-for-boldsymbol-beta')]} + 'material-for-the-lab-sessions')]} end of tocinfo --> @@ -228,58 +217,55 @@ MathJax.Hub.Config({ Contents @@ -291,109 +277,22 @@ MathJax.Hub.Config({

     

     

     

    -

    More examples on bootstrap and cross-validation and errors

    +

    Material for the lab sessions

    - - -
    -
    -
    -
    -
    - 
    # Common imports
-import os
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.linear_model import LinearRegression, Ridge, Lasso
-from sklearn.model_selection import train_test_split
-from sklearn.utils import resample
-from sklearn.metrics import mean_squared_error
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
-    os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
-    os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
-    os.makedirs(DATA_ID)
-
-def image_path(fig_id):
-    return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
-    return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
-    plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("EoS.csv"),'r')
-
-# Read the EoS data as  csv file and organize the data into two arrays with density and energies
-EoS = pd.read_csv(infile, names=('Density', 'Energy'))
-EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')
-EoS = EoS.dropna()
-Energies = EoS['Energy']
-Density = EoS['Density']
-#  The design matrix now as function of various polytrops
-
-Maxpolydegree = 30
-X = np.zeros((len(Density),Maxpolydegree))
-X[:,0] = 1.0
-testerror = np.zeros(Maxpolydegree)
-trainingerror = np.zeros(Maxpolydegree)
-polynomial = np.zeros(Maxpolydegree)
-
-trials = 100
-for polydegree in range(1, Maxpolydegree):
-    polynomial[polydegree] = polydegree
-    for degree in range(polydegree):
-        X[:,degree] = Density**(degree/3.0)
-
-# loop over trials in order to estimate the expectation value of the MSE
-    testerror[polydegree] = 0.0
-    trainingerror[polydegree] = 0.0
-    for samples in range(trials):
-        x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)
-        model = LinearRegression(fit_intercept=False).fit(x_train, y_train)
-        ypred = model.predict(x_train)
-        ytilde = model.predict(x_test)
-        testerror[polydegree] += mean_squared_error(y_test, ytilde)
-        trainingerror[polydegree] += mean_squared_error(y_train, ypred) 
-
-    testerror[polydegree] /= trials
-    trainingerror[polydegree] /= trials
-    print("Degree of polynomial: %3d"% polynomial[polydegree])
-    print("Mean squared error on training data: %.8f" % trainingerror[polydegree])
-    print("Mean squared error on test data: %.8f" % testerror[polydegree])
-
-plt.plot(polynomial, np.log10(trainingerror), label='Training Error')
-plt.plot(polynomial, np.log10(testerror), label='Test Error')
-plt.xlabel('Polynomial degree')
-plt.ylabel('log10[MSE]')
-plt.legend()
-plt.show()
-
    +
    +
    + +
      +
    1. Exercise set for week 37
      2. +
    2. Work on project 1 +
      3. +
        +
      • For more discussions of Ridge regression and calculation of averages, Wessel van Wieringen's article is highly recommended.
        • +
      +
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    Note that we kept the intercept column in the fitting here. This means that we need to set the intercept in the call to the Scikit-Learn function as False. Alternatively, we could have set up the design matrix \( X \) without the first column of ones.

    +

    @@ -410,13 +309,6 @@ plt.show()

  • 46
  • 47
  • 48
    • -
  • 49
    • -
  • 50
    • -
  • 51
    • -
  • 52
    • -
  • 53
    • -
  • 54
    • -
  • »
    • diff --git a/doc/pub/week37/html/week37-bs.html b/doc/pub/week37/html/week37-bs.html index 541ecc814..cfe72140a 100644 --- a/doc/pub/week37/html/week37-bs.html +++ b/doc/pub/week37/html/week37-bs.html @@ -82,11 +82,14 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -129,19 +132,30 @@ doconce format html week37.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -216,37 +230,42 @@ MathJax.Hub.Config({
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
  • Improving gradient descent with momentum
    • -
  • Same code but now with momentum gradient descent
    • -
  • Overview video on Stochastic Gradient Descent
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent (SGD)
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • -
  • Practical tips
    • -
  • Sneaking in automatic differentiation using Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • But none of these can compete with Newton's method
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • Same code but now with momentum gradient descent
    • -
  • Similar (second order function now) problem but now with AdaGrad
    • -
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • -
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • -
  • Material for the lab sessions
    • +
  • Pros and cons
    • +
  • Convergence rates
    • +
  • Accuracy
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Algorithms and codes for Adagrad, RMSprop and Adam
    • +
  • AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html"
    • +
  • Practical tips
    • +
  • Sneaking in auotmatic differentiation using Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • Same code but now with momentum gradient descent
    • +
  • Similar (second order function now) problem but now with AdaGrad
    • +
  • RMSprop for adaptive learning rate with Stochastic Gradient Descent
    • +
  • And finally "ADAM":"https://arxiv.org/pdf/1412.6980.pdf"
    • +
  • Material for the lab sessions
    • @@ -300,7 +319,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 46
    • +
  • 48
  • »
    • diff --git a/doc/pub/week37/html/week37-reveal.html b/doc/pub/week37/html/week37-reveal.html index c8984a7e6..f33912778 100644 --- a/doc/pub/week37/html/week37-reveal.html +++ b/doc/pub/week37/html/week37-reveal.html @@ -217,7 +217,7 @@ MathJax.Hub.Config({

      -

    1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at URL::https://www.deeplearningbook.org/contents/optimization.html"
      2. +

    2. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at https://www.deeplearningbook.org/contents/optimization.html

    3. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.

    4. Video on gradient descent at https://www.youtube.com/watch?v=sDv4f4s2SB8

    5. Video on Stochastic gradient descent at https://www.youtube.com/watch?v=vMh0zPT0tLI
      6. @@ -743,9 +743,16 @@ pyplot.show()
      -

      Overview video on Stochastic Gradient Descent

      +

      Overview video on Stochastic Gradient Descent (SGD)

      What is Stochastic Gradient Descent +

      There are several reasons for using stochastic gradient descent. Some of these are:

      + +
        +

      1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.
        2. +

      2. Hopefully avoid Local Minima
        3. +

      3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset.
        4. +
      @@ -764,6 +771,50 @@ perform a parameter update.

      +
      +

      Pros and cons

      + +
        +

      1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.
        2. +

      2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.
        3. +

      3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient.
        4. +
      +
      + +
      +

      Convergence rates

      + +
        +

      1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.
        2. +

      2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration.
        3. +
      +
      + +
      +

      Accuracy

      + +

      In general, stochastic Gradient Descent is Less accurate than gradient +descent, as it calculates the gradient on single examples, which may +not accurately represent the overall dataset. Gradient Descent is +more accurate because it uses the average gradient calculated over the +entire dataset. +

      + +

      There are other disadvantages to using SGD. The main drawback is that +its convergence behaviour can be more erratic due to the random +sampling of individual training examples. This can lead to less +accurate results, as the algorithm may not converge to the true +minimum of the cost function. Additionally, the learning rate, which +determines the step size of each update to the model’s parameters, +must be carefully chosen to ensure convergence. +

      + +

      It is however the method of choice in deep learning algorithms where +SGD is often used in combination with other optimization techniques, +such as momentum or adaptive learning rates +

      +
      +

      Stochastic Gradient Descent (SGD)

      @@ -1249,7 +1300,9 @@ $$ $$

       
      -

      One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of \( \gamma \).

      +

      One of the major advantages of NAG is that it allows for the use of a +larger learning rate than GDM for the same choice of \( \gamma \). +

      @@ -1272,7 +1325,7 @@ the steep computational price of calculating or approximating Hessians.

      -

      Recently, a number of methods have been introduced that accomplish +

      During the last decade a number of methods have been introduced that accomplish this by tracking not only the gradient, but also the second moment of the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and ADAM. @@ -1375,7 +1428,28 @@ $$

      The algorithms we have implemented are well described in the text by Goodfellow, Bengio and Courville, chapter 8.

      -

      The codes which implement these algorithms are discussed below here.

      +

      The codes which implement these algorithms are discussed after our presentation of automatic differentiation.

      +

      AdaGrad algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +

      +

      RMSProp algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +

      +

      ADAM algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +

      @@ -1390,10 +1464,11 @@ $$
      -

      Sneaking in automatic differentiation using Autograd

      +

      Sneaking in auotmatic differentiation using Autograd

      -

      We anticipate our discussions to come in connection with neural networks and automatic differentiation -by showing how we can use autograd for the cases above. Later we will replace autograd with JAX. +

      We conclude the part on optmization by showing how we can make codes +for linear regression and logistic regression using autograd. The +first example shows results with ordinary leats squares.

      @@ -1546,70 +1621,6 @@ delta_momentum = 0.3
    -
    -

    But none of these can compete with Newton's method

    - - - -
    -
    -
    -
    -
    - 
    # Using Newton's method
-from random import random, seed
-import numpy as np
-import autograd.numpy as np
-import matplotlib.pyplot as plt
-from autograd import grad
-
-def CostOLS(beta):
-    return (1.0/n)*np.sum((y-X @ beta)**2)
-
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
-
-X = np.c_[np.ones((n,1)), x]
-XT_X = X.T @ X
-beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
-print("Own inversion")
-print(beta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-# Note that here the Hessian does not depend on the parameters beta
-invH = np.linalg.pinv(H)
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
-
-beta = np.random.randn(2,1)
-Niterations = 5
-
-# define the gradient
-training_gradient = grad(CostOLS)
-
-for iter in range(Niterations):
-    gradients = training_gradient(beta)
-    beta -= invH @ gradients
-    print(iter,gradients[0],gradients[1])
-print("beta from own Newton code")
-print(beta)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -

    Including Stochastic Gradient Descent with Autograd

    In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using autograd.

    diff --git a/doc/pub/week37/html/week37-solarized.html b/doc/pub/week37/html/week37-solarized.html index f1b99880d..db444578b 100644 --- a/doc/pub/week37/html/week37-solarized.html +++ b/doc/pub/week37/html/week37-solarized.html @@ -109,11 +109,14 @@ div.toc p,a { 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -156,19 +159,30 @@ div.toc p,a { 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -258,7 +272,7 @@ MathJax.Hub.Config({

      -
    1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at URL::https://www.deeplearningbook.org/contents/optimization.html"
      2. +
    2. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at https://www.deeplearningbook.org/contents/optimization.html
    3. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.
    4. Video on gradient descent at https://www.youtube.com/watch?v=sDv4f4s2SB8
    5. Video on Stochastic gradient descent at https://www.youtube.com/watch?v=vMh0zPT0tLI
      6. @@ -753,10 +767,16 @@ pyplot.show()









      -

      Overview video on Stochastic Gradient Descent

      +

      Overview video on Stochastic Gradient Descent (SGD)

      What is Stochastic Gradient Descent +

      There are several reasons for using stochastic gradient descent. Some of these are:

      +
        +
      1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.
        2. +
      2. Hopefully avoid Local Minima
        3. +
      3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset.
        4. +










      Batches and mini-batches

      @@ -772,6 +792,45 @@ an entire training set of several millions. This batch is then used to perform a parameter update.

      +









      +

      Pros and cons

      + +
        +
      1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.
        2. +
      2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.
        3. +
      3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient.
        4. +
      +









      +

      Convergence rates

      + +
        +
      1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.
        2. +
      2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration.
        3. +
      +









      +

      Accuracy

      + +

      In general, stochastic Gradient Descent is Less accurate than gradient +descent, as it calculates the gradient on single examples, which may +not accurately represent the overall dataset. Gradient Descent is +more accurate because it uses the average gradient calculated over the +entire dataset. +

      + +

      There are other disadvantages to using SGD. The main drawback is that +its convergence behaviour can be more erratic due to the random +sampling of individual training examples. This can lead to less +accurate results, as the algorithm may not converge to the true +minimum of the cost function. Additionally, the learning rate, which +determines the step size of each update to the model’s parameters, +must be carefully chosen to ensure convergence. +

      + +

      It is however the method of choice in deep learning algorithms where +SGD is often used in combination with other optimization techniques, +such as momentum or adaptive learning rates +

      +









      Stochastic Gradient Descent (SGD)

      @@ -1225,7 +1284,9 @@ $$ \end{align} $$ -

      One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of \( \gamma \).

      +

      One of the major advantages of NAG is that it allows for the use of a +larger learning rate than GDM for the same choice of \( \gamma \). +











      Second moment of the gradient

      @@ -1247,7 +1308,7 @@ the steep computational price of calculating or approximating Hessians.

      -

      Recently, a number of methods have been introduced that accomplish +

      During the last decade a number of methods have been introduced that accomplish this by tracking not only the gradient, but also the second moment of the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and ADAM. @@ -1342,7 +1403,28 @@ $$

      The algorithms we have implemented are well described in the text by Goodfellow, Bengio and Courville, chapter 8.

      -

      The codes which implement these algorithms are discussed below here.

      +

      The codes which implement these algorithms are discussed after our presentation of automatic differentiation.

      +

      AdaGrad algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +

      +

      RMSProp algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +

      +

      ADAM algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +











      Practical tips

      @@ -1354,10 +1436,11 @@ $$
    6. Adaptive optimization methods don't always have good generalization. Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications.










      7. -

      Sneaking in automatic differentiation using Autograd

      +

      Sneaking in auotmatic differentiation using Autograd

      -

      We anticipate our discussions to come in connection with neural networks and automatic differentiation -by showing how we can use autograd for the cases above. Later we will replace autograd with JAX. +

      We conclude the part on optmization by showing how we can make codes +for linear regression and logistic regression using autograd. The +first example shows results with ordinary leats squares.

      @@ -1510,70 +1593,6 @@ delta_momentum = 0.3
    -









    -

    But none of these can compete with Newton's method

    - - - -
    -
    -
    -
    -
    - 
    # Using Newton's method
-from random import random, seed
-import numpy as np
-import autograd.numpy as np
-import matplotlib.pyplot as plt
-from autograd import grad
-
-def CostOLS(beta):
-    return (1.0/n)*np.sum((y-X @ beta)**2)
-
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
-
-X = np.c_[np.ones((n,1)), x]
-XT_X = X.T @ X
-beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
-print("Own inversion")
-print(beta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-# Note that here the Hessian does not depend on the parameters beta
-invH = np.linalg.pinv(H)
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
-
-beta = np.random.randn(2,1)
-Niterations = 5
-
-# define the gradient
-training_gradient = grad(CostOLS)
-
-for iter in range(Niterations):
-    gradients = training_gradient(beta)
-    beta -= invH @ gradients
-    print(iter,gradients[0],gradients[1])
-print("beta from own Newton code")
-print(beta)
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -









    Including Stochastic Gradient Descent with Autograd

    In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using autograd.

    @@ -2020,6 +2039,7 @@ delta = 1e-7
    +
    © 1999-2025, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license diff --git a/doc/pub/week37/html/week37.html b/doc/pub/week37/html/week37.html index 3187de778..73e5119f6 100644 --- a/doc/pub/week37/html/week37.html +++ b/doc/pub/week37/html/week37.html @@ -186,11 +186,14 @@ div.toc p,a { 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ('Overview video on Stochastic Gradient Descent', + ('Overview video on Stochastic Gradient Descent (SGD)', 2, None, - 'overview-video-on-stochastic-gradient-descent'), + 'overview-video-on-stochastic-gradient-descent-sgd'), ('Batches and mini-batches', 2, None, 'batches-and-mini-batches'), + ('Pros and cons', 2, None, 'pros-and-cons'), + ('Convergence rates', 2, None, 'convergence-rates'), + ('Accuracy', 2, None, 'accuracy'), ('Stochastic Gradient Descent (SGD)', 2, None, @@ -233,19 +236,30 @@ div.toc p,a { 2, None, 'algorithms-and-codes-for-adagrad-rmsprop-and-adam'), - ('Practical tips', 2, None, 'practical-tips'), - ('Sneaking in automatic differentiation using Autograd', + ('AdaGrad algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', 2, None, - 'sneaking-in-automatic-differentiation-using-autograd'), + 'adagrad-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('RMSProp algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'rmsprop-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('ADAM algorithm, taken from "Goodfellow et ' + 'al":"https://www.deeplearningbook.org/contents/optimization.html"', + 2, + None, + 'adam-algorithm-taken-from-goodfellow-et-al-https-www-deeplearningbook-org-contents-optimization-html'), + ('Practical tips', 2, None, 'practical-tips'), + ('Sneaking in auotmatic differentiation using Autograd', + 2, + None, + 'sneaking-in-auotmatic-differentiation-using-autograd'), ('Same code but now with momentum gradient descent', 2, None, 'same-code-but-now-with-momentum-gradient-descent'), - ("But none of these can compete with Newton's method", - 2, - None, - 'but-none-of-these-can-compete-with-newton-s-method'), ('Including Stochastic Gradient Descent with Autograd', 2, None, @@ -335,7 +349,7 @@ MathJax.Hub.Config({

      -
    1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at URL::https://www.deeplearningbook.org/contents/optimization.html"
      2. +
    2. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at https://www.deeplearningbook.org/contents/numerical.html and chapter 8.3-8.5 at https://www.deeplearningbook.org/contents/optimization.html
    3. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.
    4. Video on gradient descent at https://www.youtube.com/watch?v=sDv4f4s2SB8
    5. Video on Stochastic gradient descent at https://www.youtube.com/watch?v=vMh0zPT0tLI
      6. @@ -830,10 +844,16 @@ pyplot.show()









      -

      Overview video on Stochastic Gradient Descent

      +

      Overview video on Stochastic Gradient Descent (SGD)

      What is Stochastic Gradient Descent +

      There are several reasons for using stochastic gradient descent. Some of these are:

      +
        +
      1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.
        2. +
      2. Hopefully avoid Local Minima
        3. +
      3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset.
        4. +










      Batches and mini-batches

      @@ -849,6 +869,45 @@ an entire training set of several millions. This batch is then used to perform a parameter update.

      +









      +

      Pros and cons

      + +
        +
      1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.
        2. +
      2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.
        3. +
      3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient.
        4. +
      +









      +

      Convergence rates

      + +
        +
      1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.
        2. +
      2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration.
        3. +
      +









      +

      Accuracy

      + +

      In general, stochastic Gradient Descent is Less accurate than gradient +descent, as it calculates the gradient on single examples, which may +not accurately represent the overall dataset. Gradient Descent is +more accurate because it uses the average gradient calculated over the +entire dataset. +

      + +

      There are other disadvantages to using SGD. The main drawback is that +its convergence behaviour can be more erratic due to the random +sampling of individual training examples. This can lead to less +accurate results, as the algorithm may not converge to the true +minimum of the cost function. Additionally, the learning rate, which +determines the step size of each update to the model’s parameters, +must be carefully chosen to ensure convergence. +

      + +

      It is however the method of choice in deep learning algorithms where +SGD is often used in combination with other optimization techniques, +such as momentum or adaptive learning rates +

      +









      Stochastic Gradient Descent (SGD)

      @@ -1302,7 +1361,9 @@ $$ \end{align} $$ -

      One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of \( \gamma \).

      +

      One of the major advantages of NAG is that it allows for the use of a +larger learning rate than GDM for the same choice of \( \gamma \). +











      Second moment of the gradient

      @@ -1324,7 +1385,7 @@ the steep computational price of calculating or approximating Hessians.

      -

      Recently, a number of methods have been introduced that accomplish +

      During the last decade a number of methods have been introduced that accomplish this by tracking not only the gradient, but also the second moment of the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and ADAM. @@ -1419,7 +1480,28 @@ $$

      The algorithms we have implemented are well described in the text by Goodfellow, Bengio and Courville, chapter 8.

      -

      The codes which implement these algorithms are discussed below here.

      +

      The codes which implement these algorithms are discussed after our presentation of automatic differentiation.

      +

      AdaGrad algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +

      +

      RMSProp algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +

      +

      ADAM algorithm, taken from Goodfellow et al

      + +

      +
      +

      +
      +











      Practical tips

      @@ -1431,10 +1513,11 @@ $$
    6. Adaptive optimization methods don't always have good generalization. Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications.










      7. -

      Sneaking in automatic differentiation using Autograd

      +

      Sneaking in auotmatic differentiation using Autograd

      -

      We anticipate our discussions to come in connection with neural networks and automatic differentiation -by showing how we can use autograd for the cases above. Later we will replace autograd with JAX. +

      We conclude the part on optmization by showing how we can make codes +for linear regression and logistic regression using autograd. The +first example shows results with ordinary leats squares.

      @@ -1587,70 +1670,6 @@ delta_momentum = But none of these can compete with Newton's method - - - -
      -
      -
      -
      -
      - 
      # Using Newton's method
-from random import random, seed
-import numpy as np
-import autograd.numpy as np
-import matplotlib.pyplot as plt
-from autograd import grad
-
-def CostOLS(beta):
-    return (1.0/n)*np.sum((y-X @ beta)**2)
-
-n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x+np.random.randn(n,1)
-
-X = np.c_[np.ones((n,1)), x]
-XT_X = X.T @ X
-beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)
-print("Own inversion")
-print(beta_linreg)
-# Hessian matrix
-H = (2.0/n)* XT_X
-# Note that here the Hessian does not depend on the parameters beta
-invH = np.linalg.pinv(H)
-EigValues, EigVectors = np.linalg.eig(H)
-print(f"Eigenvalues of Hessian Matrix:{EigValues}")
-
-beta = np.random.randn(2,1)
-Niterations = 5
-
-# define the gradient
-training_gradient = grad(CostOLS)
-
-for iter in range(Niterations):
-    gradients = training_gradient(beta)
-    beta -= invH @ gradients
-    print(iter,gradients[0],gradients[1])
-print("beta from own Newton code")
-print(beta)
-
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -









      Including Stochastic Gradient Descent with Autograd

      In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using autograd.

      @@ -2097,6 +2116,7 @@ delta = 1e-7
    +
    © 1999-2025, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license diff --git a/doc/pub/week37/ipynb/ipynb-week37-src.tar.gz b/doc/pub/week37/ipynb/ipynb-week37-src.tar.gz index 3a4cefdc6..9dcd12eef 100644 Binary files a/doc/pub/week37/ipynb/ipynb-week37-src.tar.gz and b/doc/pub/week37/ipynb/ipynb-week37-src.tar.gz differ diff --git a/doc/pub/week37/ipynb/week37.ipynb b/doc/pub/week37/ipynb/week37.ipynb index c40ec2419..386e755d2 100644 --- a/doc/pub/week37/ipynb/week37.ipynb +++ b/doc/pub/week37/ipynb/week37.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "ebae0536", + "id": "26195c6c", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "c87edf63", + "id": "2f3d0562", "metadata": { "editable": true }, @@ -29,7 +29,7 @@ }, { "cell_type": "markdown", - "id": "9b6436c9", + "id": "3a04c093", "metadata": { "editable": true }, @@ -52,13 +52,13 @@ }, { "cell_type": "markdown", - "id": "01055296", + "id": "71cbeb43", "metadata": { "editable": true }, "source": [ "## Readings and Videos:\n", - "1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at and chapter 8.3-8.5 at URL::https://www.deeplearningbook.org/contents/optimization.html\"\n", + "1. Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at and chapter 8.3-8.5 at \n", "\n", "2. Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression.\n", "\n", @@ -69,7 +69,7 @@ }, { "cell_type": "markdown", - "id": "5b4c3f44", + "id": "eaa820fc", "metadata": { "editable": true }, @@ -79,7 +79,7 @@ }, { "cell_type": "markdown", - "id": "23544472", + "id": "ff06566a", "metadata": { "editable": true }, @@ -103,7 +103,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "d74adaf2", + "id": "059f5649", "metadata": { "collapsed": false, "editable": true @@ -116,7 +116,7 @@ }, { "cell_type": "markdown", - "id": "fa36c64d", + "id": "3cdf34a5", "metadata": { "editable": true }, @@ -127,7 +127,7 @@ }, { "cell_type": "markdown", - "id": "571c791d", + "id": "d66f2b53", "metadata": { "editable": true }, @@ -139,7 +139,7 @@ }, { "cell_type": "markdown", - "id": "12d3e9eb", + "id": "e0c22d3d", "metadata": { "editable": true }, @@ -149,7 +149,7 @@ }, { "cell_type": "markdown", - "id": "17610490", + "id": "8cb6c439", "metadata": { "editable": true }, @@ -161,7 +161,7 @@ }, { "cell_type": "markdown", - "id": "47fefa69", + "id": "45ba3ffb", "metadata": { "editable": true }, @@ -175,7 +175,7 @@ }, { "cell_type": "markdown", - "id": "52eb2d30", + "id": "9a8ffab0", "metadata": { "editable": true }, @@ -191,7 +191,7 @@ }, { "cell_type": "markdown", - "id": "7d89f32b", + "id": "9bb32152", "metadata": { "editable": true }, @@ -201,7 +201,7 @@ }, { "cell_type": "markdown", - "id": "5b0fb490", + "id": "7d386532", "metadata": { "editable": true }, @@ -213,7 +213,7 @@ }, { "cell_type": "markdown", - "id": "891f388a", + "id": "3eaf521e", "metadata": { "editable": true }, @@ -223,7 +223,7 @@ }, { "cell_type": "markdown", - "id": "d05ffb4b", + "id": "8e6191e9", "metadata": { "editable": true }, @@ -235,7 +235,7 @@ }, { "cell_type": "markdown", - "id": "f59689ea", + "id": "ac7e5a33", "metadata": { "editable": true }, @@ -249,7 +249,7 @@ }, { "cell_type": "markdown", - "id": "9a119fe4", + "id": "f5d64e82", "metadata": { "editable": true }, @@ -259,7 +259,7 @@ }, { "cell_type": "markdown", - "id": "ef5c109f", + "id": "65cef2b1", "metadata": { "editable": true }, @@ -270,7 +270,7 @@ }, { "cell_type": "markdown", - "id": "5412ed07", + "id": "dd1c97dd", "metadata": { "editable": true }, @@ -285,7 +285,7 @@ }, { "cell_type": "markdown", - "id": "33c5d9c9", + "id": "4771d71e", "metadata": { "editable": true }, @@ -295,7 +295,7 @@ }, { "cell_type": "markdown", - "id": "a8cf7d20", + "id": "8140a000", "metadata": { "editable": true }, @@ -307,7 +307,7 @@ }, { "cell_type": "markdown", - "id": "40f0daf3", + "id": "16bbcf2f", "metadata": { "editable": true }, @@ -319,7 +319,7 @@ }, { "cell_type": "markdown", - "id": "b09c9d54", + "id": "f2b8a434", "metadata": { "editable": true }, @@ -334,7 +334,7 @@ }, { "cell_type": "markdown", - "id": "081f1192", + "id": "4515d126", "metadata": { "editable": true }, @@ -347,7 +347,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "5842a0a2", + "id": "0fbb2309", "metadata": { "collapsed": false, "editable": true @@ -406,7 +406,7 @@ }, { "cell_type": "markdown", - "id": "923874ba", + "id": "9e083c31", "metadata": { "editable": true }, @@ -418,7 +418,7 @@ }, { "cell_type": "markdown", - "id": "442c5abb", + "id": "10a41983", "metadata": { "editable": true }, @@ -430,7 +430,7 @@ }, { "cell_type": "markdown", - "id": "743e05ca", + "id": "9e612405", "metadata": { "editable": true }, @@ -440,7 +440,7 @@ }, { "cell_type": "markdown", - "id": "27dfced9", + "id": "618d8dd0", "metadata": { "editable": true }, @@ -454,7 +454,7 @@ }, { "cell_type": "markdown", - "id": "3ff66697", + "id": "d0c88404", "metadata": { "editable": true }, @@ -464,7 +464,7 @@ }, { "cell_type": "markdown", - "id": "f63f80f9", + "id": "96f35638", "metadata": { "editable": true }, @@ -476,7 +476,7 @@ }, { "cell_type": "markdown", - "id": "dc6a4fff", + "id": "99d2a6dd", "metadata": { "editable": true }, @@ -487,7 +487,7 @@ }, { "cell_type": "markdown", - "id": "56a4b43d", + "id": "7f61638e", "metadata": { "editable": true }, @@ -502,7 +502,7 @@ }, { "cell_type": "markdown", - "id": "38c6aab3", + "id": "256e6480", "metadata": { "editable": true }, @@ -516,7 +516,7 @@ }, { "cell_type": "markdown", - "id": "a4f724c3", + "id": "3b9bd1b5", "metadata": { "editable": true }, @@ -527,7 +527,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "ae5d09dc", + "id": "d99082f7", "metadata": { "collapsed": false, "editable": true @@ -588,7 +588,7 @@ }, { "cell_type": "markdown", - "id": "184d283f", + "id": "65bea527", "metadata": { "editable": true }, @@ -610,7 +610,7 @@ }, { "cell_type": "markdown", - "id": "cfb021a5", + "id": "6d7105d1", "metadata": { "editable": true }, @@ -623,7 +623,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "a89ea0e9", + "id": "6b3f6111", "metadata": { "collapsed": false, "editable": true @@ -689,7 +689,7 @@ }, { "cell_type": "markdown", - "id": "4a29024a", + "id": "085e9053", "metadata": { "editable": true }, @@ -700,7 +700,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "9212126a", + "id": "4dc281e1", "metadata": { "collapsed": false, "editable": true @@ -774,19 +774,26 @@ }, { "cell_type": "markdown", - "id": "c12221ee", + "id": "095fdc5b", "metadata": { "editable": true }, "source": [ - "## Overview video on Stochastic Gradient Descent\n", + "## Overview video on Stochastic Gradient Descent (SGD)\n", "\n", - "[What is Stochastic Gradient Descent](https://www.youtube.com/watch?v=vMh0zPT0tLI&ab_channel=StatQuestwithJoshStarmer)" + "[What is Stochastic Gradient Descent](https://www.youtube.com/watch?v=vMh0zPT0tLI&ab_channel=StatQuestwithJoshStarmer)\n", + "There are several reasons for using stochastic gradient descent. Some of these are:\n", + "\n", + "1. Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence.\n", + "\n", + "2. Hopefully avoid Local Minima\n", + "\n", + "3. Memory Usage: Requires less memory compared to computing gradients for the entire dataset." ] }, { "cell_type": "markdown", - "id": "89e21421", + "id": "c1acf305", "metadata": { "editable": true }, @@ -807,7 +814,65 @@ }, { "cell_type": "markdown", - "id": "b3d6706b", + "id": "bb9ccd97", + "metadata": { + "editable": true + }, + "source": [ + "## Pros and cons\n", + "\n", + "1. Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases.\n", + "\n", + "2. Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples.\n", + "\n", + "3. Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient." + ] + }, + { + "cell_type": "markdown", + "id": "f5aaa59d", + "metadata": { + "editable": true + }, + "source": [ + "## Convergence rates\n", + "\n", + "1. Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration.\n", + "\n", + "2. Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration." + ] + }, + { + "cell_type": "markdown", + "id": "b6e06eb4", + "metadata": { + "editable": true + }, + "source": [ + "## Accuracy\n", + "\n", + "In general, stochastic Gradient Descent is Less accurate than gradient\n", + "descent, as it calculates the gradient on single examples, which may\n", + "not accurately represent the overall dataset. Gradient Descent is\n", + "more accurate because it uses the average gradient calculated over the\n", + "entire dataset.\n", + "\n", + "There are other disadvantages to using SGD. The main drawback is that\n", + "its convergence behaviour can be more erratic due to the random\n", + "sampling of individual training examples. This can lead to less\n", + "accurate results, as the algorithm may not converge to the true\n", + "minimum of the cost function. Additionally, the learning rate, which\n", + "determines the step size of each update to the model’s parameters,\n", + "must be carefully chosen to ensure convergence.\n", + "\n", + "It is however the method of choice in deep learning algorithms where\n", + "SGD is often used in combination with other optimization techniques,\n", + "such as momentum or adaptive learning rates" + ] + }, + { + "cell_type": "markdown", + "id": "e6de9732", "metadata": { "editable": true }, @@ -839,7 +904,7 @@ }, { "cell_type": "markdown", - "id": "f2900e4e", + "id": "5d6f2336", "metadata": { "editable": true }, @@ -856,7 +921,7 @@ }, { "cell_type": "markdown", - "id": "b6745dec", + "id": "9c24946f", "metadata": { "editable": true }, @@ -869,7 +934,7 @@ }, { "cell_type": "markdown", - "id": "b6f524f1", + "id": "15ae46aa", "metadata": { "editable": true }, @@ -882,7 +947,7 @@ }, { "cell_type": "markdown", - "id": "db7c028a", + "id": "eba44d5b", "metadata": { "editable": true }, @@ -895,7 +960,7 @@ }, { "cell_type": "markdown", - "id": "813b7f85", + "id": "fa5a0e9f", "metadata": { "editable": true }, @@ -909,7 +974,7 @@ }, { "cell_type": "markdown", - "id": "a562fa9a", + "id": "86e67dff", "metadata": { "editable": true }, @@ -931,7 +996,7 @@ }, { "cell_type": "markdown", - "id": "75d84b18", + "id": "1ffda2a5", "metadata": { "editable": true }, @@ -946,7 +1011,7 @@ }, { "cell_type": "markdown", - "id": "0cf6f343", + "id": "dd9589cf", "metadata": { "editable": true }, @@ -958,7 +1023,7 @@ }, { "cell_type": "markdown", - "id": "44f85de2", + "id": "bccd17a0", "metadata": { "editable": true }, @@ -971,7 +1036,7 @@ }, { "cell_type": "markdown", - "id": "92851949", + "id": "fa4254bc", "metadata": { "editable": true }, @@ -985,7 +1050,7 @@ }, { "cell_type": "markdown", - "id": "21d2691e", + "id": "d6134b3d", "metadata": { "editable": true }, @@ -996,7 +1061,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "6536b711", + "id": "46bbb72f", "metadata": { "collapsed": false, "editable": true @@ -1021,7 +1086,7 @@ }, { "cell_type": "markdown", - "id": "4005ce6c", + "id": "f354fc55", "metadata": { "editable": true }, @@ -1037,7 +1102,7 @@ }, { "cell_type": "markdown", - "id": "adec9808", + "id": "3ff93d52", "metadata": { "editable": true }, @@ -1058,7 +1123,7 @@ }, { "cell_type": "markdown", - "id": "deecc226", + "id": "3010708d", "metadata": { "editable": true }, @@ -1078,7 +1143,7 @@ }, { "cell_type": "markdown", - "id": "73685769", + "id": "0052d791", "metadata": { "editable": true }, @@ -1097,7 +1162,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "6484bffa", + "id": "aaa345b9", "metadata": { "collapsed": false, "editable": true @@ -1132,7 +1197,7 @@ }, { "cell_type": "markdown", - "id": "2438f642", + "id": "851c6b4d", "metadata": { "editable": true }, @@ -1145,7 +1210,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "8d624cf1", + "id": "664d6772", "metadata": { "collapsed": false, "editable": true @@ -1222,7 +1287,7 @@ }, { "cell_type": "markdown", - "id": "6e9eb916", + "id": "6905c3ba", "metadata": { "editable": true }, @@ -1237,7 +1302,7 @@ }, { "cell_type": "markdown", - "id": "c60a0137", + "id": "6f7347c9", "metadata": { "editable": true }, @@ -1252,7 +1317,7 @@ }, { "cell_type": "markdown", - "id": "2d8174d4", + "id": "7a8fac99", "metadata": { "editable": true }, @@ -1264,7 +1329,7 @@ }, { "cell_type": "markdown", - "id": "afc41240", + "id": "61c534ef", "metadata": { "editable": true }, @@ -1282,7 +1347,7 @@ }, { "cell_type": "markdown", - "id": "e3c96a9f", + "id": "cbe325b2", "metadata": { "editable": true }, @@ -1301,7 +1366,7 @@ }, { "cell_type": "markdown", - "id": "74e0e345", + "id": "18c4f71f", "metadata": { "editable": true }, @@ -1313,7 +1378,7 @@ }, { "cell_type": "markdown", - "id": "288fcc66", + "id": "b84f0488", "metadata": { "editable": true }, @@ -1323,7 +1388,7 @@ }, { "cell_type": "markdown", - "id": "63b49aca", + "id": "c5772ef7", "metadata": { "editable": true }, @@ -1339,7 +1404,7 @@ }, { "cell_type": "markdown", - "id": "94b37496", + "id": "c86bbefb", "metadata": { "editable": true }, @@ -1351,7 +1416,7 @@ }, { "cell_type": "markdown", - "id": "3a439485", + "id": "88aea644", "metadata": { "editable": true }, @@ -1361,7 +1426,7 @@ }, { "cell_type": "markdown", - "id": "ff92e318", + "id": "8d0f09dd", "metadata": { "editable": true }, @@ -1373,7 +1438,7 @@ }, { "cell_type": "markdown", - "id": "142e0f95", + "id": "3c0f263a", "metadata": { "editable": true }, @@ -1383,7 +1448,7 @@ }, { "cell_type": "markdown", - "id": "88d4ce9a", + "id": "e022e30b", "metadata": { "editable": true }, @@ -1395,7 +1460,7 @@ }, { "cell_type": "markdown", - "id": "5c662618", + "id": "e29f7ed3", "metadata": { "editable": true }, @@ -1411,7 +1476,7 @@ }, { "cell_type": "markdown", - "id": "2c4a2172", + "id": "745ac9fb", "metadata": { "editable": true }, @@ -1423,7 +1488,7 @@ }, { "cell_type": "markdown", - "id": "630c69f8", + "id": "e7608c5b", "metadata": { "editable": true }, @@ -1456,7 +1521,7 @@ }, { "cell_type": "markdown", - "id": "7fc49052", + "id": "3af69259", "metadata": { "editable": true }, @@ -1468,7 +1533,7 @@ }, { "cell_type": "markdown", - "id": "800856ba", + "id": "fe7f114b", "metadata": { "editable": true }, @@ -1486,17 +1551,18 @@ }, { "cell_type": "markdown", - "id": "154d4907", + "id": "aac7d523", "metadata": { "editable": true }, "source": [ - "One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of $\\gamma$." + "One of the major advantages of NAG is that it allows for the use of a\n", + "larger learning rate than GDM for the same choice of $\\gamma$." ] }, { "cell_type": "markdown", - "id": "17557243", + "id": "0f6129f0", "metadata": { "editable": true }, @@ -1519,7 +1585,7 @@ "the steep computational price of calculating or approximating\n", "Hessians.\n", "\n", - "Recently, a number of methods have been introduced that accomplish\n", + "During the last decade a number of methods have been introduced that accomplish\n", "this by tracking not only the gradient, but also the second moment of\n", "the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and\n", "[ADAM](https://arxiv.org/abs/1412.6980)." @@ -1527,7 +1593,7 @@ }, { "cell_type": "markdown", - "id": "f3d09840", + "id": "4ee6d23c", "metadata": { "editable": true }, @@ -1542,7 +1608,7 @@ }, { "cell_type": "markdown", - "id": "175455a1", + "id": "410f1144", "metadata": { "editable": true }, @@ -1560,7 +1626,7 @@ }, { "cell_type": "markdown", - "id": "cf1e9538", + "id": "b3336df7", "metadata": { "editable": true }, @@ -1572,7 +1638,7 @@ }, { "cell_type": "markdown", - "id": "da9b6589", + "id": "a50a2257", "metadata": { "editable": true }, @@ -1584,7 +1650,7 @@ }, { "cell_type": "markdown", - "id": "cbc7a25f", + "id": "1fb1ab6a", "metadata": { "editable": true }, @@ -1602,7 +1668,7 @@ }, { "cell_type": "markdown", - "id": "e8064f4a", + "id": "9e04c2f8", "metadata": { "editable": true }, @@ -1631,7 +1697,7 @@ }, { "cell_type": "markdown", - "id": "12912817", + "id": "9450e0f8", "metadata": { "editable": true }, @@ -1649,7 +1715,7 @@ }, { "cell_type": "markdown", - "id": "7864ca89", + "id": "0ec4afd8", "metadata": { "editable": true }, @@ -1661,7 +1727,7 @@ }, { "cell_type": "markdown", - "id": "02a5d4ef", + "id": "7d09f8b8", "metadata": { "editable": true }, @@ -1673,7 +1739,7 @@ }, { "cell_type": "markdown", - "id": "327a1f34", + "id": "219826a3", "metadata": { "editable": true }, @@ -1685,7 +1751,7 @@ }, { "cell_type": "markdown", - "id": "c5ac526b", + "id": "d3f0e5c5", "metadata": { "editable": true }, @@ -1697,7 +1763,7 @@ }, { "cell_type": "markdown", - "id": "e6a33564", + "id": "46fcf54e", "metadata": { "editable": true }, @@ -1709,7 +1775,7 @@ }, { "cell_type": "markdown", - "id": "4ffe0088", + "id": "9f3f1e9c", "metadata": { "editable": true }, @@ -1726,7 +1792,7 @@ }, { "cell_type": "markdown", - "id": "34eb03fb", + "id": "e4d859d2", "metadata": { "editable": true }, @@ -1745,7 +1811,7 @@ }, { "cell_type": "markdown", - "id": "18f62c05", + "id": "d9fb412a", "metadata": { "editable": true }, @@ -1757,7 +1823,7 @@ }, { "cell_type": "markdown", - "id": "5539d5a5", + "id": "a4ed4fe4", "metadata": { "editable": true }, @@ -1766,12 +1832,60 @@ "\n", "The algorithms we have implemented are well described in the text by [Goodfellow, Bengio and Courville, chapter 8](https://www.deeplearningbook.org/contents/optimization.html).\n", "\n", - "The codes which implement these algorithms are discussed below here." + "The codes which implement these algorithms are discussed after our presentation of automatic differentiation." ] }, { "cell_type": "markdown", - "id": "ae04cfe0", + "id": "c8b687aa", + "metadata": { + "editable": true + }, + "source": [ + "## AdaGrad algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

    Figure 1:

    \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "d42508aa", + "metadata": { + "editable": true + }, + "source": [ + "## RMSProp algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

    Figure 1:

    \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "54d1c89f", + "metadata": { + "editable": true + }, + "source": [ + "## ADAM algorithm, taken from [Goodfellow et al](https://www.deeplearningbook.org/contents/optimization.html)\n", + "\n", + "\n", + "\n", + "\n", + "

    Figure 1:

    \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "90c96a47", "metadata": { "editable": true }, @@ -1789,21 +1903,22 @@ }, { "cell_type": "markdown", - "id": "3ffaaf9a", + "id": "79eb9b5f", "metadata": { "editable": true }, "source": [ - "## Sneaking in automatic differentiation using Autograd\n", + "## Sneaking in auotmatic differentiation using Autograd\n", "\n", - "We anticipate our discussions to come in connection with neural networks and automatic differentiation\n", - "by showing how we can use **autograd** for the cases above. Later we will replace **autograd** with **JAX**." + "We conclude the part on optmization by showing how we can make codes\n", + "for linear regression and logistic regression using **autograd**. The\n", + "first example shows results with ordinary leats squares." ] }, { "cell_type": "code", "execution_count": 9, - "id": "51503c7d", + "id": "8749e605", "metadata": { "collapsed": false, "editable": true @@ -1863,7 +1978,7 @@ }, { "cell_type": "markdown", - "id": "67d85da7", + "id": "4f7cb53b", "metadata": { "editable": true }, @@ -1874,7 +1989,7 @@ { "cell_type": "code", "execution_count": 10, - "id": "586da287", + "id": "2ca61a91", "metadata": { "collapsed": false, "editable": true @@ -1938,67 +2053,7 @@ }, { "cell_type": "markdown", - "id": "b0d53639", - "metadata": { - "editable": true - }, - "source": [ - "## But none of these can compete with Newton's method" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "81089279", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "# Using Newton's method\n", - "from random import random, seed\n", - "import numpy as np\n", - "import autograd.numpy as np\n", - "import matplotlib.pyplot as plt\n", - "from autograd import grad\n", - "\n", - "def CostOLS(beta):\n", - " return (1.0/n)*np.sum((y-X @ beta)**2)\n", - "\n", - "n = 100\n", - "x = 2*np.random.rand(n,1)\n", - "y = 4+3*x+np.random.randn(n,1)\n", - "\n", - "X = np.c_[np.ones((n,1)), x]\n", - "XT_X = X.T @ X\n", - "beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y)\n", - "print(\"Own inversion\")\n", - "print(beta_linreg)\n", - "# Hessian matrix\n", - "H = (2.0/n)* XT_X\n", - "# Note that here the Hessian does not depend on the parameters beta\n", - "invH = np.linalg.pinv(H)\n", - "EigValues, EigVectors = np.linalg.eig(H)\n", - "print(f\"Eigenvalues of Hessian Matrix:{EigValues}\")\n", - "\n", - "beta = np.random.randn(2,1)\n", - "Niterations = 5\n", - "\n", - "# define the gradient\n", - "training_gradient = grad(CostOLS)\n", - "\n", - "for iter in range(Niterations):\n", - " gradients = training_gradient(beta)\n", - " beta -= invH @ gradients\n", - " print(iter,gradients[0],gradients[1])\n", - "print(\"beta from own Newton code\")\n", - "print(beta)" - ] - }, - { - "cell_type": "markdown", - "id": "e302b6c8", + "id": "5ab2acb5", "metadata": { "editable": true }, @@ -2009,8 +2064,8 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "88efb71c", + "execution_count": 11, + "id": "47c2c7fd", "metadata": { "collapsed": false, "editable": true @@ -2094,7 +2149,7 @@ }, { "cell_type": "markdown", - "id": "a3e14887", + "id": "fd8262d3", "metadata": { "editable": true }, @@ -2104,8 +2159,8 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "607e3bc9", + "execution_count": 12, + "id": "38b4d9b4", "metadata": { "collapsed": false, "editable": true @@ -2183,7 +2238,7 @@ }, { "cell_type": "markdown", - "id": "d96e1c2c", + "id": "5aa8d23f", "metadata": { "editable": true }, @@ -2193,8 +2248,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "id": "6cbe854b", + "execution_count": 13, + "id": "ad237e29", "metadata": { "collapsed": false, "editable": true @@ -2253,7 +2308,7 @@ }, { "cell_type": "markdown", - "id": "0ed15b7b", + "id": "d7b69389", "metadata": { "editable": true }, @@ -2263,7 +2318,7 @@ }, { "cell_type": "markdown", - "id": "d97ab879", + "id": "b3d0a76d", "metadata": { "editable": true }, @@ -2273,8 +2328,8 @@ }, { "cell_type": "code", - "execution_count": 15, - "id": "c58a19a2", + "execution_count": 14, + "id": "62e76ebb", "metadata": { "collapsed": false, "editable": true @@ -2339,7 +2394,7 @@ }, { "cell_type": "markdown", - "id": "d7106089", + "id": "296f1151", "metadata": { "editable": true }, @@ -2349,8 +2404,8 @@ }, { "cell_type": "code", - "execution_count": 16, - "id": "d3b9cdb9", + "execution_count": 15, + "id": "004c5081", "metadata": { "collapsed": false, "editable": true @@ -2420,7 +2475,7 @@ }, { "cell_type": "markdown", - "id": "e6a7abff", + "id": "d51c3400", "metadata": { "editable": true }, diff --git a/doc/src/week37/week37.do.txt b/doc/src/week37/week37.do.txt index fc160fb0e..20b5caa14 100644 --- a/doc/src/week37/week37.do.txt +++ b/doc/src/week37/week37.do.txt @@ -21,7 +21,7 @@ o More advanced updates of the learning rate: ADAgrad, RMSprop and ADAM !split ===== Readings and Videos: ===== !bblock -o Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at URL:"https://www.deeplearningbook.org/contents/numerical.html" and chapter 8.3-8.5 at URL::https://www.deeplearningbook.org/contents/optimization.html" +o Recommended: Goodfellow et al, Deep Learning, introduction to gradient descent, see sections 4.3-4.5 at URL:"https://www.deeplearningbook.org/contents/numerical.html" and chapter 8.3-8.5 at URL:"https://www.deeplearningbook.org/contents/optimization.html" o Rashcka et al, pages 37-44 and pages 278-283 with focus on linear regression. o Video on gradient descent at URL:"https://www.youtube.com/watch?v=sDv4f4s2SB8" o Video on Stochastic gradient descent at URL:"https://www.youtube.com/watch?v=vMh0zPT0tLI" @@ -452,13 +452,15 @@ pyplot.plot(solutions, scores, '.-', color='red') pyplot.show() !ec - - !split -===== Overview video on Stochastic Gradient Descent ===== +===== Overview video on Stochastic Gradient Descent (SGD) ===== "What is Stochastic Gradient Descent":"https://www.youtube.com/watch?v=vMh0zPT0tLI&ab_channel=StatQuestwithJoshStarmer" +There are several reasons for using stochastic gradient descent. Some of these are: +o Efficiency: Updates weights more frequently using a single or a small batch of samples, which speeds up convergence. +o Hopefully avoid Local Minima +o Memory Usage: Requires less memory compared to computing gradients for the entire dataset. !split ===== Batches and mini-batches ===== @@ -474,6 +476,44 @@ gradient over batches of the training data. For example, a typical batch could c an entire training set of several millions. This batch is then used to perform a parameter update. + +!split +===== Pros and cons ===== + + +o Speed: SGD is faster than gradient descent because it uses only one training example per iteration, whereas gradient descent requires the entire dataset. This speed advantage becomes more significant as the size of the dataset increases. +o Convergence: Gradient descent has a more predictable convergence behaviour because it uses the average gradient of the entire dataset. In contrast, SGD’s convergence behaviour can be more erratic due to its random sampling of individual training examples. +o Memory: Gradient descent requires more memory than SGD because it must store the entire dataset for each iteration. SGD only needs to store the current training example, making it more memory-efficient. + + +!split +===== Convergence rates ===== + +o Stochastic Gradient Descent has a faster convergence rate due to the use of single training examples in each iteration. +o Gradient Descent as a slower convergence rate, as it uses the entire dataset for each iteration. + +!split +===== Accuracy ===== + +In general, stochastic Gradient Descent is Less accurate than gradient +descent, as it calculates the gradient on single examples, which may +not accurately represent the overall dataset. Gradient Descent is +more accurate because it uses the average gradient calculated over the +entire dataset. + + +There are other disadvantages to using SGD. The main drawback is that +its convergence behaviour can be more erratic due to the random +sampling of individual training examples. This can lead to less +accurate results, as the algorithm may not converge to the true +minimum of the cost function. Additionally, the learning rate, which +determines the step size of each update to the model’s parameters, +must be carefully chosen to ensure convergence. + +It is however the method of choice in deep learning algorithms where +SGD is often used in combination with other optimization techniques, +such as momentum or adaptive learning rates + !split ===== Stochastic Gradient Descent (SGD) ===== @@ -670,10 +710,6 @@ print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j)) !ec - - - - !split ===== Code with a Number of Minibatches which varies ===== @@ -870,8 +906,8 @@ our current momentum, $\nabla_\theta E(\boldsymbol{\theta}_t +\gamma \end{align} !et -One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of $\gamma$. - +One of the major advantages of NAG is that it allows for the use of a +larger learning rate than GDM for the same choice of $\gamma$. !split ===== Second moment of the gradient ===== @@ -893,7 +929,7 @@ adaptively change the step size to match the landscape without paying the steep computational price of calculating or approximating Hessians. -Recently, a number of methods have been introduced that accomplish +During the last decade a number of methods have been introduced that accomplish this by tracking not only the gradient, but also the second moment of the gradient. These methods include AdaGrad, AdaDelta, Root Mean Squared Propagation (RMS-Prop), and "ADAM":"https://arxiv.org/abs/1412.6980". @@ -980,7 +1016,26 @@ update rule for this parameter is given by The algorithms we have implemented are well described in the text by "Goodfellow, Bengio and Courville, chapter 8":"https://www.deeplearningbook.org/contents/optimization.html". -The codes which implement these algorithms are discussed below here. +The codes which implement these algorithms are discussed after our presentation of automatic differentiation. + + +===== AdaGrad algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html" ===== + +FIGURE: [figures/adagrad.png, width=600 frac=0.8] + + +===== RMSProp algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html" ===== + +FIGURE: [figures/rmsprop.png, width=600 frac=0.8] + + + +===== ADAM algorithm, taken from "Goodfellow et al":"https://www.deeplearningbook.org/contents/optimization.html" ===== + +FIGURE: [figures/adam.png, width=600 frac=0.8] + + + !split @@ -995,15 +1050,12 @@ The codes which implement these algorithms are discussed below here. * _Adaptive optimization methods don't always have good generalization._ Recent studies have shown that adaptive methods such as ADAM, RMSPorp, and AdaGrad tend to have poor generalization compared to SGD or SGD with momentum, particularly in the high-dimensional limit (i.e. the number of parameters exceeds the number of data points). Although it is not clear at this stage why these methods perform so well in training deep neural networks, simpler procedures like properly-tuned SGD may work as well or better in these applications. - - - !split -===== Sneaking in automatic differentiation using Autograd ===== - -We anticipate our discussions to come in connection with neural networks and automatic differentiation -by showing how we can use _autograd_ for the cases above. Later we will replace _autograd_ with _JAX_. +===== Sneaking in auotmatic differentiation using Autograd ===== +We conclude the part on optmization by showing how we can make codes +for linear regression and logistic regression using _autograd_. The +first example shows results with ordinary leats squares. !bc pycod # Using Autograd to calculate gradients for OLS @@ -1118,51 +1170,6 @@ print(theta) !ec -!split -===== But none of these can compete with Newton's method ===== - -!bc pycod -# Using Newton's method -from random import random, seed -import numpy as np -import autograd.numpy as np -import matplotlib.pyplot as plt -from autograd import grad - -def CostOLS(beta): - return (1.0/n)*np.sum((y-X @ beta)**2) - -n = 100 -x = 2*np.random.rand(n,1) -y = 4+3*x+np.random.randn(n,1) - -X = np.c_[np.ones((n,1)), x] -XT_X = X.T @ X -beta_linreg = np.linalg.pinv(XT_X) @ (X.T @ y) -print("Own inversion") -print(beta_linreg) -# Hessian matrix -H = (2.0/n)* XT_X -# Note that here the Hessian does not depend on the parameters beta -invH = np.linalg.pinv(H) -EigValues, EigVectors = np.linalg.eig(H) -print(f"Eigenvalues of Hessian Matrix:{EigValues}") - -beta = np.random.randn(2,1) -Niterations = 5 - -# define the gradient -training_gradient = grad(CostOLS) - -for iter in range(Niterations): - gradients = training_gradient(beta) - beta -= invH @ gradients - print(iter,gradients[0],gradients[1]) -print("beta from own Newton code") -print(beta) -!ec - - !split ===== Including Stochastic Gradient Descent with Autograd ===== In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using _autograd_. @@ -1512,3 +1519,11 @@ o Work on project 1 # * "Video of exercise sessions week 37":"https://youtu.be/bK4AEcTu-oM" * For more discussions of Ridge regression and calculation of averages, "Wessel van Wieringen's":"https://arxiv.org/abs/1509.09169" article is highly recommended. !eblock + + + + + + + +