fixed some typos in GD
This commit is contained in:
mhjensen
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -297,7 +297,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 25, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Sep 26, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,22 +278,17 @@ MathJax.Hub.Config({
|
||||
<a name="part0027"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec26" class="anchor">Stochastic Gradient Descent </h2>
|
||||
<h2 id="___sec26" class="anchor">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<p>
|
||||
Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
the shortcomings of the Gradient descent method discussed above.
|
||||
<ul>
|
||||
<li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
|
||||
<p>
|
||||
The underlying idea of SGD comes from the observation that the cost
|
||||
function, which we want to minimize, can almost always be written as a
|
||||
sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \),
|
||||
$$
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,22 +278,20 @@ MathJax.Hub.Config({
|
||||
<a name="part0028"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec27" class="anchor">Computation of gradients </h2>
|
||||
<h2 id="___sec27" class="anchor">Stochastic Gradient Descent </h2>
|
||||
|
||||
<p>
|
||||
This in turn means that the gradient can be
|
||||
computed as a sum over \( i \)-gradients
|
||||
$$
|
||||
\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
the shortcomings of the Gradient descent method discussed above.
|
||||
|
||||
<p>
|
||||
Stochasticity/randomness is introduced by only taking the
|
||||
gradient on a subset of the data called minibatches. If there are \( n \)
|
||||
data points and the size of each minibatch is \( M \), there will be \( n/M \)
|
||||
minibatches. We denote these minibatches by \( B_k \) where
|
||||
\( k=1,\cdots,n/M \).
|
||||
The underlying idea of SGD comes from the observation that the cost
|
||||
function, which we want to minimize, can almost always be written as a
|
||||
sum over \( n \) data points \( \{\mathbf{x}_i\}_{i=1}^n \),
|
||||
$$
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,27 +278,23 @@ MathJax.Hub.Config({
|
||||
<a name="part0029"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec28" class="anchor">SGD example </h2>
|
||||
As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 =
|
||||
(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you
|
||||
have only a single batch with all data points and on the other extreme,
|
||||
you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e
|
||||
\( B_k = \mathbf{x}_k \).
|
||||
<h2 id="___sec28" class="anchor">Computation of gradients </h2>
|
||||
|
||||
<p>
|
||||
The idea is now to approximate the gradient by replacing the sum over
|
||||
all data points with a sum over the data points in one the minibatches
|
||||
picked at random in each gradient descent step
|
||||
This in turn means that the gradient can be
|
||||
computed as a sum over \( i \)-gradients
|
||||
$$
|
||||
\nabla_{\beta}
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta
|
||||
c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}).
|
||||
$$
|
||||
|
||||
<p>
|
||||
Stochasticity/randomness is introduced by only taking the
|
||||
gradient on a subset of the data called minibatches. If there are \( n \)
|
||||
data points and the size of each minibatch is \( M \), there will be \( n/M \)
|
||||
minibatches. We denote these minibatches by \( B_k \) where
|
||||
\( k=1,\cdots,n/M \).
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,22 +278,27 @@ MathJax.Hub.Config({
|
||||
<a name="part0030"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec29" class="anchor">The gradient step </h2>
|
||||
<h2 id="___sec29" class="anchor">SGD example </h2>
|
||||
As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
\( B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 =
|
||||
(\mathbf{x}_9,\mathbf{x}_{10}) \). Note that if you choose \( M=1 \) you
|
||||
have only a single batch with all data points and on the other extreme,
|
||||
you may choose \( M=n \) resulting in a minibatch for each datapoint, i.e
|
||||
\( B_k = \mathbf{x}_k \).
|
||||
|
||||
<p>
|
||||
Thus a gradient descent step now looks like
|
||||
The idea is now to approximate the gradient by replacing the sum over
|
||||
all data points with a sum over the data points in one the minibatches
|
||||
picked at random in each gradient descent step
|
||||
$$
|
||||
\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta})
|
||||
\nabla_{\beta}
|
||||
C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta
|
||||
c_i(\mathbf{x}_i, \mathbf{\beta}).
|
||||
$$
|
||||
|
||||
<p>
|
||||
where \( k \) is picked at random with equal
|
||||
probability from \( [1,n/M] \). An iteration over the number of
|
||||
minibathces (n/M) is commonly referred to as an epoch. Thus it is
|
||||
typical to choose a number of epochs and for each epoch iterate over
|
||||
the number of minibatches, as exemplified in the code below.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,34 +278,21 @@ MathJax.Hub.Config({
|
||||
<a name="part0031"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec30" class="anchor">Simple example code </h2>
|
||||
<h2 id="___sec30" class="anchor">The gradient step </h2>
|
||||
|
||||
<p>
|
||||
Thus a gradient descent step now looks like
|
||||
$$
|
||||
\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i,
|
||||
\mathbf{\beta})
|
||||
$$
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
|
||||
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">10</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
|
||||
|
||||
j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,n_epochs<span style="color: #666666">+1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
k <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m) <span style="color: #408080; font-style: italic">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute new suggestion for </span>
|
||||
j <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
Taking the gradient only on a subset of the data has two important
|
||||
benefits. First, it introduces randomness which decreases the chance
|
||||
that our opmization scheme gets stuck in a local minima. Second, if
|
||||
the size of the minibatches are small relative to the number of
|
||||
datapoints (\( M < n \)), the computation of the gradient is much
|
||||
cheaper since we sum over the datapoints in the \( k-th \) minibatch and not
|
||||
all \( n \) datapoints.
|
||||
where \( k \) is picked at random with equal
|
||||
probability from \( [1,n/M] \). An iteration over the number of
|
||||
minibathces (n/M) is commonly referred to as an epoch. Thus it is
|
||||
typical to choose a number of epochs and for each epoch iterate over
|
||||
the number of minibatches, as exemplified in the code below.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,19 +278,34 @@ MathJax.Hub.Config({
|
||||
<a name="part0032"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec31" class="anchor">When do we stop? </h2>
|
||||
<h2 id="___sec31" class="anchor">Simple example code </h2>
|
||||
|
||||
<p>
|
||||
A natural question is when do we stop the search for a new minimum?
|
||||
One possibility is to compute the full gradient after a given number
|
||||
of epochs and check if the norm of the gradient is smaller than some
|
||||
threshold and stop if true. However, the condition that the gradient
|
||||
is zero is valid also for local minima, so this would only tell us
|
||||
that we are close to a local/global minimum. However, we could also
|
||||
evaluate the cost function at this point, store the result and
|
||||
continue the search. If the test kicks in at a later stage we can
|
||||
compare the values of the cost function and keep the \( \beta \) that
|
||||
gave the lowest value.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
|
||||
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">10</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
|
||||
|
||||
j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,n_epochs<span style="color: #666666">+1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
k <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m) <span style="color: #408080; font-style: italic">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute new suggestion for </span>
|
||||
j <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
Taking the gradient only on a subset of the data has two important
|
||||
benefits. First, it introduces randomness which decreases the chance
|
||||
that our opmization scheme gets stuck in a local minima. Second, if
|
||||
the size of the minibatches are small relative to the number of
|
||||
datapoints (\( M < n \)), the computation of the gradient is much
|
||||
cheaper since we sum over the datapoints in the \( k-th \) minibatch and not
|
||||
all \( n \) datapoints.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,51 +278,20 @@ MathJax.Hub.Config({
|
||||
<a name="part0033"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec32" class="anchor">Slightly different approach </h2>
|
||||
<h2 id="___sec32" class="anchor">When do we stop? </h2>
|
||||
|
||||
<p>
|
||||
Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
A natural question is when do we stop the search for a new minimum?
|
||||
One possibility is to compute the full gradient after a given number
|
||||
of epochs and check if the norm of the gradient is smaller than some
|
||||
threshold and stop if true. However, the condition that the gradient
|
||||
is zero is valid also for local minima, so this would only tell us
|
||||
that we are close to a local/global minimum. However, we could also
|
||||
evaluate the cost function at this point, store the result and
|
||||
continue the search. If the test kicks in at a later stage we can
|
||||
compare the values of the cost function and keep the \( \beta \) that
|
||||
gave the lowest value.
|
||||
|
||||
<p>
|
||||
As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).
|
||||
|
||||
<p>
|
||||
In this way we can fix the number of epochs, compute \( \beta \) and
|
||||
evaluate the cost function at the end. Repeating the computation will
|
||||
give a different result since the scheme is random by design. Then we
|
||||
pick the final \( \beta \) that gives the lowest value of the cost
|
||||
function.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">step_length</span>(t,t0,t1):
|
||||
<span style="color: #008000; font-weight: bold">return</span> t0<span style="color: #666666">/</span>(t<span style="color: #666666">+</span>t1)
|
||||
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
|
||||
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">500</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
|
||||
t0 <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
|
||||
t1 <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
|
||||
gamma_j <span style="color: #666666">=</span> t0<span style="color: #666666">/</span>t1
|
||||
j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,n_epochs<span style="color: #666666">+1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
k <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m) <span style="color: #408080; font-style: italic">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute new suggestion for beta</span>
|
||||
t <span style="color: #666666">=</span> epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i
|
||||
gamma_j <span style="color: #666666">=</span> step_length(t,t0,t1)
|
||||
j <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"gamma_j after </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> epochs: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> (n_epochs,gamma_j))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,76 +278,50 @@ MathJax.Hub.Config({
|
||||
<a name="part0034"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec33" class="anchor">Program for stochastic gradient </h2>
|
||||
<h2 id="___sec33" class="anchor">Slightly different approach </h2>
|
||||
|
||||
<p>
|
||||
Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
number of epochs in such a way that it becomes very small after a
|
||||
reasonable time such that we do not move at all.
|
||||
|
||||
<p>
|
||||
As an example, let \( e = 0,1,2,3,\cdots \) denote the current epoch and let \( t_0, t_1 > 0 \) be two fixed numbers. Furthermore, let \( t = e \cdot m + i \) where \( m \) is the number of minibatches and \( i=0,\cdots,m-1 \). Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length \( \gamma_j (0; t_0, t_1) = t_0/t_1 \) which decays in <em>time</em> \( t \).
|
||||
|
||||
<p>
|
||||
In this way we can fix the number of epochs, compute \( \beta \) and
|
||||
evaluate the cost function at the end. Repeating the computation will
|
||||
give a different result since the scheme is random by design. Then we
|
||||
pick the final \( \beta \) that gives the lowest value of the cost
|
||||
function.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)), x]
|
||||
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Own inversion"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta_linreg)
|
||||
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000">None</span>, eta0<span style="color: #666666">=0.1</span>)
|
||||
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"sgdreg from scikit"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(sgdreg<span style="color: #666666">.</span>intercept_, sgdreg<span style="color: #666666">.</span>coef_)
|
||||
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
|
||||
Niterations <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>dot(theta)<span style="color: #666666">-</span>y)
|
||||
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"theta frm own gd"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta)
|
||||
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
||||
xbnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
|
||||
ypredict <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta)
|
||||
ypredict2 <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta_linreg)
|
||||
|
||||
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</span>, <span style="color: #666666">50</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">learning_schedule</span>(t):
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">step_length</span>(t,t0,t1):
|
||||
<span style="color: #008000; font-weight: bold">return</span> t0<span style="color: #666666">/</span>(t<span style="color: #666666">+</span>t1)
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
n <span style="color: #666666">=</span> <span style="color: #666666">100</span> <span style="color: #408080; font-style: italic">#100 datapoints </span>
|
||||
M <span style="color: #666666">=</span> <span style="color: #666666">5</span> <span style="color: #408080; font-style: italic">#size of each minibatch</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #008000">int</span>(n<span style="color: #666666">/</span>M) <span style="color: #408080; font-style: italic">#number of minibatches</span>
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">500</span> <span style="color: #408080; font-style: italic">#number of epochs</span>
|
||||
t0 <span style="color: #666666">=</span> <span style="color: #666666">1.0</span>
|
||||
t1 <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
|
||||
gamma_j <span style="color: #666666">=</span> t0<span style="color: #666666">/</span>t1
|
||||
j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,n_epochs<span style="color: #666666">+1</span>):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
|
||||
xi <span style="color: #666666">=</span> xb[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xi<span style="color: #666666">.</span>dot(theta)<span style="color: #666666">-</span>yi)
|
||||
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
|
||||
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"theta from own sdg"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta)
|
||||
k <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m) <span style="color: #408080; font-style: italic">#Pick the k-th minibatch at random</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute the gradient using the data in minibatch Bk</span>
|
||||
<span style="color: #408080; font-style: italic">#Compute new suggestion for beta</span>
|
||||
t <span style="color: #666666">=</span> epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i
|
||||
gamma_j <span style="color: #666666">=</span> step_length(t,t0,t1)
|
||||
j <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict2, <span style="color: #BA2121">"b-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Random numbers '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"gamma_j after </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121"> epochs: </span><span style="color: #BB6688; font-weight: bold">%g</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> (n_epochs,gamma_j))
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,8 +278,77 @@ MathJax.Hub.Config({
|
||||
<a name="part0035"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec34" class="anchor">Logistic Regression example </h2>
|
||||
<h2 id="___sec34" class="anchor">Program for stochastic gradient </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># Importing various packages</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">math</span> <span style="color: #008000; font-weight: bold">import</span> exp, sqrt
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
|
||||
|
||||
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)), x]
|
||||
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"Own inversion"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta_linreg)
|
||||
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000">None</span>, eta0<span style="color: #666666">=0.1</span>)
|
||||
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"sgdreg from scikit"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(sgdreg<span style="color: #666666">.</span>intercept_, sgdreg<span style="color: #666666">.</span>coef_)
|
||||
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>
|
||||
Niterations <span style="color: #666666">=</span> <span style="color: #666666">1000</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T @ ((xb @ theta)<span style="color: #666666">-</span>y)
|
||||
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"theta frm own gd"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta)
|
||||
|
||||
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
|
||||
xbnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
|
||||
ypredict <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta)
|
||||
ypredict2 <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta_linreg)
|
||||
|
||||
|
||||
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
t0, t1 <span style="color: #666666">=</span> <span style="color: #666666">5</span>, <span style="color: #666666">50</span>
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">learning_schedule</span>(t):
|
||||
<span style="color: #008000; font-weight: bold">return</span> t0<span style="color: #666666">/</span>(t<span style="color: #666666">+</span>t1)
|
||||
|
||||
theta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
|
||||
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
|
||||
xi <span style="color: #666666">=</span> xb[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T @ ((xi @ theta)<span style="color: #666666">-</span>yi)
|
||||
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
|
||||
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"theta from own sdg"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta)
|
||||
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict, <span style="color: #BA2121">"r-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(xnew, ypredict2, <span style="color: #BA2121">"b-"</span>)
|
||||
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">'ro'</span>)
|
||||
plt<span style="color: #666666">.</span>axis([<span style="color: #666666">0</span>,<span style="color: #666666">2.0</span>,<span style="color: #666666">0</span>, <span style="color: #666666">15.0</span>])
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">r'$x$'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">r'$y$'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">r'Random numbers '</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -278,17 +278,9 @@ MathJax.Hub.Config({
|
||||
<a name="part0036"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="___sec35" class="anchor">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<ul>
|
||||
<li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
<h2 id="___sec35" class="anchor">Logistic Regression example </h2>
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
|
||||
@@ -80,19 +80,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -216,16 +216,16 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs024.html#___sec23" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs025.html#___sec24" style="font-size: 80%;">Gradient descent and Ridge</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs026.html#___sec25" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs027.html#___sec26" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs028.html#___sec27" style="font-size: 80%;">Stochastic Gradient Descent</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs029.html#___sec28" style="font-size: 80%;">Computation of gradients</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs030.html#___sec29" style="font-size: 80%;">SGD example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs031.html#___sec30" style="font-size: 80%;">The gradient step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs032.html#___sec31" style="font-size: 80%;">Simple example code</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs033.html#___sec32" style="font-size: 80%;">When do we stop?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs034.html#___sec33" style="font-size: 80%;">Slightly different approach</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs035.html#___sec34" style="font-size: 80%;">Program for stochastic gradient</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs036.html#___sec35" style="font-size: 80%;">Logistic Regression example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs037.html#___sec36" style="font-size: 80%;">Momentum based GD</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs038.html#___sec37" style="font-size: 80%;">More on momentum based approaches</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._Splines-bs039.html#___sec38" style="font-size: 80%;">Momentum parameter</a></li>
|
||||
@@ -297,7 +297,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 25, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Sep 26, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
|
||||
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p> <br>
|
||||
<center><h4>Sep 25, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Sep 26, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -905,7 +905,21 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec26">Stochastic Gradient Descent </h2>
|
||||
<h2 id="___sec26">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<p><li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<p><li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<p><li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<p><li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<p><li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec27">Stochastic Gradient Descent </h2>
|
||||
|
||||
<p>
|
||||
Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
@@ -925,7 +939,7 @@ $$
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec27">Computation of gradients </h2>
|
||||
<h2 id="___sec28">Computation of gradients </h2>
|
||||
|
||||
<p>
|
||||
This in turn means that the gradient can be
|
||||
@@ -947,7 +961,7 @@ minibatches. We denote these minibatches by \( B_k \) where
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec28">SGD example </h2>
|
||||
<h2 id="___sec29">SGD example </h2>
|
||||
As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
@@ -973,7 +987,7 @@ $$
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec29">The gradient step </h2>
|
||||
<h2 id="___sec30">The gradient step </h2>
|
||||
|
||||
<p>
|
||||
Thus a gradient descent step now looks like
|
||||
@@ -994,7 +1008,7 @@ the number of minibatches, as exemplified in the code below.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec30">Simple example code </h2>
|
||||
<h2 id="___sec31">Simple example code </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1026,7 +1040,7 @@ all \( n \) datapoints.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec31">When do we stop? </h2>
|
||||
<h2 id="___sec32">When do we stop? </h2>
|
||||
|
||||
<p>
|
||||
A natural question is when do we stop the search for a new minimum?
|
||||
@@ -1043,7 +1057,7 @@ gave the lowest value.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec32">Slightly different approach </h2>
|
||||
<h2 id="___sec33">Slightly different approach </h2>
|
||||
|
||||
<p>
|
||||
Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
@@ -1094,7 +1108,7 @@ j = <span style="color: #B452CD">0</span>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec33">Program for stochastic gradient </h2>
|
||||
<h2 id="___sec34">Program for stochastic gradient </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1126,7 +1140,7 @@ Niterations = <span style="color: #B452CD">1000</span>
|
||||
m = <span style="color: #B452CD">100</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
||||
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T.dot(xb.dot(theta)-y)
|
||||
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T <span style="color: #a61717; background-color: #e3d2d2">@</span> ((xb <span style="color: #a61717; background-color: #e3d2d2">@</span> theta)-y)
|
||||
theta -= eta*gradients
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"theta frm own gd"</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(theta)
|
||||
@@ -1150,7 +1164,7 @@ theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color
|
||||
random_index = np.random.randint(m)
|
||||
xi = xb[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
gradients = <span style="color: #B452CD">2</span> * xi.T.dot(xi.dot(theta)-yi)
|
||||
gradients = <span style="color: #B452CD">2</span> * xi.T <span style="color: #a61717; background-color: #e3d2d2">@</span> ((xi <span style="color: #a61717; background-color: #e3d2d2">@</span> theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"theta from own sdg"</span>)
|
||||
@@ -1169,21 +1183,7 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec34">Logistic Regression example </h2>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec35">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<p><li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<p><li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<p><li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<p><li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<p><li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
<h2 id="___sec35">Logistic Regression example </h2>
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
@@ -100,19 +100,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -214,7 +214,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 25, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Sep 26, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -906,7 +906,20 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec26">Stochastic Gradient Descent </h2>
|
||||
<h2 id="___sec26">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<ul>
|
||||
<li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">Stochastic Gradient Descent </h2>
|
||||
|
||||
<p>
|
||||
Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
@@ -924,7 +937,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">Computation of gradients </h2>
|
||||
<h2 id="___sec28">Computation of gradients </h2>
|
||||
|
||||
<p>
|
||||
This in turn means that the gradient can be
|
||||
@@ -944,7 +957,7 @@ minibatches. We denote these minibatches by \( B_k \) where
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">SGD example </h2>
|
||||
<h2 id="___sec29">SGD example </h2>
|
||||
As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
@@ -968,7 +981,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec29">The gradient step </h2>
|
||||
<h2 id="___sec30">The gradient step </h2>
|
||||
|
||||
<p>
|
||||
Thus a gradient descent step now looks like
|
||||
@@ -987,7 +1000,7 @@ the number of minibatches, as exemplified in the code below.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">Simple example code </h2>
|
||||
<h2 id="___sec31">Simple example code </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1019,7 +1032,7 @@ all \( n \) datapoints.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec31">When do we stop? </h2>
|
||||
<h2 id="___sec32">When do we stop? </h2>
|
||||
|
||||
<p>
|
||||
A natural question is when do we stop the search for a new minimum?
|
||||
@@ -1036,7 +1049,7 @@ gave the lowest value.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec32">Slightly different approach </h2>
|
||||
<h2 id="___sec33">Slightly different approach </h2>
|
||||
|
||||
<p>
|
||||
Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
@@ -1084,7 +1097,7 @@ j = <span style="color: #B452CD">0</span>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec33">Program for stochastic gradient </h2>
|
||||
<h2 id="___sec34">Program for stochastic gradient </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1116,7 +1129,7 @@ Niterations = <span style="color: #B452CD">1000</span>
|
||||
m = <span style="color: #B452CD">100</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
|
||||
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T.dot(xb.dot(theta)-y)
|
||||
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T <span style="color: #a61717; background-color: #e3d2d2">@</span> ((xb <span style="color: #a61717; background-color: #e3d2d2">@</span> theta)-y)
|
||||
theta -= eta*gradients
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"theta frm own gd"</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(theta)
|
||||
@@ -1140,7 +1153,7 @@ theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color
|
||||
random_index = np.random.randint(m)
|
||||
xi = xb[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
|
||||
gradients = <span style="color: #B452CD">2</span> * xi.T.dot(xi.dot(theta)-yi)
|
||||
gradients = <span style="color: #B452CD">2</span> * xi.T <span style="color: #a61717; background-color: #e3d2d2">@</span> ((xi <span style="color: #a61717; background-color: #e3d2d2">@</span> theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">"theta from own sdg"</span>)
|
||||
@@ -1158,24 +1171,11 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec34">Logistic Regression example </h2>
|
||||
<h2 id="___sec35">Logistic Regression example </h2>
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec35">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<ul>
|
||||
<li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec36">Momentum based GD </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -105,19 +105,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
2,
|
||||
None,
|
||||
'___sec25'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec26'),
|
||||
('Computation of gradients', 2, None, '___sec27'),
|
||||
('SGD example', 2, None, '___sec28'),
|
||||
('The gradient step', 2, None, '___sec29'),
|
||||
('Simple example code', 2, None, '___sec30'),
|
||||
('When do we stop?', 2, None, '___sec31'),
|
||||
('Slightly different approach', 2, None, '___sec32'),
|
||||
('Program for stochastic gradient', 2, None, '___sec33'),
|
||||
('Logistic Regression example', 2, None, '___sec34'),
|
||||
('Using gradient descent methods, limitations',
|
||||
2,
|
||||
None,
|
||||
'___sec35'),
|
||||
'___sec26'),
|
||||
('Stochastic Gradient Descent', 2, None, '___sec27'),
|
||||
('Computation of gradients', 2, None, '___sec28'),
|
||||
('SGD example', 2, None, '___sec29'),
|
||||
('The gradient step', 2, None, '___sec30'),
|
||||
('Simple example code', 2, None, '___sec31'),
|
||||
('When do we stop?', 2, None, '___sec32'),
|
||||
('Slightly different approach', 2, None, '___sec33'),
|
||||
('Program for stochastic gradient', 2, None, '___sec34'),
|
||||
('Logistic Regression example', 2, None, '___sec35'),
|
||||
('Momentum based GD', 2, None, '___sec36'),
|
||||
('More on momentum based approaches', 2, None, '___sec37'),
|
||||
('Momentum parameter', 2, None, '___sec38'),
|
||||
@@ -219,7 +219,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Sep 25, 2019</h4></center> <!-- date -->
|
||||
<center><h4>Sep 26, 2019</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -911,7 +911,20 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec26">Stochastic Gradient Descent </h2>
|
||||
<h2 id="___sec26">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<ul>
|
||||
<li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">Stochastic Gradient Descent </h2>
|
||||
|
||||
<p>
|
||||
Stochastic gradient descent (SGD) and variants thereof address some of
|
||||
@@ -929,7 +942,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">Computation of gradients </h2>
|
||||
<h2 id="___sec28">Computation of gradients </h2>
|
||||
|
||||
<p>
|
||||
This in turn means that the gradient can be
|
||||
@@ -949,7 +962,7 @@ minibatches. We denote these minibatches by \( B_k \) where
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">SGD example </h2>
|
||||
<h2 id="___sec29">SGD example </h2>
|
||||
As an example, suppose we have \( 10 \) data points \( (\mathbf{x}_1,\cdots, \mathbf{x}_{10}) \)
|
||||
and we choose to have \( M=5 \) minibathces,
|
||||
then each minibatch contains two data points. In particular we have
|
||||
@@ -973,7 +986,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec29">The gradient step </h2>
|
||||
<h2 id="___sec30">The gradient step </h2>
|
||||
|
||||
<p>
|
||||
Thus a gradient descent step now looks like
|
||||
@@ -992,7 +1005,7 @@ the number of minibatches, as exemplified in the code below.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">Simple example code </h2>
|
||||
<h2 id="___sec31">Simple example code </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1024,7 +1037,7 @@ all \( n \) datapoints.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec31">When do we stop? </h2>
|
||||
<h2 id="___sec32">When do we stop? </h2>
|
||||
|
||||
<p>
|
||||
A natural question is when do we stop the search for a new minimum?
|
||||
@@ -1041,7 +1054,7 @@ gave the lowest value.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec32">Slightly different approach </h2>
|
||||
<h2 id="___sec33">Slightly different approach </h2>
|
||||
|
||||
<p>
|
||||
Another approach is to let the step length \( \gamma_j \) depend on the
|
||||
@@ -1089,7 +1102,7 @@ j <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec33">Program for stochastic gradient </h2>
|
||||
<h2 id="___sec34">Program for stochastic gradient </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1121,7 +1134,7 @@ Niterations <span style="color: #666666">=</span> <span style="color: #666666">1
|
||||
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>dot(theta)<span style="color: #666666">-</span>y)
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T @ ((xb @ theta)<span style="color: #666666">-</span>y)
|
||||
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"theta frm own gd"</span>)
|
||||
<span style="color: #008000; font-weight: bold">print</span>(theta)
|
||||
@@ -1145,7 +1158,7 @@ theta <span style="color: #666666">=</span> np<span style="color: #666666">.</sp
|
||||
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
|
||||
xi <span style="color: #666666">=</span> xb[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xi<span style="color: #666666">.</span>dot(theta)<span style="color: #666666">-</span>yi)
|
||||
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T @ ((xi @ theta)<span style="color: #666666">-</span>yi)
|
||||
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
|
||||
theta <span style="color: #666666">=</span> theta <span style="color: #666666">-</span> eta<span style="color: #666666">*</span>gradients
|
||||
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">"theta from own sdg"</span>)
|
||||
@@ -1163,24 +1176,11 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec34">Logistic Regression example </h2>
|
||||
<h2 id="___sec35">Logistic Regression example </h2>
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec35">Using gradient descent methods, limitations </h2>
|
||||
|
||||
<ul>
|
||||
<li> <b>Gradient descent (GD) finds local minima of our function</b>. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.</li>
|
||||
<li> <b>GD is sensitive to initial conditions</b>. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.</li>
|
||||
<li> <b>Gradients are computationally expensive to calculate for large datasets</b>. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, \( E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2 \); for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over <em>all</em> \( n \) data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called "mini batches". This has the added benefit of introducing stochasticity into our algorithm.</li>
|
||||
<li> <b>GD is very sensitive to choices of learning rates</b>. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would <em>adaptively</em> choose the learning rates to match the landscape.</li>
|
||||
<li> <b>GD treats all directions in parameter space uniformly.</b> Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.</li>
|
||||
<li> GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.</li>
|
||||
</ul>
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec36">Momentum based GD </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"<!-- Author: --> \n",
|
||||
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
|
||||
"\n",
|
||||
"Date: **Sep 25, 2019**\n",
|
||||
"Date: **Sep 26, 2019**\n",
|
||||
"\n",
|
||||
"Copyright 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
|
||||
"\n",
|
||||
@@ -923,6 +923,20 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Using gradient descent methods, limitations\n",
|
||||
"\n",
|
||||
"* **Gradient descent (GD) finds local minima of our function**. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.\n",
|
||||
"\n",
|
||||
"* **GD is sensitive to initial conditions**. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.\n",
|
||||
"\n",
|
||||
"* **Gradients are computationally expensive to calculate for large datasets**. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, $E \\propto \\sum_{i=1}^n (y_i - \\mathbf{w}^T\\cdot\\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called \"mini batches\". This has the added benefit of introducing stochasticity into our algorithm.\n",
|
||||
"\n",
|
||||
"* **GD is very sensitive to choices of learning rates**. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape.\n",
|
||||
"\n",
|
||||
"* **GD treats all directions in parameter space uniformly.** Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive. \n",
|
||||
"\n",
|
||||
"* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.\n",
|
||||
"\n",
|
||||
"## Stochastic Gradient Descent\n",
|
||||
"\n",
|
||||
"Stochastic gradient descent (SGD) and variants thereof address some of\n",
|
||||
@@ -1172,7 +1186,7 @@
|
||||
"m = 100\n",
|
||||
"\n",
|
||||
"for iter in range(Niterations):\n",
|
||||
" gradients = 2.0/m*xb.T.dot(xb.dot(theta)-y)\n",
|
||||
" gradients = 2.0/m*xb.T @ ((xb @ theta)-y)\n",
|
||||
" theta -= eta*gradients\n",
|
||||
"print(\"theta frm own gd\")\n",
|
||||
"print(theta)\n",
|
||||
@@ -1196,7 +1210,7 @@
|
||||
" random_index = np.random.randint(m)\n",
|
||||
" xi = xb[random_index:random_index+1]\n",
|
||||
" yi = y[random_index:random_index+1]\n",
|
||||
" gradients = 2 * xi.T.dot(xi.dot(theta)-yi)\n",
|
||||
" gradients = 2 * xi.T @ ((xi @ theta)-yi)\n",
|
||||
" eta = learning_schedule(epoch*m+i)\n",
|
||||
" theta = theta - eta*gradients\n",
|
||||
"print(\"theta from own sdg\")\n",
|
||||
@@ -1219,19 +1233,6 @@
|
||||
"## Logistic Regression example\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Using gradient descent methods, limitations\n",
|
||||
"\n",
|
||||
"* **Gradient descent (GD) finds local minima of our function**. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.\n",
|
||||
"\n",
|
||||
"* **GD is sensitive to initial conditions**. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.\n",
|
||||
"\n",
|
||||
"* **Gradients are computationally expensive to calculate for large datasets**. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, $E \\propto \\sum_{i=1}^n (y_i - \\mathbf{w}^T\\cdot\\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called \"mini batches\". This has the added benefit of introducing stochasticity into our algorithm.\n",
|
||||
"\n",
|
||||
"* **GD is very sensitive to choices of learning rates**. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape.\n",
|
||||
"\n",
|
||||
"* **GD treats all directions in parameter space uniformly.** Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive. \n",
|
||||
"\n",
|
||||
"* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.\n",
|
||||
"\n",
|
||||
"## Momentum based GD\n",
|
||||
"\n",
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,44 @@
|
||||
import numpy as np
|
||||
from sklearn import datasets, linear_model
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def generate_data():
|
||||
np.random.seed(0)
|
||||
X, y = datasets.make_moons(200, noise=0.20)
|
||||
return X, y
|
||||
|
||||
|
||||
def visualize(X, y, clf):
|
||||
plot_decision_boundary(lambda x: clf.predict(x), X, y)
|
||||
|
||||
def plot_decision_boundary(pred_func, X, y):
|
||||
# Set min and max values and give it some padding
|
||||
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
|
||||
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
|
||||
h = 0.01
|
||||
# Generate a grid of points with distance h between them
|
||||
xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
|
||||
# Predict the function value for the whole gid
|
||||
Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
|
||||
Z = Z.reshape(xx.shape)
|
||||
# Plot the contour and training examples
|
||||
plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
|
||||
plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral)
|
||||
plt.show()
|
||||
|
||||
|
||||
def classify(X, y):
|
||||
clf = linear_model.LogisticRegressionCV()
|
||||
clf.fit(X, y)
|
||||
return clf
|
||||
|
||||
|
||||
def main():
|
||||
X, y = generate_data()
|
||||
# visualize(X, y)
|
||||
clf = classify(X, y)
|
||||
visualize(X, y, clf)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -625,6 +625,21 @@ plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Using gradient descent methods, limitations =====
|
||||
|
||||
* _Gradient descent (GD) finds local minima of our function_. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.
|
||||
|
||||
* _GD is sensitive to initial conditions_. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.
|
||||
|
||||
* _Gradients are computationally expensive to calculate for large datasets_. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, $E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called ``mini batches''. This has the added benefit of introducing stochasticity into our algorithm.
|
||||
|
||||
* _GD is very sensitive to choices of learning rates_. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape.
|
||||
|
||||
* _GD treats all directions in parameter space uniformly._ Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.
|
||||
|
||||
* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.
|
||||
|
||||
|
||||
|
||||
!split
|
||||
@@ -820,7 +835,7 @@ Niterations = 1000
|
||||
m = 100
|
||||
|
||||
for iter in range(Niterations):
|
||||
gradients = 2.0/m*xb.T.dot(xb.dot(theta)-y)
|
||||
gradients = 2.0/m*xb.T @ ((xb @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
print("theta frm own gd")
|
||||
print(theta)
|
||||
@@ -844,7 +859,7 @@ for epoch in range(n_epochs):
|
||||
random_index = np.random.randint(m)
|
||||
xi = xb[random_index:random_index+1]
|
||||
yi = y[random_index:random_index+1]
|
||||
gradients = 2 * xi.T.dot(xi.dot(theta)-yi)
|
||||
gradients = 2 * xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
theta = theta - eta*gradients
|
||||
print("theta from own sdg")
|
||||
@@ -866,21 +881,6 @@ plt.show()
|
||||
===== Logistic Regression example =====
|
||||
|
||||
|
||||
!split
|
||||
===== Using gradient descent methods, limitations =====
|
||||
|
||||
* _Gradient descent (GD) finds local minima of our function_. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our energy function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance.
|
||||
|
||||
* _GD is sensitive to initial conditions_. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD.
|
||||
|
||||
* _Gradients are computationally expensive to calculate for large datasets_. In many cases in statistics and ML, the energy function is a sum of terms, with one term for each data point. For example, in linear regression, $E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called ``mini batches''. This has the added benefit of introducing stochasticity into our algorithm.
|
||||
|
||||
* _GD is very sensitive to choices of learning rates_. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape.
|
||||
|
||||
* _GD treats all directions in parameter space uniformly._ Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.
|
||||
|
||||
* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.
|
||||
|
||||
|
||||
!split
|
||||
===== Momentum based GD =====
|
||||
|
||||
+11
-10
@@ -1,34 +1,34 @@
|
||||
from math import exp, sqrt
|
||||
from random import random, seed
|
||||
import numpy as np
|
||||
from sklearn import datasets, linear_model
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.linear_model import SGDRegressor
|
||||
|
||||
x = 2*np.random.rand(100,1)
|
||||
y = 4+3*x#+np.random.randn(100,1)
|
||||
|
||||
xb = np.c_[np.ones((100,1)), x]
|
||||
np.random.seed(0)
|
||||
X, y = datasets.make_moons(200, noise=0.20)
|
||||
#X = np.c_[np.ones((100,1)), x]
|
||||
|
||||
sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)
|
||||
sgdreg.fit(x,y.ravel())
|
||||
sgdreg.fit(X,y.ravel())
|
||||
print("sgdreg from scikit")
|
||||
print(sgdreg.intercept_, sgdreg.coef_)
|
||||
|
||||
|
||||
"""
|
||||
theta = np.random.randn(2,1)
|
||||
eta = 0.1
|
||||
Niterations = 1000
|
||||
m = 100
|
||||
|
||||
for iter in range(Niterations):
|
||||
gradients = 2.0/m*xb.T @ ((xb @ theta)-y)
|
||||
gradients = 2.0/m*X.T @ ((X @ theta)-y)
|
||||
theta -= eta*gradients
|
||||
print("theta from own gd")
|
||||
print(theta)
|
||||
|
||||
xnew = np.array([[0],[2]])
|
||||
xbnew = np.c_[np.ones((2,1)), xnew]
|
||||
ypredict = xbnew.dot(theta)
|
||||
Xnew = np.c_[np.ones((2,1)), xnew]
|
||||
ypredict = Xnew.dot(theta)
|
||||
|
||||
n_epochs = 50
|
||||
t0, t1 = 5, 50
|
||||
@@ -41,7 +41,7 @@ theta = np.random.randn(2,1)
|
||||
for epoch in range(n_epochs):
|
||||
for i in range(m):
|
||||
random_index = np.random.randint(m)
|
||||
xi = xb[random_index:random_index+1]
|
||||
xi = X[random_index:random_index+1]
|
||||
yi = y[random_index:random_index+1]
|
||||
gradients = 2 * xi.T @ ((xi @ theta)-yi)
|
||||
eta = learning_schedule(epoch*m+i)
|
||||
@@ -56,3 +56,4 @@ plt.xlabel(r'$x$')
|
||||
plt.ylabel(r'$y$')
|
||||
plt.title(r'Random numbers ')
|
||||
plt.show()
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user