From ef56fde91717389db100e94a4b0e18461ac71b08 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Thu, 29 Sep 2022 05:27:01 +0200 Subject: [PATCH] update week 39 --- doc/pub/week39/html/week39-bs.html | 84 +++++---- doc/pub/week39/html/week39-reveal.html | 174 +++++++++++++++++- doc/pub/week39/html/week39-solarized.html | 182 ++++++++++++++++++- doc/pub/week39/html/week39.html | 182 ++++++++++++++++++- doc/pub/week39/ipynb/ipynb-week39-src.tar.gz | Bin 192 -> 192 bytes doc/src/week39/week39.do.txt | 138 +++++++++++++- 6 files changed, 719 insertions(+), 41 deletions(-) diff --git a/doc/pub/week39/html/week39-bs.html b/doc/pub/week39/html/week39-bs.html index e853e960d..c7edb4c96 100644 --- a/doc/pub/week39/html/week39-bs.html +++ b/doc/pub/week39/html/week39-bs.html @@ -152,6 +152,14 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'using-gradient-descent-methods-limitations'), + ('Improving gradient descent with momentum', + 2, + None, + 'improving-gradient-descent-with-momentum'), + ('Same code but now with momentum gradient descent', + 2, + None, + 'same-code-but-now-with-momentum-gradient-descent'), ('Overview video on Stochastic Gradient Descent', 2, None, @@ -310,41 +318,43 @@ MathJax.Hub.Config({
  • The Hessian matrix for Ridge Regression
  • Program example for gradient descent with Ridge Regression
  • Using gradient descent methods, limitations
    • -
  • Overview video on Stochastic Gradient Descent
    • -
  • Batches and mini-batches
    • -
  • Stochastic Gradient Descent (SGD)
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Time decay rate
    • -
  • Code with a Number of Minibatches which varies
    • -
  • Replace or not
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • -
  • Practical tips
    • -
  • Automatic differentiation
    • -
  • Using autograd
    • -
  • Autograd with more complicated functions
    • -
  • More complicated functions using the elements of their arguments directly
    • -
  • Functions using mathematical functions from Numpy
    • -
  • More autograd
    • -
  • And with loops
    • -
  • Using recursion
    • -
  • Unsupported functions
    • -
  • The syntax a.dot(b) when finding the dot product
    • -
  • Recommended to avoid
    • -
  • Using Autograd with OLS
    • -
  • Including Stochastic Gradient Descent with Autograd
    • -
  • And Logistic Regression
    • -
  • Introducing "JAX":"https://jax.readthedocs.io/en/latest/"
    • +
  • Improving gradient descent with momentum
    • +
  • Same code but now with momentum gradient descent
    • +
  • Overview video on Stochastic Gradient Descent
    • +
  • Batches and mini-batches
    • +
  • Stochastic Gradient Descent (SGD)
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Time decay rate
    • +
  • Code with a Number of Minibatches which varies
    • +
  • Replace or not
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • "ADAM optimizer":"https://arxiv.org/abs/1412.6980"
    • +
  • Practical tips
    • +
  • Automatic differentiation
    • +
  • Using autograd
    • +
  • Autograd with more complicated functions
    • +
  • More complicated functions using the elements of their arguments directly
    • +
  • Functions using mathematical functions from Numpy
    • +
  • More autograd
    • +
  • And with loops
    • +
  • Using recursion
    • +
  • Unsupported functions
    • +
  • The syntax a.dot(b) when finding the dot product
    • +
  • Recommended to avoid
    • +
  • Using Autograd with OLS
    • +
  • Including Stochastic Gradient Descent with Autograd
    • +
  • And Logistic Regression
    • +
  • Introducing "JAX":"https://jax.readthedocs.io/en/latest/"
    • @@ -374,7 +384,7 @@ MathJax.Hub.Config({
    -

    Sep 27, 2022

    +

    Sep 29, 2022


    @@ -399,7 +409,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 79
    • +
  • 81
  • »
    • diff --git a/doc/pub/week39/html/week39-reveal.html b/doc/pub/week39/html/week39-reveal.html index 0a95369af..5a7bd4d43 100644 --- a/doc/pub/week39/html/week39-reveal.html +++ b/doc/pub/week39/html/week39-reveal.html @@ -184,7 +184,7 @@ MathJax.Hub.Config({
    -

    Sep 27, 2022

    +

    Sep 29, 2022


    @@ -1621,6 +1621,178 @@ plt.show() +
    +

    Improving gradient descent with momentum

    + +

    We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.

    + + + +
    +
    +
    +
    +
    + 
    from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# take a step
+		solution = solution - step_size * gradient
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# perform the gradient descent search
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    +

    Same code but now with momentum gradient descent

    + + + +
    +
    +
    +
    +
    + 
    from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# keep track of the change
+	change = 0.0
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# calculate update
+		new_change = step_size * gradient + momentum * change
+		# take a step
+		solution = solution - new_change
+		# save the change
+		change = new_change
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# define momentum
+momentum = 0.3
+# perform the gradient descent search with momentum
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    Overview video on Stochastic Gradient Descent

    diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html index cf8a6af8b..02f730271 100644 --- a/doc/pub/week39/html/week39-solarized.html +++ b/doc/pub/week39/html/week39-solarized.html @@ -179,6 +179,14 @@ div.toc p,a { 2, None, 'using-gradient-descent-methods-limitations'), + ('Improving gradient descent with momentum', + 2, + None, + 'improving-gradient-descent-with-momentum'), + ('Same code but now with momentum gradient descent', + 2, + None, + 'same-code-but-now-with-momentum-gradient-descent'), ('Overview video on Stochastic Gradient Descent', 2, None, @@ -297,7 +305,7 @@ MathJax.Hub.Config({
    -

    Sep 27, 2022

    +

    Sep 29, 2022


    @@ -1579,6 +1587,178 @@ plt.show()
  • GD treats all directions in parameter space uniformly. Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.
  • GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.
    • +









    +

    Improving gradient descent with momentum

    + +

    We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.

    + + + +
    +
    +
    +
    +
    + 
    from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# take a step
+		solution = solution - step_size * gradient
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# perform the gradient descent search
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + + +









    +

    Same code but now with momentum gradient descent

    + + + +
    +
    +
    +
    +
    + 
    from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# keep track of the change
+	change = 0.0
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# calculate update
+		new_change = step_size * gradient + momentum * change
+		# take a step
+		solution = solution - new_change
+		# save the change
+		change = new_change
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# define momentum
+momentum = 0.3
+# perform the gradient descent search with momentum
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    Overview video on Stochastic Gradient Descent

    diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html index 1e49d0904..48dfcbf03 100644 --- a/doc/pub/week39/html/week39.html +++ b/doc/pub/week39/html/week39.html @@ -256,6 +256,14 @@ div.toc p,a { 2, None, 'using-gradient-descent-methods-limitations'), + ('Improving gradient descent with momentum', + 2, + None, + 'improving-gradient-descent-with-momentum'), + ('Same code but now with momentum gradient descent', + 2, + None, + 'same-code-but-now-with-momentum-gradient-descent'), ('Overview video on Stochastic Gradient Descent', 2, None, @@ -374,7 +382,7 @@ MathJax.Hub.Config({
    -

    Sep 27, 2022

    +

    Sep 29, 2022


    @@ -1656,6 +1664,178 @@ plt.show()
  • GD treats all directions in parameter space uniformly. Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive.
  • GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points.
    • +









    +

    Improving gradient descent with momentum

    + +

    We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent.

    + + + +
    +
    +
    +
    +
    + 
    from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# take a step
+		solution = solution - step_size * gradient
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# perform the gradient descent search
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + + +









    +

    Same code but now with momentum gradient descent

    + + + +
    +
    +
    +
    +
    + 
    from numpy import asarray
+from numpy import arange
+from numpy.random import rand
+from numpy.random import seed
+from matplotlib import pyplot
+ 
+# objective function
+def objective(x):
+	return x**2.0
+ 
+# derivative of objective function
+def derivative(x):
+	return x * 2.0
+ 
+# gradient descent algorithm
+def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum):
+	# track all solutions
+	solutions, scores = list(), list()
+	# generate an initial point
+	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
+	# keep track of the change
+	change = 0.0
+	# run the gradient descent
+	for i in range(n_iter):
+		# calculate gradient
+		gradient = derivative(solution)
+		# calculate update
+		new_change = step_size * gradient + momentum * change
+		# take a step
+		solution = solution - new_change
+		# save the change
+		change = new_change
+		# evaluate candidate point
+		solution_eval = objective(solution)
+		# store solution
+		solutions.append(solution)
+		scores.append(solution_eval)
+		# report progress
+		print('>%d f(%s) = %.5f' % (i, solution, solution_eval))
+	return [solutions, scores]
+ 
+# seed the pseudo random number generator
+seed(4)
+# define range for input
+bounds = asarray([[-1.0, 1.0]])
+# define the total iterations
+n_iter = 30
+# define the step size
+step_size = 0.1
+# define momentum
+momentum = 0.3
+# perform the gradient descent search with momentum
+solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum)
+# sample input range uniformly at 0.1 increments
+inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1)
+# compute targets
+results = objective(inputs)
+# create a line plot of input vs result
+pyplot.plot(inputs, results)
+# plot the solutions found
+pyplot.plot(solutions, scores, '.-', color='red')
+# show the plot
+pyplot.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    Overview video on Stochastic Gradient Descent

    diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz index 81b0905a39614355d5dbf247574076f7eda87192..6482edc57356bd2641f3511442ddbe1a598f9ecf 100644 GIT binary patch literal 192 zcmV;x06+g9iwFR65H(`}1MSbv3c@f92k@Qu6nTQtY-`1%;0_)H5nrHVovXTbwjH{+ zcORf9#mf+(zssMH5R!eiT5q$+-CZynLI_J!7;?erm?WO+5h4Xl6QnT`WHAs-D6#;^ zd?&rM)^XFHQdcLTtWfXf`mwV7uxEM&p7|#Zm9((gb*|D1ly)N5`V2QA&dD;8O{a1w uw6H@9jJUSa2;iy%UKG+vt@tHuj6ND(+bI0?GoI&p-q#-J$wxZ?2mk<;mssBb literal 192 zcmV;x06+g9iwFSq@G@fn1MSaE3c@fD1z^`b#hjodCT(#k*o6y0#0#W!YGZ9ulN9ak z?GNZmaZ^Odw|RtlgqcIS-t5xQ-Q8j~gpinX7&3{YG0Adzk0_0Raz=TS5Y`t6Wf5Zw zAoH#C(po1>ze-)6QCU>)dVQ@ZKKwJC0?+&t$5L9@?mJg%1xh>2w65TWSg}?>y2mk;K3|EW* diff --git a/doc/src/week39/week39.do.txt b/doc/src/week39/week39.do.txt index 1329d9d67..48d1a7a5b 100644 --- a/doc/src/week39/week39.do.txt +++ b/doc/src/week39/week39.do.txt @@ -1064,7 +1064,143 @@ plt.show() * _GD treats all directions in parameter space uniformly._ Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive. * GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points. - + +!split +===== Improving gradient descent with momentum ===== + +We discuss here some simple examples where we introduce what is called 'memory'about previous steps, or what is normally called momentum gradient descent. The mathematics is explained below in connection with Stochastic gradient descent. + +!bc pycod +from numpy import asarray +from numpy import arange +from numpy.random import rand +from numpy.random import seed +from matplotlib import pyplot + +# objective function +def objective(x): + return x**2.0 + +# derivative of objective function +def derivative(x): + return x * 2.0 + +# gradient descent algorithm +def gradient_descent(objective, derivative, bounds, n_iter, step_size): + # track all solutions + solutions, scores = list(), list() + # generate an initial point + solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0]) + # run the gradient descent + for i in range(n_iter): + # calculate gradient + gradient = derivative(solution) + # take a step + solution = solution - step_size * gradient + # evaluate candidate point + solution_eval = objective(solution) + # store solution + solutions.append(solution) + scores.append(solution_eval) + # report progress + print('>%d f(%s) = %.5f' % (i, solution, solution_eval)) + return [solutions, scores] + +# seed the pseudo random number generator +seed(4) +# define range for input +bounds = asarray([[-1.0, 1.0]]) +# define the total iterations +n_iter = 30 +# define the step size +step_size = 0.1 +# perform the gradient descent search +solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size) +# sample input range uniformly at 0.1 increments +inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1) +# compute targets +results = objective(inputs) +# create a line plot of input vs result +pyplot.plot(inputs, results) +# plot the solutions found +pyplot.plot(solutions, scores, '.-', color='red') +# show the plot +pyplot.show() + +!ec + + +!split +===== Same code but now with momentum gradient descent ===== + +!bc pycod +from numpy import asarray +from numpy import arange +from numpy.random import rand +from numpy.random import seed +from matplotlib import pyplot + +# objective function +def objective(x): + return x**2.0 + +# derivative of objective function +def derivative(x): + return x * 2.0 + +# gradient descent algorithm +def gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum): + # track all solutions + solutions, scores = list(), list() + # generate an initial point + solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0]) + # keep track of the change + change = 0.0 + # run the gradient descent + for i in range(n_iter): + # calculate gradient + gradient = derivative(solution) + # calculate update + new_change = step_size * gradient + momentum * change + # take a step + solution = solution - new_change + # save the change + change = new_change + # evaluate candidate point + solution_eval = objective(solution) + # store solution + solutions.append(solution) + scores.append(solution_eval) + # report progress + print('>%d f(%s) = %.5f' % (i, solution, solution_eval)) + return [solutions, scores] + +# seed the pseudo random number generator +seed(4) +# define range for input +bounds = asarray([[-1.0, 1.0]]) +# define the total iterations +n_iter = 30 +# define the step size +step_size = 0.1 +# define momentum +momentum = 0.3 +# perform the gradient descent search with momentum +solutions, scores = gradient_descent(objective, derivative, bounds, n_iter, step_size, momentum) +# sample input range uniformly at 0.1 increments +inputs = arange(bounds[0,0], bounds[0,1]+0.1, 0.1) +# compute targets +results = objective(inputs) +# create a line plot of input vs result +pyplot.plot(inputs, results) +# plot the solutions found +pyplot.plot(solutions, scores, '.-', color='red') +# show the plot +pyplot.show() +!ec + + + !split ===== Overview video on Stochastic Gradient Descent =====