diff --git a/doc/pub/week39/html/._week39-bs000.html b/doc/pub/week39/html/._week39-bs000.html index 8ee7bad96..ee689d11b 100644 --- a/doc/pub/week39/html/._week39-bs000.html +++ b/doc/pub/week39/html/._week39-bs000.html @@ -301,7 +301,7 @@ MathJax.Hub.Config({
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

-

Sep 16, 2020

+

Sep 22, 2020


diff --git a/doc/pub/week39/html/._week39-bs001.html b/doc/pub/week39/html/._week39-bs001.html index 2491cd7e5..8c4eb24d6 100644 --- a/doc/pub/week39/html/._week39-bs001.html +++ b/doc/pub/week39/html/._week39-bs001.html @@ -289,6 +289,9 @@ MathJax.Hub.Config({

  • Friday: Stochastic Gradient descent with examples and automatic differeantion
    • +Reading suggestions for both days: Aurelien Geron's chapter 4 and Murphy sections 8.3 and 8.5 + +

    +

    + +Reading suggestions for both days: Aurelien Geron's chapter 4 and Murphy sections 8.3 and 8.5

    Thursday September 24

    + +

    +"Overview Video, why do we care about gradient methods?":""

    @@ -784,7 +790,7 @@ y = 4+3* xb = np.c_[np.ones((m,1)), x] beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print(beta_linreg) +print(beta_linreg) beta = np.random.randn(2,1) eta = 0.1 @@ -794,7 +800,7 @@ Niterations = 1000 gradients = 2.0/m*xb.T.dot(xb.dot(beta)-y) beta -= eta*gradients -print(beta) +print(beta) xnew = np.array([[0],[2]]) xbnew = np.c_[np.ones((2,1)), xnew] ypredict = xbnew.dot(beta) @@ -828,10 +834,10 @@ y = 4+3* xb = np.c_[np.ones((100,1)), x] beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print(beta_linreg) -sgdreg = SGDRegressor(n_iter = 50, penalty=None, eta0=0.1) +print(beta_linreg) +sgdreg = SGDRegressor(n_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) -print(sgdreg.intercept_, sgdreg.coef_) +print(sgdreg.intercept_, sgdreg.coef_) @@ -886,14 +892,14 @@ x = 2*np.random.rand(m,4+3*x+np.random.randn(m,1) xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +XT_X = xb.T @ xb #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y -print(beta_linreg) +beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -901,12 +907,12 @@ eta = 0.1 Niterations = 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta beta -= eta*gradients -print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +print(beta) +ypredict = xb @ beta +ypredict2 = xb @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1122,7 +1128,7 @@ j = 0 gamma_j = step_length(t,t0,t1) j += 1 -print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j)) +print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j)) @@ -1146,12 +1152,12 @@ y = 4+3* xb = np.c_[np.ones((m,1)), x] theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print("Own inversion") -print(theta_linreg) -sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) +print("Own inversion") +print(theta_linreg) +sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) -print("sgdreg from scikit") -print(sgdreg.intercept_, sgdreg.coef_) +print("sgdreg from scikit") +print(sgdreg.intercept_, sgdreg.coef_) theta = np.random.randn(2,1) @@ -1160,10 +1166,10 @@ Niterations = 1000 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*xb.T @ ((xb @ theta)-y) theta -= eta*gradients -print("theta frm own gd") -print(theta) +print("theta frm own gd") +print(theta) xnew = np.array([[0],[2]]) xbnew = np.c_[np.ones((2,1)), xnew] @@ -1183,11 +1189,11 @@ theta = np.random.randn(2,1] yi = y[random_index:random_index+1] - gradients = 2 * xi.T @ ((xi @ theta)-yi) + gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients -print("theta from own sdg") -print(theta) +print("theta from own sdg") +print(theta) plt.plot(xnew, ypredict, "r-") plt.plot(xnew, ypredict2, "b-") @@ -1555,7 +1561,7 @@ plt.legend() plt.show() -print("The max absolute difference is: %g"%(np.max(np.abs(computed - analytic)))) +print("The max absolute difference is: %g"%(np.max(np.abs(computed - analytic)))) @@ -1585,11 +1591,11 @@ f1_grad = grad(f1) a = 1.0 # See the evaluated gradient at a using autograd: -print("The gradient of f1 evaluated at a = %g using autograd is: %g"%(a,f1_grad(a))) +print("The gradient of f1 evaluated at a = %g using autograd is: %g"%(a,f1_grad(a))) # Compare with the analytical derivative, that is f1'(x) = 3*x**2 grad_analytical = 3*a**2 -print("The gradient of f1 evaluated at a = %g by finding the analytic expression is: %g"%(a,grad_analytical)) +print("The gradient of f1 evaluated at a = %g by finding the analytic expression is: %g"%(a,grad_analytical)) @@ -1619,8 +1625,8 @@ f2_grad_x2 = grad(f2,1) x1 = 1.0 x2 = 3.0 -print("Evaluating at x1 = %g, x2 = %g"%(x1,x2)) -print("-"*30) +print("Evaluating at x1 = %g, x2 = %g"%(x1,x2)) +print("-"*30) # Compare with the analytical derivatives: @@ -1631,13 +1637,13 @@ f2_grad_x1_analytical = 9*x1**5 # See the evaluated derivations: -print("The derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) -print("The analytical derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) +print("The derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) +print("The analytical derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) -print() +print() -print("The derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) -print("The analytical derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) +print("The derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) +print("The analytical derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) ))

    Note that the grad function will not produce the true gradient of the function. The true gradient of a function with two or more variables will produce a vector, where each element is the function differentiated w.r.t a variable. @@ -1660,13 +1666,13 @@ f3_grad = grad(f3) x = np.linspace(0,4,5) # Print the computed gradient: -print("The computed gradient of f3 is: ", f3_grad(x)) +print("The computed gradient of f3 is: ", f3_grad(x)) # The analytical gradient is: (2, 3, 5, 7, 22*x[4]) f3_grad_analytical = np.array([2, 3, 5, 7, 22*x[4]]) # Print the analytical gradient: -print("The analytical gradient of f3 is: ", f3_grad_analytical) +print("The analytical gradient of f3 is: ", f3_grad_analytical)

    Note that in this case, when sending an array as input argument, the @@ -1694,13 +1700,13 @@ f4_grad = grad(f4) x = 2.7 # Print the computed derivative: -print("The computed derivative of f4 at x = %g is: %g"%(x,f4_grad(x))) +print("The computed derivative of f4 at x = %g is: %g"%(x,f4_grad(x))) # The analytical derivative is: x/sqrt(1 + x**2) + exp(x) + cos(2*pi*x)*2*pi f4_grad_analytical = x/np.sqrt(1 + x**2) + np.exp(x) + np.cos(2*np.pi*x)*2*np.pi # Print the analytical gradient: -print("The analytical gradient of f4 at x = %g is: %g"%(x,f4_grad_analytical)) +print("The analytical gradient of f4 at x = %g is: %g"%(x,f4_grad_analytical)) @@ -1724,7 +1730,7 @@ f5_grad = grad(f5) x = 2.7 # Print the computed derivative: -print("The computed derivative of f5 at x = %g is: %g"%(x,f5_grad(x))) +print("The computed derivative of f5 at x = %g is: %g"%(x,f5_grad(x))) @@ -1757,8 +1763,8 @@ f6_while_grad = grad(f6_while) x = 0.5 # Print the computed derivaties of f6_for and f6_while -print("The computed derivative of f6_for at x = %g is: %g"%(x,f6_for_grad(x))) -print("The computed derivative of f6_while at x = %g is: %g"%(x,f6_while_grad(x))) +print("The computed derivative of f6_for at x = %g is: %g"%(x,f6_for_grad(x))) +print("The computed derivative of f6_while at x = %g is: %g"%(x,f6_while_grad(x)))

    @@ -1771,7 +1777,7 @@ f6_grad_analytical = 0 for i in range(10): f6_grad_analytical += i*x**(i-1) -print("The analytical derivative of f6 at x = %g is: %g"%(x,f6_grad_analytical)) +print("The analytical derivative of f6 at x = %g is: %g"%(x,f6_grad_analytical)) @@ -1794,7 +1800,7 @@ f7_grad = grad(f7) n = 2.0 -print("The computed derivative of f7 at n = %d is: %g"%(n,f7_grad(n))) +print("The computed derivative of f7 at n = %d is: %g"%(n,f7_grad(n))) # The function f7 is an implementation of the factorial of n. # By using the product rule, one can find that the derivative is: @@ -1807,7 +1813,7 @@ f7_grad_analytical = 0 tmp *= (n - k) f7_grad_analytical += tmp -print("The analytical derivative of f7 at n = %d is: %g"%(n,f7_grad_analytical)) +print("The analytical derivative of f7 at n = %d is: %g"%(n,f7_grad_analytical))

    Note that if n is equal to zero or one, Autograd will give an error message. This message appears when the output is independent on input. @@ -1833,7 +1839,7 @@ f8_grad = grad(f8) x = 8.4 -print("The derivative of f8 is:",f8_grad(x)) +print("The derivative of f8 is:",f8_grad(x))

    Here, Autograd tells us that an 'ArrayBox' does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible. @@ -1855,7 +1861,7 @@ f9_grad = grad(f9) x = np.array([1.0,0.0]) -print("The derivative of f9 is:",f9_grad(x)) +print("The derivative of f9 is:",f9_grad(x))

    Here we are told that the 'dot' function does not belong to Autograd's @@ -1875,7 +1881,7 @@ f9_alternative_grad = grad(f9_alternative) x = np.array([3.0,0.0]) -print("The gradient of f9 is:",f9_alternative_grad(x)) +print("The gradient of f9 is:",f9_alternative_grad(x)) # The analytical gradient of the dot product of vectors x and b with two elements (x_1,x_2) and (b_1, b_2) respectively # w.r.t x is (b_1, b_2). @@ -2168,7 +2174,7 @@ $$ fig = pt.figure() ax = fig.gca(projection="3d") -xmesh, ymesh = np.mgrid[-2:2:50j,-2:2:50j] +xmesh, ymesh = np.mgrid[-2:2:50j,-2:2:50j] fmesh = f(np.array([xmesh, ymesh])) ax.plot_surface(xmesh, ymesh, fmesh) @@ -2200,7 +2206,7 @@ Run it! alpha_opt = sopt.golden(f1d) next_guess = x + alpha_opt * s guesses.append(next_guess) -print(next_guess) +print(next_guess)

    What happened? diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html index 164656896..0acc11378 100644 --- a/doc/pub/week39/html/week39-solarized.html +++ b/doc/pub/week39/html/week39-solarized.html @@ -216,7 +216,7 @@ MathJax.Hub.Config({

    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

    -

    Sep 16, 2020

    +

    Sep 22, 2020












    @@ -228,10 +228,16 @@ MathJax.Hub.Config({

  • Friday: Stochastic Gradient descent with examples and automatic differeantion
    • +Reading suggestions for both days: Aurelien Geron's chapter 4 and Murphy sections 8.3 and 8.5 + +











    Thursday September 24

    +

    +"Overview Video, why do we care about gradient methods?":"" +











    @@ -795,7 +801,7 @@ y = 4+3* xb = np.c_[np.ones((m,1)), x] beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print(beta_linreg) +print(beta_linreg) beta = np.random.randn(2,1) eta = 0.1 @@ -805,7 +811,7 @@ Niterations = 1000 gradients = 2.0/m*xb.T.dot(xb.dot(beta)-y) beta -= eta*gradients -print(beta) +print(beta) xnew = np.array([[0],[2]]) xbnew = np.c_[np.ones((2,1)), xnew] ypredict = xbnew.dot(beta) @@ -838,10 +844,10 @@ y = 4+3* xb = np.c_[np.ones((100,1)), x] beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print(beta_linreg) -sgdreg = SGDRegressor(n_iter = 50, penalty=None, eta0=0.1) +print(beta_linreg) +sgdreg = SGDRegressor(n_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) -print(sgdreg.intercept_, sgdreg.coef_) +print(sgdreg.intercept_, sgdreg.coef_)

    @@ -889,14 +895,14 @@ x = 2*np.random.rand(m,4+3*x+np.random.randn(m,1) xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +XT_X = xb.T @ xb #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y -print(beta_linreg) +beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -904,12 +910,12 @@ eta = 0.1 Niterations = 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta beta -= eta*gradients -print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +print(beta) +ypredict = xb @ beta +ypredict2 = xb @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1113,7 +1119,7 @@ j = 0 gamma_j = step_length(t,t0,t1) j += 1 -print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j)) +print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))











    @@ -1136,12 +1142,12 @@ y = 4+3* xb = np.c_[np.ones((m,1)), x] theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print("Own inversion") -print(theta_linreg) -sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) +print("Own inversion") +print(theta_linreg) +sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) -print("sgdreg from scikit") -print(sgdreg.intercept_, sgdreg.coef_) +print("sgdreg from scikit") +print(sgdreg.intercept_, sgdreg.coef_) theta = np.random.randn(2,1) @@ -1150,10 +1156,10 @@ Niterations = 1000 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*xb.T @ ((xb @ theta)-y) theta -= eta*gradients -print("theta frm own gd") -print(theta) +print("theta frm own gd") +print(theta) xnew = np.array([[0],[2]]) xbnew = np.c_[np.ones((2,1)), xnew] @@ -1173,11 +1179,11 @@ theta = np.random.randn(2,1] yi = y[random_index:random_index+1] - gradients = 2 * xi.T @ ((xi @ theta)-yi) + gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients -print("theta from own sdg") -print(theta) +print("theta from own sdg") +print(theta) plt.plot(xnew, ypredict, "r-") plt.plot(xnew, ypredict2, "b-") @@ -1519,7 +1525,7 @@ plt.legend() plt.show() -print("The max absolute difference is: %g"%(np.max(np.abs(computed - analytic)))) +print("The max absolute difference is: %g"%(np.max(np.abs(computed - analytic))))

    @@ -1548,11 +1554,11 @@ f1_grad = grad(f1) a = 1.0 # See the evaluated gradient at a using autograd: -print("The gradient of f1 evaluated at a = %g using autograd is: %g"%(a,f1_grad(a))) +print("The gradient of f1 evaluated at a = %g using autograd is: %g"%(a,f1_grad(a))) # Compare with the analytical derivative, that is f1'(x) = 3*x**2 grad_analytical = 3*a**2 -print("The gradient of f1 evaluated at a = %g by finding the analytic expression is: %g"%(a,grad_analytical)) +print("The gradient of f1 evaluated at a = %g by finding the analytic expression is: %g"%(a,grad_analytical))











    @@ -1581,8 +1587,8 @@ f2_grad_x2 = grad(f2,1) x1 = 1.0 x2 = 3.0 -print("Evaluating at x1 = %g, x2 = %g"%(x1,x2)) -print("-"*30) +print("Evaluating at x1 = %g, x2 = %g"%(x1,x2)) +print("-"*30) # Compare with the analytical derivatives: @@ -1593,13 +1599,13 @@ f2_grad_x1_analytical = 9*x1**5 # See the evaluated derivations: -print("The derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) -print("The analytical derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) +print("The derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) +print("The analytical derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) -print() +print() -print("The derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) -print("The analytical derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) +print("The derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) +print("The analytical derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) ))

    Note that the grad function will not produce the true gradient of the function. The true gradient of a function with two or more variables will produce a vector, where each element is the function differentiated w.r.t a variable. @@ -1622,13 +1628,13 @@ f3_grad = grad(f3) x = np.linspace(0,4,5) # Print the computed gradient: -print("The computed gradient of f3 is: ", f3_grad(x)) +print("The computed gradient of f3 is: ", f3_grad(x)) # The analytical gradient is: (2, 3, 5, 7, 22*x[4]) f3_grad_analytical = np.array([2, 3, 5, 7, 22*x[4]]) # Print the analytical gradient: -print("The analytical gradient of f3 is: ", f3_grad_analytical) +print("The analytical gradient of f3 is: ", f3_grad_analytical)

    Note that in this case, when sending an array as input argument, the @@ -1656,13 +1662,13 @@ f4_grad = grad(f4) x = 2.7 # Print the computed derivative: -print("The computed derivative of f4 at x = %g is: %g"%(x,f4_grad(x))) +print("The computed derivative of f4 at x = %g is: %g"%(x,f4_grad(x))) # The analytical derivative is: x/sqrt(1 + x**2) + exp(x) + cos(2*pi*x)*2*pi f4_grad_analytical = x/np.sqrt(1 + x**2) + np.exp(x) + np.cos(2*np.pi*x)*2*np.pi # Print the analytical gradient: -print("The analytical gradient of f4 at x = %g is: %g"%(x,f4_grad_analytical)) +print("The analytical gradient of f4 at x = %g is: %g"%(x,f4_grad_analytical))











    @@ -1685,7 +1691,7 @@ f5_grad = grad(f5) x = 2.7 # Print the computed derivative: -print("The computed derivative of f5 at x = %g is: %g"%(x,f5_grad(x))) +print("The computed derivative of f5 at x = %g is: %g"%(x,f5_grad(x)))











    @@ -1717,8 +1723,8 @@ f6_while_grad = grad(f6_while) x = 0.5 # Print the computed derivaties of f6_for and f6_while -print("The computed derivative of f6_for at x = %g is: %g"%(x,f6_for_grad(x))) -print("The computed derivative of f6_while at x = %g is: %g"%(x,f6_while_grad(x))) +print("The computed derivative of f6_for at x = %g is: %g"%(x,f6_for_grad(x))) +print("The computed derivative of f6_while at x = %g is: %g"%(x,f6_while_grad(x)))

    @@ -1731,7 +1737,7 @@ f6_grad_analytical = 0 for i in range(10): f6_grad_analytical += i*x**(i-1) -print("The analytical derivative of f6 at x = %g is: %g"%(x,f6_grad_analytical)) +print("The analytical derivative of f6 at x = %g is: %g"%(x,f6_grad_analytical))











    @@ -1753,7 +1759,7 @@ f7_grad = grad(f7) n = 2.0 -print("The computed derivative of f7 at n = %d is: %g"%(n,f7_grad(n))) +print("The computed derivative of f7 at n = %d is: %g"%(n,f7_grad(n))) # The function f7 is an implementation of the factorial of n. # By using the product rule, one can find that the derivative is: @@ -1766,7 +1772,7 @@ f7_grad_analytical = 0 tmp *= (n - k) f7_grad_analytical += tmp -print("The analytical derivative of f7 at n = %d is: %g"%(n,f7_grad_analytical)) +print("The analytical derivative of f7 at n = %d is: %g"%(n,f7_grad_analytical))

    Note that if n is equal to zero or one, Autograd will give an error message. This message appears when the output is independent on input. @@ -1792,7 +1798,7 @@ f8_grad = grad(f8) x = 8.4 -print("The derivative of f8 is:",f8_grad(x)) +print("The derivative of f8 is:",f8_grad(x))

    Here, Autograd tells us that an 'ArrayBox' does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible. @@ -1814,7 +1820,7 @@ f9_grad = grad(f9) x = np.array([1.0,0.0]) -print("The derivative of f9 is:",f9_grad(x)) +print("The derivative of f9 is:",f9_grad(x))

    Here we are told that the 'dot' function does not belong to Autograd's @@ -1834,7 +1840,7 @@ f9_alternative_grad = grad(f9_alternative) x = np.array([3.0,0.0]) -print("The gradient of f9 is:",f9_alternative_grad(x)) +print("The gradient of f9 is:",f9_alternative_grad(x)) # The analytical gradient of the dot product of vectors x and b with two elements (x_1,x_2) and (b_1, b_2) respectively # w.r.t x is (b_1, b_2). @@ -2107,7 +2113,7 @@ $$ fig = pt.figure() ax = fig.gca(projection="3d") -xmesh, ymesh = np.mgrid[-2:2:50j,-2:2:50j] +xmesh, ymesh = np.mgrid[-2:2:50j,-2:2:50j] fmesh = f(np.array([xmesh, ymesh])) ax.plot_surface(xmesh, ymesh, fmesh) @@ -2139,7 +2145,7 @@ Run it! alpha_opt = sopt.golden(f1d) next_guess = x + alpha_opt * s guesses.append(next_guess) -print(next_guess) +print(next_guess)

    What happened? diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html index 06a81fc70..0ab20f229 100644 --- a/doc/pub/week39/html/week39.html +++ b/doc/pub/week39/html/week39.html @@ -221,7 +221,7 @@ MathJax.Hub.Config({

    [2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University

    -

    Sep 16, 2020

    +

    Sep 22, 2020












    @@ -233,10 +233,16 @@ MathJax.Hub.Config({

  • Friday: Stochastic Gradient descent with examples and automatic differeantion
    • +Reading suggestions for both days: Aurelien Geron's chapter 4 and Murphy sections 8.3 and 8.5 + +











    Thursday September 24

    +

    +"Overview Video, why do we care about gradient methods?":"" +











    @@ -800,7 +806,7 @@ y = 4+3* xb = np.c_[np.ones((m,1)), x] beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print(beta_linreg) +print(beta_linreg) beta = np.random.randn(2,1) eta = 0.1 @@ -810,7 +816,7 @@ Niterations = 1 gradients = 2.0/m*xb.T.dot(xb.dot(beta)-y) beta -= eta*gradients -print(beta) +print(beta) xnew = np.array([[0],[2]]) xbnew = np.c_[np.ones((2,1)), xnew] ypredict = xbnew.dot(beta) @@ -843,10 +849,10 @@ y = 4+3* xb = np.c_[np.ones((100,1)), x] beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print(beta_linreg) -sgdreg = SGDRegressor(n_iter = 50, penalty=None, eta0=0.1) +print(beta_linreg) +sgdreg = SGDRegressor(n_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) -print(sgdreg.intercept_, sgdreg.coef_) +print(sgdreg.intercept_, sgdreg.coef_)

    @@ -894,14 +900,14 @@ x = 2*np y = 4+3*x+np.random.randn(m,1) xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +XT_X = xb.T @ xb #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y -print(beta_linreg) +beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -909,12 +915,12 @@ eta = 0.1= 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta beta -= eta*gradients -print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +print(beta) +ypredict = xb @ beta +ypredict2 = xb @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1118,7 +1124,7 @@ j = 0 gamma_j = step_length(t,t0,t1) j += 1 -print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j)) +print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j))











    @@ -1141,12 +1147,12 @@ y = 4+3* xb = np.c_[np.ones((m,1)), x] theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) -print("Own inversion") -print(theta_linreg) -sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) +print("Own inversion") +print(theta_linreg) +sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) -print("sgdreg from scikit") -print(sgdreg.intercept_, sgdreg.coef_) +print("sgdreg from scikit") +print(sgdreg.intercept_, sgdreg.coef_) theta = np.random.randn(2,1) @@ -1155,10 +1161,10 @@ Niterations = 1 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*xb.T @ ((xb @ theta)-y) theta -= eta*gradients -print("theta frm own gd") -print(theta) +print("theta frm own gd") +print(theta) xnew = np.array([[0],[2]]) xbnew = np.c_[np.ones((2,1)), xnew] @@ -1178,11 +1184,11 @@ theta = np.= np.random.randint(m) xi = xb[random_index:random_index+1] yi = y[random_index:random_index+1] - gradients = 2 * xi.T @ ((xi @ theta)-yi) + gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients -print("theta from own sdg") -print(theta) +print("theta from own sdg") +print(theta) plt.plot(xnew, ypredict, "r-") plt.plot(xnew, ypredict2, "b-") @@ -1524,7 +1530,7 @@ plt.legend() plt.show() -print("The max absolute difference is: %g"%(np.max(np.abs(computed - analytic)))) +print("The max absolute difference is: %g"%(np.max(np.abs(computed - analytic))))

    @@ -1553,11 +1559,11 @@ f1_grad = grad(f1) a = 1.0 # See the evaluated gradient at a using autograd: -print("The gradient of f1 evaluated at a = %g using autograd is: %g"%(a,f1_grad(a))) +print("The gradient of f1 evaluated at a = %g using autograd is: %g"%(a,f1_grad(a))) # Compare with the analytical derivative, that is f1'(x) = 3*x**2 grad_analytical = 3*a**2 -print("The gradient of f1 evaluated at a = %g by finding the analytic expression is: %g"%(a,grad_analytical)) +print("The gradient of f1 evaluated at a = %g by finding the analytic expression is: %g"%(a,grad_analytical))











    @@ -1586,8 +1592,8 @@ f2_grad_x2 = grad(f2,= 1.0 x2 = 3.0 -print("Evaluating at x1 = %g, x2 = %g"%(x1,x2)) -print("-"*30) +print("Evaluating at x1 = %g, x2 = %g"%(x1,x2)) +print("-"*30) # Compare with the analytical derivatives: @@ -1598,13 +1604,13 @@ f2_grad_x1_analytical = = x1 - 5 # See the evaluated derivations: -print("The derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) -print("The analytical derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) +print("The derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) +print("The analytical derivative of f2 w.r.t x1: %g"%( f2_grad_x1(x1,x2) )) -print() +print() -print("The derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) -print("The analytical derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) +print("The derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) )) +print("The analytical derivative of f2 w.r.t x2: %g"%( f2_grad_x2(x1,x2) ))

    Note that the grad function will not produce the true gradient of the function. The true gradient of a function with two or more variables will produce a vector, where each element is the function differentiated w.r.t a variable. @@ -1627,13 +1633,13 @@ f3_grad = grad(f3) x = np.linspace(0,4,5) # Print the computed gradient: -print("The computed gradient of f3 is: ", f3_grad(x)) +print("The computed gradient of f3 is: ", f3_grad(x)) # The analytical gradient is: (2, 3, 5, 7, 22*x[4]) f3_grad_analytical = np.array([2, 3, 5, 7, 22*x[4]]) # Print the analytical gradient: -print("The analytical gradient of f3 is: ", f3_grad_analytical) +print("The analytical gradient of f3 is: ", f3_grad_analytical)

    Note that in this case, when sending an array as input argument, the @@ -1661,13 +1667,13 @@ f4_grad = grad(f4) x = 2.7 # Print the computed derivative: -print("The computed derivative of f4 at x = %g is: %g"%(x,f4_grad(x))) +print("The computed derivative of f4 at x = %g is: %g"%(x,f4_grad(x))) # The analytical derivative is: x/sqrt(1 + x**2) + exp(x) + cos(2*pi*x)*2*pi f4_grad_analytical = x/np.sqrt(1 + x**2) + np.exp(x) + np.cos(2*np.pi*x)*2*np.pi # Print the analytical gradient: -print("The analytical gradient of f4 at x = %g is: %g"%(x,f4_grad_analytical)) +print("The analytical gradient of f4 at x = %g is: %g"%(x,f4_grad_analytical))











    @@ -1690,7 +1696,7 @@ f5_grad = grad(f5) x = 2.7 # Print the computed derivative: -print("The computed derivative of f5 at x = %g is: %g"%(x,f5_grad(x))) +print("The computed derivative of f5 at x = %g is: %g"%(x,f5_grad(x)))











    @@ -1722,8 +1728,8 @@ f6_while_grad = grad(f6_while) x = 0.5 # Print the computed derivaties of f6_for and f6_while -print("The computed derivative of f6_for at x = %g is: %g"%(x,f6_for_grad(x))) -print("The computed derivative of f6_while at x = %g is: %g"%(x,f6_while_grad(x))) +print("The computed derivative of f6_for at x = %g is: %g"%(x,f6_for_grad(x))) +print("The computed derivative of f6_while at x = %g is: %g"%(x,f6_while_grad(x)))

    @@ -1736,7 +1742,7 @@ f6_grad_analytical = for i in range(10): f6_grad_analytical += i*x**(i-1) -print("The analytical derivative of f6 at x = %g is: %g"%(x,f6_grad_analytical)) +print("The analytical derivative of f6 at x = %g is: %g"%(x,f6_grad_analytical))











    @@ -1758,7 +1764,7 @@ f7_grad = grad(f7) n = 2.0 -print("The computed derivative of f7 at n = %d is: %g"%(n,f7_grad(n))) +print("The computed derivative of f7 at n = %d is: %g"%(n,f7_grad(n))) # The function f7 is an implementation of the factorial of n. # By using the product rule, one can find that the derivative is: @@ -1771,7 +1777,7 @@ f7_grad_analytical = *= (n - k) f7_grad_analytical += tmp -print("The analytical derivative of f7 at n = %d is: %g"%(n,f7_grad_analytical)) +print("The analytical derivative of f7 at n = %d is: %g"%(n,f7_grad_analytical))

    Note that if n is equal to zero or one, Autograd will give an error message. This message appears when the output is independent on input. @@ -1797,7 +1803,7 @@ f8_grad = grad(f8) x = 8.4 -print("The derivative of f8 is:",f8_grad(x)) +print("The derivative of f8 is:",f8_grad(x))

    Here, Autograd tells us that an 'ArrayBox' does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible. @@ -1819,7 +1825,7 @@ f9_grad = grad(f9) x = np.array([1.0,0.0]) -print("The derivative of f9 is:",f9_grad(x)) +print("The derivative of f9 is:",f9_grad(x))

    Here we are told that the 'dot' function does not belong to Autograd's @@ -1839,7 +1845,7 @@ f9_alternative_grad = grad(f9_alternative) x = np.array([3.0,0.0]) -print("The gradient of f9 is:",f9_alternative_grad(x)) +print("The gradient of f9 is:",f9_alternative_grad(x)) # The analytical gradient of the dot product of vectors x and b with two elements (x_1,x_2) and (b_1, b_2) respectively # w.r.t x is (b_1, b_2). @@ -2112,7 +2118,7 @@ $$ fig = pt.figure() ax = fig.gca(projection="3d") -xmesh, ymesh = np.mgrid[-2:2:50j,-2:2:50j] +xmesh, ymesh = np.mgrid[-2:2:50j,-2:2:50j] fmesh = f(np.array([xmesh, ymesh])) ax.plot_surface(xmesh, ymesh, fmesh) @@ -2144,7 +2150,7 @@ Run it! alpha_opt = sopt.golden(f1d) next_guess = x + alpha_opt * s guesses.append(next_guess) -print(next_guess) +print(next_guess)

    What happened? diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz index 8973b1f79..297e61eae 100644 Binary files a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz and b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz differ diff --git a/doc/pub/week39/ipynb/week39.ipynb b/doc/pub/week39/ipynb/week39.ipynb index 29d5ff2bc..fa04b5162 100644 --- a/doc/pub/week39/ipynb/week39.ipynb +++ b/doc/pub/week39/ipynb/week39.ipynb @@ -10,7 +10,7 @@ " \n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", "\n", - "Date: **Sep 16, 2020**\n", + "Date: **Sep 22, 2020**\n", "\n", "Copyright 1999-2020, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n", "\n", @@ -22,8 +22,12 @@ "\n", "* Friday: Stochastic Gradient descent with examples and automatic differeantion\n", "\n", + "Reading suggestions for both days: [Aurelien Geron's chapter 4](https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf) and [Murphy sections 8.3 and 8.5](https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/MachineLearningMurphy.pdf) \n", + "\n", "## Thursday September 24\n", "\n", + "\"Overview Video, why do we care about gradient methods?\":\"\"\n", + "\n", "## Optimization, the central part of any Machine Learning algortithm\n", "\n", "Almost every problem in machine learning and data science starts with\n", @@ -3039,5 +3043,5 @@ ], "metadata": {}, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/doc/src/week39/week39.do.txt b/doc/src/week39/week39.do.txt index 67f4d7fd4..b8936a83b 100644 --- a/doc/src/week39/week39.do.txt +++ b/doc/src/week39/week39.do.txt @@ -657,7 +657,6 @@ plt.show() !split ===== Friday September 25 ===== -"Overview Video, why stochastic gradient methods?":"" !split ===== Stochastic Gradient Descent =====