updating GD slides

2019-09-26 14:09:41 +02:00
parent 042f60741d
commit 1a012e5421
83 changed files with 5795 additions and 5772 deletions
@@ -0,0 +1,9 @@
+from sklearn.datasets import load_iris
+from sklearn import datasets, linear_model
+from sklearn.linear_model import LogisticRegressionCV
+X, y = datasets.make_moons(200, noise=0.20)
+#X, y = load_iris(return_X_y=True)
+clf = LogisticRegressionCV(cv=5, random_state=0,multi_class='multinomial').fit(X, y)
+#clf.predict(X[:2, :])
+#clf.predict_proba(X[:2, :]).shape
+print(clf.score(X, y) )
@@ -815,10 +815,11 @@ import numpy as np
 import matplotlib.pyplot as plt
 from sklearn.linear_model import SGDRegressor

-x = 2*np.random.rand(100,1)
-y = 4+3*x+np.random.randn(100,1)
+m = 100
+x = 2*np.random.rand(m,1)
+y = 4+3*x+np.random.randn(m,1)

-xb = np.c_[np.ones((100,1)), x]
+xb = np.c_[np.ones((m,1)), x]
 theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
 print("Own inversion")
 print(theta_linreg)
@@ -829,10 +830,9 @@ print(sgdreg.intercept_, sgdreg.coef_)


 theta = np.random.randn(2,1)
-
 eta = 0.1
 Niterations = 1000
-m = 100
+

 for iter in range(Niterations):
    gradients = 2.0/m*xb.T @ ((xb @ theta)-y)
@@ -848,7 +848,6 @@ ypredict2 = xbnew.dot(theta_linreg)

 n_epochs = 50
 t0, t1 = 5, 50
-m = 100
 def learning_schedule(t):
    return t0/(t+t1)

@@ -876,10 +875,7 @@ plt.show()

 !ec

-
-!split
-===== Logistic Regression example =====
-
+_Challenge_: try to write a similar code for a Logistic Regression case.


 !split
@@ -1028,6 +1024,33 @@ Geron's text, see chapter 11, has several interesting discussions.

 !split
 ===== Automatic differentiation =====
+
+"Automatic differentiation (AD)":"https://en.wikipedia.org/wiki/Automatic_differentiation", 
+also called algorithmic
+differentiation or computational differentiation,is a set of
+techniques to numerically evaluate the derivative of a function
+specified by a computer program. AD exploits the fact that every
+computer program, no matter how complicated, executes a sequence of
+elementary arithmetic operations (addition, subtraction,
+multiplication, division, etc.) and elementary functions (exp, log,
+sin, cos, etc.). By applying the chain rule repeatedly to these
+operations, derivatives of arbitrary order can be computed
+automatically, accurately to working precision, and using at most a
+small constant factor more arithmetic operations than the original
+program.
+
+Automatic differentiation is neither:
+
+* Symbolic differentiation, nor
+* Numerical differentiation (the method of finite differences).
+
+Symbolic differentiation can lead to inefficient code and faces the
+difficulty of converting a computer program into a single expression,
+while numerical differentiation can introduce round-off errors in the
+discretization process and cancellation
+
+
+
 Python has tools for so-called _automatic differentiation_.
 Consider the following example
 !bt
@@ -1,59 +1,41 @@
-from math import exp, sqrt
-from random import random, seed
 import numpy as np
 from sklearn import datasets, linear_model
 import matplotlib.pyplot as plt
-from sklearn.linear_model import SGDRegressor

-np.random.seed(0)
-X, y = datasets.make_moons(200, noise=0.20)
-#X = np.c_[np.ones((100,1)), x]

-sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)
-sgdreg.fit(X,y.ravel())
-print("sgdreg from scikit")
-print(sgdreg.intercept_, sgdreg.coef_)
+def generate_data():
+    np.random.seed(0)
+    X, y = datasets.make_moons(200, noise=0.20)
+    return X, y

-"""
-theta = np.random.randn(2,1)
-eta = 0.1
-Niterations = 1000
-m = 100

-for iter in range(Niterations):
-    gradients = 2.0/m*X.T @ ((X @ theta)-y)
-    theta -= eta*gradients
-print("theta from own gd")
-print(theta)
+def visualize(X, y, clf):
+    plot_decision_boundary(lambda x: clf.predict(x), X, y)

-xnew = np.array([[0],[2]])
-Xnew = np.c_[np.ones((2,1)), xnew]
-ypredict = Xnew.dot(theta)
+def plot_decision_boundary(pred_func, X, y):
+    # Set min and max values and give it some padding
+    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
+    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
+    h = 0.01
+    # Generate a grid of points with distance h between them
+    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
+    # Predict the function value for the whole gid
+    Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
+    Z = Z.reshape(xx.shape)
+    # Plot the contour and training examples
+    plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
+    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral)
+    plt.show()

-n_epochs = 50
-t0, t1 = 5, 50
-m = 100
-def learning_schedule(t):
-    return t0/(t+t1)

-theta = np.random.randn(2,1)
+def classify(X, y):
+    clf = linear_model.LogisticRegressionCV()
+    clf.fit(X, y)
+    return clf

-for epoch in range(n_epochs):
-    for i in range(m):
-        random_index = np.random.randint(m)
-        xi = X[random_index:random_index+1]
-        yi = y[random_index:random_index+1]
-        gradients = 2 * xi.T @ ((xi @ theta)-yi)
-        eta = learning_schedule(epoch*m+i)
-        theta = theta - eta*gradients
-print("theta from own sdg")
-print(theta)

-plt.plot(xnew, ypredict, "r-")
-plt.plot(x, y ,'ro')
-plt.axis([0,2.0,0, 15.0])
-plt.xlabel(r'$x$')
-plt.ylabel(r'$y$')
-plt.title(r'Random numbers ')
-plt.show()
-"""
+
+X, y = generate_data()
+clf = classify(X, y)
+visualize(X, y, clf)
+