added codes and more

doc/src/week39/codes/test10.py

@@ -1,44 +1,52 @@
+"""
+Code to test Ridge with own gradient descent and SGD
+"""
 
-from random import random, seed
 import numpy as np
+import pandas as pd
 import matplotlib.pyplot as plt
-from mpl_toolkits.mplot3d import Axes3D
-from matplotlib import cm
-from matplotlib.ticker import LinearLocator, FormatStrFormatter
-import sys
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.neural_network import MLPRegressor
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+import autograd.numpy as np
+from autograd import grad
+
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
 
-# the number of datapoints
 n = 100
-x = 2*np.random.rand(n,1)
-y = 4+3*x*x+np.random.randn(n,1)
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
 
-X = np.c_[np.ones((n,1)), x, x*x]
-XT_X = X.T @ X
+Maxpolydegree = 5
+X = np.zeros((n,Maxpolydegree-1))
 
-#Ridge parameter lambda
-lmbda  = 0.001
-Id = lmbda* np.eye(XT_X.shape[0])
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
 
-beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y
-print(beta_linreg)
-# Start plain gradient descent
-beta = np.random.randn(2,1)
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
 
-eta = 0.1
-Niterations = 100
 
-for iter in range(Niterations):
-    gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta
-    beta -= eta*gradients
+nlambdas = 10
+lmbd_vals = np.logspace(-4, 0, nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+for i in range(nlambdas):
+    lmb = lmbd_vals[i]
+    RegRidge = linear_model.Ridge(lmb,fit_intercept=False)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
 
-print(beta)
-ypredict = X @ beta
-ypredict2 = X @ beta_linreg
-plt.plot(x, ypredict, "r-")
-plt.plot(x, ypredict2, "b-")
-plt.plot(x, y ,'ro')
-plt.axis([0,2.0,0, 15.0])
-plt.xlabel(r'$x$')
-plt.ylabel(r'$y$')
-plt.title(r'Gradient descent example for Ridge')
-plt.show()
+beta = np.random.randn(X_train.shape[1],1)
+loss = np.mean((y_train.reshape(-1,1) - X_train@beta)**2)
+print(loss)
+get_grad = grad(loss,argnum=2) 
+grad_beta = get_grad(X_train,y_train,beta)