update on today's lectures

2021-10-21 08:10:45 +02:00
parent 8630bcf4b3
commit f263050011
101 changed files with 3251 additions and 1854 deletions
@@ -0,0 +1,44 @@
+
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.mplot3d import Axes3D
+from matplotlib import cm
+from matplotlib.ticker import LinearLocator, FormatStrFormatter
+import sys
+
+# the number of datapoints
+n = 100
+x = 2*np.random.rand(n,1)
+y = 4+3*x*x+np.random.randn(n,1)
+
+X = np.c_[np.ones((n,1)), x, x*x]
+XT_X = X.T @ X
+
+#Ridge parameter lambda
+lmbda  = 0.001
+Id = lmbda* np.eye(XT_X.shape[0])
+
+beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y
+print(beta_linreg)
+# Start plain gradient descent
+beta = np.random.randn(2,1)
+
+eta = 0.1
+Niterations = 100
+
+for iter in range(Niterations):
+    gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta
+    beta -= eta*gradients
+
+print(beta)
+ypredict = X @ beta
+ypredict2 = X @ beta_linreg
+plt.plot(x, ypredict, "r-")
+plt.plot(x, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Gradient descent example for Ridge')
+plt.show()
@@ -0,0 +1,77 @@
+"""
+Code to test Ridge and NNs using Scikit-Learn only
+"""
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.neural_network import MLPRegressor
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 5
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+# Decide which values of lambda to use
+
+nlambdas = 10
+lmbd_vals = np.logspace(-4, 0, nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+for i in range(nlambdas):
+    lmb = lmbd_vals[i]
+    RegRidge = linear_model.Ridge(lmb)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+
+plt.figure()
+plt.plot(np.log10(lmbd_vals), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+# Neural Network part
+
+n_hidden_neurons = 50
+epochs = 100
+# store models for later use
+eta_vals = np.logspace(-4, 0, 10)
+# store the models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+sns.set()
+for i, eta in enumerate(eta_vals):
+    for j, lmbd in enumerate(lmbd_vals):
+        dnn = MLPRegressor(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+                            alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+        dnn.fit(X_train, y_train)
+        ypredictMLP = dnn.predict(X_test)
+        test_accuracy[i][j] = MSE(ypredictMLP, y_test)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Training Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
@@ -0,0 +1,84 @@
+"""
+Code to test Ridge with own gradient descent and SGD
+"""
+
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.neural_network import MLPRegressor
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+import autograd.numpy as np
+from autograd import grad
+
+
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 5
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+
+nlambdas = 10
+lmbd_vals = np.logspace(-4, 0, nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+for i in range(nlambdas):
+    lmb = lmbd_vals[i]
+    RegRidge = linear_model.Ridge(lmb,fit_intercept=False)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+
+beta = np.random.randn(X_train.shape[1],1)
+loss = np.mean((y_train.reshape(-1,1) - X_train@beta)**2)
+print(loss)
+get_grad = grad(loss,argnum=2) 
+grad_beta = get_grad(X_train,y_train,beta) 
+#print(grad_beta)    
+
+"""
+print(beta)
+print( (X_train.T @ y_train).T)
+# Make own gradient descent and define precalculated quantities, saves cycles
+XT_X = X_train.T @ X_train
+XTy  = X_train.T @ y_train
+MSERidgeGDPredict = np.zeros(nlambdas)
+for i in range(nlambdas):
+    lmb = lmbd_vals[i]
+    Id = lmb* np.eye(XT_X.shape[0])
+    beta = np.random.randn(X_train.shape[1],1)
+    eta = 0.01
+    Niterations = 2    
+#    beta_linreg = np.linalg.pinv(XT_X+Id) @ X_train.T @ y_train
+    for iter in range(Niterations):
+        XX = XT_X @ beta-XTy
+        gradients = (2.0/n)*XX *lmb*beta
+        beta -= eta*gradients
+    ypredictRidgeGD = X_test @ beta
+    MSERidgeGDPredict[i] = MSE(y_test,ypredictRidgeGD)
+
+plt.figure()
+plt.plot(np.log10(lmbd_vals), MSERidgePredict, 'g--', label = 'MSE Sklearn Ridge Test')
+plt.plot(np.log10(lmbd_vals), MSERidgeGDPredict, 'r', label = 'MSE GD Ridge Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+"""
+