starting to update week 38
This commit is contained in:
Morten Hjorth-Jensen
@@ -6,15 +6,10 @@ DATE: today
|
||||
!split
|
||||
===== Plans for week 38 =====
|
||||
|
||||
* Thursday: Summary of regression methods and discussion of project 1. We revisit also cross-validation and bootstrap as resampling techniques with examples. Recommended reading: "Hastie et al":"https://www.springer.com/gp/book/9780387848570" chapters 3 and 7.1-7.6 and 7.10-7.12.
|
||||
* Friday: Logistic Regression. Recommended reading: "Hastie et al":"https://www.springer.com/gp/book/9780387848570" chapters 4.1-4.4 and "Murphy":"https://mitpress.mit.edu/books/machine-learning-1" chapter 8.1-8.2
|
||||
* Thursday: Summary of regression methods and discussion of project 1. Start Logistic Regression
|
||||
* Friday: Logistic Regression and Optimization methods
|
||||
|
||||
|
||||
!split
|
||||
===== Thursday September 17 =====
|
||||
|
||||
"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember17.mp4?vrtx=view-as-webpage" and "link to handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember17.pdf".
|
||||
|
||||
!split
|
||||
===== Ridge and LASSO Regression, reminder =====
|
||||
|
||||
@@ -240,192 +235,7 @@ plt.show()
|
||||
|
||||
|
||||
!split
|
||||
===== Bias-Variance tradeoff with Bootstrap =====
|
||||
!bc pycod
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.linear_model import LinearRegression, Ridge, Lasso
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.pipeline import make_pipeline
|
||||
from sklearn.utils import resample
|
||||
|
||||
np.random.seed(2018)
|
||||
|
||||
n = 40
|
||||
n_boostraps = 100
|
||||
maxdegree = 14
|
||||
|
||||
|
||||
# Make data set.
|
||||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||||
error = np.zeros(maxdegree)
|
||||
bias = np.zeros(maxdegree)
|
||||
variance = np.zeros(maxdegree)
|
||||
polydegree = np.zeros(maxdegree)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
|
||||
|
||||
for degree in range(maxdegree):
|
||||
model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
|
||||
y_pred = np.empty((y_test.shape[0], n_boostraps))
|
||||
for i in range(n_boostraps):
|
||||
x_, y_ = resample(x_train, y_train)
|
||||
y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()
|
||||
|
||||
polydegree[degree] = degree
|
||||
error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
|
||||
bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
|
||||
variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
|
||||
print('Polynomial degree:', degree)
|
||||
print('Error:', error[degree])
|
||||
print('Bias^2:', bias[degree])
|
||||
print('Var:', variance[degree])
|
||||
print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
|
||||
|
||||
plt.plot(polydegree, error, label='Error')
|
||||
plt.plot(polydegree, bias, label='bias')
|
||||
plt.plot(polydegree, variance, label='Variance')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
|
||||
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Another Example from Scikit-Learn's Repository =====
|
||||
!bc pycod
|
||||
"""
|
||||
============================
|
||||
Underfitting vs. Overfitting
|
||||
============================
|
||||
|
||||
This example demonstrates the problems of underfitting and overfitting and
|
||||
how we can use linear regression with polynomial features to approximate
|
||||
nonlinear functions. The plot shows the function that we want to approximate,
|
||||
which is a part of the cosine function. In addition, the samples from the
|
||||
real function and the approximations of different models are displayed. The
|
||||
models have polynomial features of different degrees. We can see that a
|
||||
linear function (polynomial with degree 1) is not sufficient to fit the
|
||||
training samples. This is called **underfitting**. A polynomial of degree 4
|
||||
approximates the true function almost perfectly. However, for higher degrees
|
||||
the model will **overfit** the training data, i.e. it learns the noise of the
|
||||
training data.
|
||||
We evaluate quantitatively **overfitting** / **underfitting** by using
|
||||
cross-validation. We calculate the mean squared error (MSE) on the validation
|
||||
set, the higher, the less likely the model generalizes correctly from the
|
||||
training data.
|
||||
"""
|
||||
|
||||
print(__doc__)
|
||||
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.pipeline import Pipeline
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.linear_model import LinearRegression
|
||||
from sklearn.model_selection import cross_val_score
|
||||
|
||||
|
||||
def true_fun(X):
|
||||
return np.cos(1.5 * np.pi * X)
|
||||
|
||||
np.random.seed(0)
|
||||
|
||||
n_samples = 30
|
||||
degrees = [1, 4, 15]
|
||||
|
||||
X = np.sort(np.random.rand(n_samples))
|
||||
y = true_fun(X) + np.random.randn(n_samples) * 0.1
|
||||
|
||||
plt.figure(figsize=(14, 5))
|
||||
for i in range(len(degrees)):
|
||||
ax = plt.subplot(1, len(degrees), i + 1)
|
||||
plt.setp(ax, xticks=(), yticks=())
|
||||
|
||||
polynomial_features = PolynomialFeatures(degree=degrees[i],
|
||||
include_bias=False)
|
||||
linear_regression = LinearRegression()
|
||||
pipeline = Pipeline([("polynomial_features", polynomial_features),
|
||||
("linear_regression", linear_regression)])
|
||||
pipeline.fit(X[:, np.newaxis], y)
|
||||
|
||||
# Evaluate the models using crossvalidation
|
||||
scores = cross_val_score(pipeline, X[:, np.newaxis], y,
|
||||
scoring="neg_mean_squared_error", cv=10)
|
||||
|
||||
X_test = np.linspace(0, 1, 100)
|
||||
plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model")
|
||||
plt.plot(X_test, true_fun(X_test), label="True function")
|
||||
plt.scatter(X, y, edgecolor='b', s=20, label="Samples")
|
||||
plt.xlabel("x")
|
||||
plt.ylabel("y")
|
||||
plt.xlim((0, 1))
|
||||
plt.ylim((-2, 2))
|
||||
plt.legend(loc="best")
|
||||
plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format(
|
||||
degrees[i], -scores.mean(), scores.std()))
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Cross-validation with Ridge =====
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import KFold
|
||||
from sklearn.linear_model import Ridge
|
||||
from sklearn.model_selection import cross_val_score
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
|
||||
# A seed just to ensure that the random numbers are the same for every run.
|
||||
np.random.seed(3155)
|
||||
# Generate the data.
|
||||
n = 100
|
||||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||||
# Decide degree on polynomial to fit
|
||||
poly = PolynomialFeatures(degree = 10)
|
||||
|
||||
# Decide which values of lambda to use
|
||||
nlambdas = 500
|
||||
lambdas = np.logspace(-3, 5, nlambdas)
|
||||
# Initialize a KFold instance
|
||||
k = 5
|
||||
kfold = KFold(n_splits = k)
|
||||
estimated_mse_sklearn = np.zeros(nlambdas)
|
||||
i = 0
|
||||
for lmb in lambdas:
|
||||
ridge = Ridge(alpha = lmb)
|
||||
estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
|
||||
estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
|
||||
i += 1
|
||||
plt.figure()
|
||||
plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
|
||||
plt.xlabel('log10(lambda)')
|
||||
plt.ylabel('MSE')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== The Ising model =====
|
||||
===== More complicated Example: The Ising model =====
|
||||
|
||||
The one-dimensional Ising model with nearest neighbor interaction, no
|
||||
external field and a constant coupling constant $J$ is given by
|
||||
@@ -914,16 +724,6 @@ other models for all values of $\lambda$.
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Friday September 18: Intro to Logistic Regression =====
|
||||
|
||||
"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureSeptember18.mp4?vrtx=view-as-webpage" and "link to handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember18.pdf".
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Logistic Regression =====
|
||||
|
||||
|
||||
Reference in New Issue
Block a user