From e398b8b62fbb413e6b18324cc3090650969be01f Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Tue, 21 Sep 2021 16:04:03 +0200 Subject: [PATCH] update week38 --- doc/pub/week38/html/._week38-bs000.html | 7 +- doc/pub/week38/html/._week38-bs001.html | 7 +- doc/pub/week38/html/._week38-bs002.html | 7 +- doc/pub/week38/html/._week38-bs003.html | 7 +- doc/pub/week38/html/._week38-bs004.html | 7 +- doc/pub/week38/html/._week38-bs005.html | 7 +- doc/pub/week38/html/._week38-bs006.html | 7 +- doc/pub/week38/html/._week38-bs007.html | 332 ++++++++++++- doc/pub/week38/html/._week38-bs008.html | 7 +- doc/pub/week38/html/._week38-bs009.html | 7 +- doc/pub/week38/html/._week38-bs010.html | 7 +- doc/pub/week38/html/._week38-bs011.html | 7 +- doc/pub/week38/html/._week38-bs012.html | 7 +- doc/pub/week38/html/._week38-bs013.html | 7 +- doc/pub/week38/html/._week38-bs014.html | 7 +- doc/pub/week38/html/._week38-bs015.html | 7 +- doc/pub/week38/html/._week38-bs016.html | 7 +- doc/pub/week38/html/._week38-bs017.html | 7 +- doc/pub/week38/html/._week38-bs018.html | 7 +- doc/pub/week38/html/._week38-bs019.html | 7 +- doc/pub/week38/html/._week38-bs020.html | 7 +- doc/pub/week38/html/._week38-bs021.html | 7 +- doc/pub/week38/html/._week38-bs022.html | 7 +- doc/pub/week38/html/._week38-bs023.html | 7 +- doc/pub/week38/html/._week38-bs024.html | 7 +- doc/pub/week38/html/._week38-bs025.html | 7 +- doc/pub/week38/html/._week38-bs026.html | 7 +- doc/pub/week38/html/._week38-bs027.html | 7 +- doc/pub/week38/html/._week38-bs028.html | 7 +- doc/pub/week38/html/._week38-bs029.html | 7 +- doc/pub/week38/html/._week38-bs030.html | 7 +- doc/pub/week38/html/._week38-bs031.html | 7 +- doc/pub/week38/html/._week38-bs032.html | 7 +- doc/pub/week38/html/._week38-bs033.html | 7 +- doc/pub/week38/html/._week38-bs034.html | 7 +- doc/pub/week38/html/._week38-bs035.html | 7 +- doc/pub/week38/html/._week38-bs036.html | 7 +- doc/pub/week38/html/._week38-bs037.html | 7 +- doc/pub/week38/html/._week38-bs038.html | 7 +- doc/pub/week38/html/week38-bs.html | 7 +- doc/pub/week38/html/week38-reveal.html | 347 +++++++++++++- doc/pub/week38/html/week38-solarized.html | 330 ++++++++++++- doc/pub/week38/html/week38.html | 330 ++++++++++++- doc/pub/week38/ipynb/ipynb-week38-src.tar.gz | Bin 192 -> 192 bytes doc/pub/week38/ipynb/week38.ipynb | 469 +++++++++++++++++- doc/src/week36/scale2.py | 4 +- doc/src/week36/scale3.py | 100 ++++ doc/src/week38/week38.do.txt | 480 +++++++++++++------ 48 files changed, 2429 insertions(+), 236 deletions(-) create mode 100644 doc/src/week36/scale3.py diff --git a/doc/pub/week38/html/._week38-bs000.html b/doc/pub/week38/html/._week38-bs000.html index ea3f3b366..9ae29ebbb 100644 --- a/doc/pub/week38/html/._week38-bs000.html +++ b/doc/pub/week38/html/._week38-bs000.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs001.html b/doc/pub/week38/html/._week38-bs001.html index 1b5179726..beed80885 100644 --- a/doc/pub/week38/html/._week38-bs001.html +++ b/doc/pub/week38/html/._week38-bs001.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs002.html b/doc/pub/week38/html/._week38-bs002.html index 63d3b2bf5..53aa1b96e 100644 --- a/doc/pub/week38/html/._week38-bs002.html +++ b/doc/pub/week38/html/._week38-bs002.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs003.html b/doc/pub/week38/html/._week38-bs003.html index b2ee881bb..d3a32fae6 100644 --- a/doc/pub/week38/html/._week38-bs003.html +++ b/doc/pub/week38/html/._week38-bs003.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs004.html b/doc/pub/week38/html/._week38-bs004.html index 25c952f9e..b18ee80fa 100644 --- a/doc/pub/week38/html/._week38-bs004.html +++ b/doc/pub/week38/html/._week38-bs004.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs005.html b/doc/pub/week38/html/._week38-bs005.html index 8b3915eba..0ecb77d1d 100644 --- a/doc/pub/week38/html/._week38-bs005.html +++ b/doc/pub/week38/html/._week38-bs005.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs006.html b/doc/pub/week38/html/._week38-bs006.html index aba052d64..c03c94679 100644 --- a/doc/pub/week38/html/._week38-bs006.html +++ b/doc/pub/week38/html/._week38-bs006.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs007.html b/doc/pub/week38/html/._week38-bs007.html index 08007745f..dd37227aa 100644 --- a/doc/pub/week38/html/._week38-bs007.html +++ b/doc/pub/week38/html/._week38-bs007.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • @@ -234,11 +237,153 @@ MathJax.Hub.Config({ -

    To think about

    +

    To think about, first part

    -When you are comparing your own code with for example Scikit-Learn's library, there are some minor things to keep in mind. -The example here shows how one can keep the intercept in order to compare own code. +When you are comparing your own code with for example Scikit-Learn's +library, there are some things to keep in mind. The examples +here demonstrate some of these aspects with potential pitfalls. + +

    +The discussion here focuses on the role of the intercept, how we can +set up the design matrix, what scaling we should use and other topics +which may confuse us. + +

    +Yes, it could be a bad idea to include the intercept column for the +exact reason you stated. If no transformation is applied to your data, +the intercept can be interpreted as the expected value of your target +variable when all your predictors are put to zero. Therefore, whenever +you cannot assume that the expected target variable is zero when all +your predictors are zero, it could be a bad idea to apply a model +which penalizes the intercept. Also, the analytical solution to the +ridge regression coefficients (when not shrinking $$\beta_0$$) is +derived under the assumption that both y and X are zero centered (mean +subtracted). What you are doing is correct, but you should also zero +center X (subtracting the mean of each column from the corresponding +column).  + +

    +If your predictors are of different scales, I would advice you to +standardize X by subtracting the mean of each column from the +corresponding column and dividing the column with its standard +deviation. If you dont do this, you will give an "unfair" penalization +of the parameters since their magnitude depends on the scale of their +corresponding predictor. Suppose that you have an input variable +"height". Human height might be measured in inches or meters or +kilometers. If measured in kilometers, a standard linear regression +model with this predictor would probably give a much bigger +coefficient term, than if measured in millimeters. You may see how +this could become a problem when considering the loss function for +ridge regression. + +

    +Remember that when you do any transformation to your dataset before +training, the exact same transformation has to be applied to new data +before making a prediction. In your case, this means: + +

    + + +

    #Model training:
+y_train_mean = np.mean(y_train)
+X_train_mean = np.mean(X_train,axis=0)
+X_train = X_train - X_train_mean
+y_train = y_train - y_train_mean
+
+trained_model = some_model.fit(X_train,y_train)
+
+#Model prediction:
+X_test = X_test - X_train_mean #Use mean from training data
+y_pred = trained_model(X_test)
+y_pred = y_pred + y_train_mean
+
    +

    +Here is a mathematical explanation of the zero centering: + +

    +The cost/loss function for Ridge regression is: + +$$ +C(\beta_0, \beta_1, ... , \beta_P) = \sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip}\beta_p)^2 + \lambda \sum_{p=1}^P \beta_p^2. +$$ + +

    +Notice that the intercept is left out of the \( L_2 \) regularization term. The design matrix +\( X \) does in this case not contain any intercept column. We want + +$$ +\frac{\partial L}{\partial \beta_j} = 0, +$$ + +

    +for all \( j \), so lets start with \( \beta_0 \). This means that we have + +$$ +\frac{\partial L}{\partial \beta_0} = -2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p). +$$ + +

    +We want to solve +$$ +-2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p) = 0, +$$ + +

    +which gives +$$ +\sum_{i=1}^{n} \beta_0 = \sum_{i=1}^{n}y_i - \sum_{i=1}^{n} \sum_{p=1}^P X_{ip} \beta_p, +$$ + +

    +or +$ n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}$. + +

    +If we assume that every column of \( X \) is centered, whic we can do by subtracting the mean, +

    + + +

    X = X - np.mean(X,axis=0)
+
    +

    +the sum $ \sum_{i=1}^{n} X_{ip} $ + +

    +can be rewritten as +$$ +\sum_{i=1}^{n} (X_{ip} - \frac{1}{n}\sum_{i=1}^{n} X_{ip}) = \sum_{i=1}^{n} X_{ip} - \sum_{i=1}^{n} \frac{1}{n} \sum_{i=1}^{n}X_{ip}, +$$ + +resulting in +$$ +\sum_{i=1}^{n} X_{ip} - n \frac{1}{n} \sum_{i=1}^{n}X_{ip} = 0. +$$ + +

    +Finally we have +$$ +n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}, +$$ + +or +$$ +\beta_0 = \frac{1}{n}\sum_{i=1}^{n} y_i = y_{average}. +$$ + +

    +Replacing \( y_i \) with \( y_i - \beta_0 = y_i - y_{average} \) in the loss function will give us (in vector-matrix disguise) +$$ +C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}) + \lambda \boldsymbol{\beta}^T\boldsymbol{\beta}, +$$ + +

    +which has the solution + +

    +\( \beta = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}} \). +where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - y_{average} \) +and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=1}^{n-1}X_{kj} \).

    @@ -327,6 +472,183 @@ plt.plot(np..xlabel('log10(lambda)') plt.ylabel('MSE') plt.legend() +plt.show() + +

    + + +

    import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.preprocessing import StandardScaler
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 5
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+
+
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+
+
+
+
+#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable
+X_train_mean = np.mean(X_train,axis=0)
+X_train_scaled = X_train - X_train_mean #Center by removing mean from each feature
+X_test_scaled = X_test - X_train_mean
+
+y_scaler = np.mean(y_train)           #The model intercept (called y_scaler) is given by the mean of target variable (IF X is centered)
+y_train_scaled = y_train - y_scaler   #Remove the intercept from the training data.
+
+
+p = Maxpolydegree-1
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 4
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+
+lambdas = np.logspace(-4, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)
+    intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data
+    
+    ypredictOwnRidge = X_test @ OwnRidgeBeta + intercept_ #Add intercept to prediction
+    #EQUIVALENT PREDICTION:
+    ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler #Add intercept to prediction
+    print("Values for own Ridge prediction")
+    print(ypredictOwnRidge)
+
+    
+
+    RegRidge = linear_model.Ridge(lmb)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    print("Values for SL Ridge prediction")
+    print(ypredictRidge)
+
+
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+
+    print("Beta values for own Ridge implementation")
+    print(OwnRidgeBeta) #Intercept is given by mean of target variable
+    print("Beta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print('Intercept from own implementation:')
+    print(intercept_)
+    print('Intercept from Scikit-Learn Ridge implementation')
+    print(RegRidge.intercept_)
+
+
+
+# Now plot the results
+
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
    +

    + + +

    import numpy as np
+import matplotlib.pyplot as plt
+
+from sklearn.linear_model import LinearRegression
+
+
+np.random.seed(2021)
+
+
+def fit_beta(X, y):
+    return np.linalg.pinv(X.T @ X) @ X.T @ y
+
+
+true_beta = [2, 0.5, 3.7]
+
+x = np.linspace(0, 1, 11)
+y = np.sum(
+    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
+) + 0.1 * np.random.normal(size=len(x))
+
+degree = 3
+X = np.zeros((len(x), degree))
+
+# Include the intercept in the design matrix
+for p in range(degree):
+    X[:, p] = x ** p
+
+beta = fit_beta(X, y)
+
+# Intercept is included in the design matrix
+clf = LinearRegression(fit_intercept=False).fit(X, y)
+
+print(f"True beta: {true_beta}")
+print(f"Fitted beta: {beta}")
+print(f"Sklearn fitted beta: {clf.coef_}")
+
+
+plt.figure()
+plt.scatter(x, y, label="Data")
+plt.plot(x, X @ beta, label="Fit")
+plt.plot(x, clf.predict(X), label="Sklearn (fit_intercept=False)")
+
+
+# Do not include the intercept in the design matrix
+X = np.zeros((len(x), degree - 1))
+
+for p in range(degree - 1):
+    X[:, p] = x ** (p + 1)
+
+# Intercept is not included in the design matrix
+clf = LinearRegression(fit_intercept=True).fit(X, y)
+
+# Use centered values for X and y when computing coefficients
+y_offset = np.average(y, axis=0)
+X_offset = np.average(X, axis=0)
+
+beta = fit_beta(X - X_offset, y - y_offset)
+intercept = np.mean(y_offset - X_offset @ beta)
+
+print(f"Manual intercept: {intercept}")
+print(f"Fitted beta (sans intercept): {beta}")
+print(f"Sklearn intercept: {clf.intercept_}")
+print(f"Sklearn fitted beta (sans intercept): {clf.coef_}")
+
+plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
+plt.plot(x, clf.predict(X), "--", label="Sklearn (fit_intercept=True)")
+plt.grid()
+plt.legend()
+
 plt.show()

    diff --git a/doc/pub/week38/html/._week38-bs008.html b/doc/pub/week38/html/._week38-bs008.html index 0363531dc..432efdae1 100644 --- a/doc/pub/week38/html/._week38-bs008.html +++ b/doc/pub/week38/html/._week38-bs008.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({

  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs009.html b/doc/pub/week38/html/._week38-bs009.html index a70cdb1a5..fdf5ca32d 100644 --- a/doc/pub/week38/html/._week38-bs009.html +++ b/doc/pub/week38/html/._week38-bs009.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs010.html b/doc/pub/week38/html/._week38-bs010.html index 83edd32f4..14abd4885 100644 --- a/doc/pub/week38/html/._week38-bs010.html +++ b/doc/pub/week38/html/._week38-bs010.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs011.html b/doc/pub/week38/html/._week38-bs011.html index 62f9e0be1..0ff5d178c 100644 --- a/doc/pub/week38/html/._week38-bs011.html +++ b/doc/pub/week38/html/._week38-bs011.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs012.html b/doc/pub/week38/html/._week38-bs012.html index 1c0c3f55c..eceb92ea3 100644 --- a/doc/pub/week38/html/._week38-bs012.html +++ b/doc/pub/week38/html/._week38-bs012.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs013.html b/doc/pub/week38/html/._week38-bs013.html index 7b74c3d07..823d063a8 100644 --- a/doc/pub/week38/html/._week38-bs013.html +++ b/doc/pub/week38/html/._week38-bs013.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs014.html b/doc/pub/week38/html/._week38-bs014.html index 5ab1015a6..3aa6a42f3 100644 --- a/doc/pub/week38/html/._week38-bs014.html +++ b/doc/pub/week38/html/._week38-bs014.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs015.html b/doc/pub/week38/html/._week38-bs015.html index 9cbaf2ba1..50c489cf1 100644 --- a/doc/pub/week38/html/._week38-bs015.html +++ b/doc/pub/week38/html/._week38-bs015.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs016.html b/doc/pub/week38/html/._week38-bs016.html index c5ccba0b1..068b9da2e 100644 --- a/doc/pub/week38/html/._week38-bs016.html +++ b/doc/pub/week38/html/._week38-bs016.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs017.html b/doc/pub/week38/html/._week38-bs017.html index d1e72a95e..6b46482d7 100644 --- a/doc/pub/week38/html/._week38-bs017.html +++ b/doc/pub/week38/html/._week38-bs017.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs018.html b/doc/pub/week38/html/._week38-bs018.html index a677677a6..4582d24f7 100644 --- a/doc/pub/week38/html/._week38-bs018.html +++ b/doc/pub/week38/html/._week38-bs018.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs019.html b/doc/pub/week38/html/._week38-bs019.html index a54a8f863..c17b37d05 100644 --- a/doc/pub/week38/html/._week38-bs019.html +++ b/doc/pub/week38/html/._week38-bs019.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs020.html b/doc/pub/week38/html/._week38-bs020.html index dccdc57e7..cf7240d7a 100644 --- a/doc/pub/week38/html/._week38-bs020.html +++ b/doc/pub/week38/html/._week38-bs020.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs021.html b/doc/pub/week38/html/._week38-bs021.html index 1d1525021..f047e072b 100644 --- a/doc/pub/week38/html/._week38-bs021.html +++ b/doc/pub/week38/html/._week38-bs021.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs022.html b/doc/pub/week38/html/._week38-bs022.html index 6197cf7c8..4766e6b66 100644 --- a/doc/pub/week38/html/._week38-bs022.html +++ b/doc/pub/week38/html/._week38-bs022.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs023.html b/doc/pub/week38/html/._week38-bs023.html index 7523d7429..844883928 100644 --- a/doc/pub/week38/html/._week38-bs023.html +++ b/doc/pub/week38/html/._week38-bs023.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs024.html b/doc/pub/week38/html/._week38-bs024.html index 28a04ceac..cd61d3c47 100644 --- a/doc/pub/week38/html/._week38-bs024.html +++ b/doc/pub/week38/html/._week38-bs024.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs025.html b/doc/pub/week38/html/._week38-bs025.html index f1d5ce2fb..f9daf0b2a 100644 --- a/doc/pub/week38/html/._week38-bs025.html +++ b/doc/pub/week38/html/._week38-bs025.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs026.html b/doc/pub/week38/html/._week38-bs026.html index 529eb67a1..ce0681e0b 100644 --- a/doc/pub/week38/html/._week38-bs026.html +++ b/doc/pub/week38/html/._week38-bs026.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs027.html b/doc/pub/week38/html/._week38-bs027.html index 38b9db190..750a7fa05 100644 --- a/doc/pub/week38/html/._week38-bs027.html +++ b/doc/pub/week38/html/._week38-bs027.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs028.html b/doc/pub/week38/html/._week38-bs028.html index 1338b682e..cddda738c 100644 --- a/doc/pub/week38/html/._week38-bs028.html +++ b/doc/pub/week38/html/._week38-bs028.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs029.html b/doc/pub/week38/html/._week38-bs029.html index 1d0de7d76..bad6f5b87 100644 --- a/doc/pub/week38/html/._week38-bs029.html +++ b/doc/pub/week38/html/._week38-bs029.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs030.html b/doc/pub/week38/html/._week38-bs030.html index da5d094c0..332ee8a13 100644 --- a/doc/pub/week38/html/._week38-bs030.html +++ b/doc/pub/week38/html/._week38-bs030.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs031.html b/doc/pub/week38/html/._week38-bs031.html index ff988575f..607fb9db8 100644 --- a/doc/pub/week38/html/._week38-bs031.html +++ b/doc/pub/week38/html/._week38-bs031.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs032.html b/doc/pub/week38/html/._week38-bs032.html index e7b51bb27..9580b3a82 100644 --- a/doc/pub/week38/html/._week38-bs032.html +++ b/doc/pub/week38/html/._week38-bs032.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs033.html b/doc/pub/week38/html/._week38-bs033.html index a941b452f..ebf1825a0 100644 --- a/doc/pub/week38/html/._week38-bs033.html +++ b/doc/pub/week38/html/._week38-bs033.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs034.html b/doc/pub/week38/html/._week38-bs034.html index ddb01022d..fa3d8cd68 100644 --- a/doc/pub/week38/html/._week38-bs034.html +++ b/doc/pub/week38/html/._week38-bs034.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs035.html b/doc/pub/week38/html/._week38-bs035.html index b0212be0b..b9c0e467f 100644 --- a/doc/pub/week38/html/._week38-bs035.html +++ b/doc/pub/week38/html/._week38-bs035.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs036.html b/doc/pub/week38/html/._week38-bs036.html index f2fc821ea..183febb49 100644 --- a/doc/pub/week38/html/._week38-bs036.html +++ b/doc/pub/week38/html/._week38-bs036.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs037.html b/doc/pub/week38/html/._week38-bs037.html index 36cda34bf..8df23b1bb 100644 --- a/doc/pub/week38/html/._week38-bs037.html +++ b/doc/pub/week38/html/._week38-bs037.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/._week38-bs038.html b/doc/pub/week38/html/._week38-bs038.html index e143fa415..c2df66b5e 100644 --- a/doc/pub/week38/html/._week38-bs038.html +++ b/doc/pub/week38/html/._week38-bs038.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/week38-bs.html b/doc/pub/week38/html/week38-bs.html index ea3f3b366..9ae29ebbb 100644 --- a/doc/pub/week38/html/week38-bs.html +++ b/doc/pub/week38/html/week38-bs.html @@ -63,7 +63,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -187,7 +190,7 @@ MathJax.Hub.Config({
  • How to set up the cross-validation for Ridge and/or Lasso
  • Cross-validation in brief
  • Code Example for Cross-validation and \( k \)-fold Cross-validation
    • -
  • To think about
    • +
  • To think about, first part
  • More complicated Example: The Ising model
  • Reformulating the problem to suit regression
  • Linear regression
    • diff --git a/doc/pub/week38/html/week38-reveal.html b/doc/pub/week38/html/week38-reveal.html index 261aba841..55ac915a9 100644 --- a/doc/pub/week38/html/week38-reveal.html +++ b/doc/pub/week38/html/week38-reveal.html @@ -419,11 +419,175 @@ plt.show()
    -

    To think about

    +

    To think about, first part

    -When you are comparing your own code with for example Scikit-Learn's library, there are some minor things to keep in mind. -The example here shows how one can keep the intercept in order to compare own code. +When you are comparing your own code with for example Scikit-Learn's +library, there are some things to keep in mind. The examples +here demonstrate some of these aspects with potential pitfalls. + +

    +The discussion here focuses on the role of the intercept, how we can +set up the design matrix, what scaling we should use and other topics +which may confuse us. + +

    +Yes, it could be a bad idea to include the intercept column for the +exact reason you stated. If no transformation is applied to your data, +the intercept can be interpreted as the expected value of your target +variable when all your predictors are put to zero. Therefore, whenever +you cannot assume that the expected target variable is zero when all +your predictors are zero, it could be a bad idea to apply a model +which penalizes the intercept. Also, the analytical solution to the +ridge regression coefficients (when not shrinking

     
    +$$\beta_0$$ +

     
    ) is +derived under the assumption that both y and X are zero centered (mean +subtracted). What you are doing is correct, but you should also zero +center X (subtracting the mean of each column from the corresponding +column).  + +

    +If your predictors are of different scales, I would advice you to +standardize X by subtracting the mean of each column from the +corresponding column and dividing the column with its standard +deviation. If you dont do this, you will give an "unfair" penalization +of the parameters since their magnitude depends on the scale of their +corresponding predictor. Suppose that you have an input variable +"height". Human height might be measured in inches or meters or +kilometers. If measured in kilometers, a standard linear regression +model with this predictor would probably give a much bigger +coefficient term, than if measured in millimeters. You may see how +this could become a problem when considering the loss function for +ridge regression. + +

    +Remember that when you do any transformation to your dataset before +training, the exact same transformation has to be applied to new data +before making a prediction. In your case, this means: + +

    + + +

    #Model training:
+y_train_mean = np.mean(y_train)
+X_train_mean = np.mean(X_train,axis=0)
+X_train = X_train - X_train_mean
+y_train = y_train - y_train_mean
+
+trained_model = some_model.fit(X_train,y_train)
+
+#Model prediction:
+X_test = X_test - X_train_mean #Use mean from training data
+y_pred = trained_model(X_test)
+y_pred = y_pred + y_train_mean
+
    +

    +Here is a mathematical explanation of the zero centering: + +

    +The cost/loss function for Ridge regression is: + +

     
    +$$ +C(\beta_0, \beta_1, ... , \beta_P) = \sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip}\beta_p)^2 + \lambda \sum_{p=1}^P \beta_p^2. +$$ +

     
    + +

    +Notice that the intercept is left out of the \( L_2 \) regularization term. The design matrix +\( X \) does in this case not contain any intercept column. We want + +

     
    +$$ +\frac{\partial L}{\partial \beta_j} = 0, +$$ +

     
    + +

    +for all \( j \), so lets start with \( \beta_0 \). This means that we have + +

     
    +$$ +\frac{\partial L}{\partial \beta_0} = -2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p). +$$ +

     
    + +

    +We want to solve +

     
    +$$ +-2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p) = 0, +$$ +

     
    + +

    +which gives +

     
    +$$ +\sum_{i=1}^{n} \beta_0 = \sum_{i=1}^{n}y_i - \sum_{i=1}^{n} \sum_{p=1}^P X_{ip} \beta_p, +$$ +

     
    + +

    +or +$ n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}$. + +

    +If we assume that every column of \( X \) is centered, whic we can do by subtracting the mean, +

    + + +

    X = X - np.mean(X,axis=0)
+
    +

    +the sum $ \sum_{i=1}^{n} X_{ip} $ + +

    +can be rewritten as +

     
    +$$ +\sum_{i=1}^{n} (X_{ip} - \frac{1}{n}\sum_{i=1}^{n} X_{ip}) = \sum_{i=1}^{n} X_{ip} - \sum_{i=1}^{n} \frac{1}{n} \sum_{i=1}^{n}X_{ip}, +$$ +

     
    + +resulting in +

     
    +$$ +\sum_{i=1}^{n} X_{ip} - n \frac{1}{n} \sum_{i=1}^{n}X_{ip} = 0. +$$ +

     
    + +

    +Finally we have +

     
    +$$ +n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}, +$$ +

     
    + +or +

     
    +$$ +\beta_0 = \frac{1}{n}\sum_{i=1}^{n} y_i = y_{average}. +$$ +

     
    + +

    +Replacing \( y_i \) with \( y_i - \beta_0 = y_i - y_{average} \) in the loss function will give us (in vector-matrix disguise) +

     
    +$$ +C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}) + \lambda \boldsymbol{\beta}^T\boldsymbol{\beta}, +$$ +

     
    + +

    +which has the solution + +

    +\( \beta = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}} \). +where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - y_{average} \) +and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=1}^{n-1}X_{kj} \).

    @@ -512,6 +676,183 @@ plt.plot(np.log10(lambdas), MSERidgePredict, 'g plt.xlabel('log10(lambda)') plt.ylabel('MSE') plt.legend() +plt.show() + +

    + + +

    import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.preprocessing import StandardScaler
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 5
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+
+
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+
+
+
+
+#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable
+X_train_mean = np.mean(X_train,axis=0)
+X_train_scaled = X_train - X_train_mean #Center by removing mean from each feature
+X_test_scaled = X_test - X_train_mean
+
+y_scaler = np.mean(y_train)           #The model intercept (called y_scaler) is given by the mean of target variable (IF X is centered)
+y_train_scaled = y_train - y_scaler   #Remove the intercept from the training data.
+
+
+p = Maxpolydegree-1
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 4
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+
+lambdas = np.logspace(-4, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)
+    intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data
+    
+    ypredictOwnRidge = X_test @ OwnRidgeBeta + intercept_ #Add intercept to prediction
+    #EQUIVALENT PREDICTION:
+    ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler #Add intercept to prediction
+    print("Values for own Ridge prediction")
+    print(ypredictOwnRidge)
+
+    
+
+    RegRidge = linear_model.Ridge(lmb)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    print("Values for SL Ridge prediction")
+    print(ypredictRidge)
+
+
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+
+    print("Beta values for own Ridge implementation")
+    print(OwnRidgeBeta) #Intercept is given by mean of target variable
+    print("Beta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print('Intercept from own implementation:')
+    print(intercept_)
+    print('Intercept from Scikit-Learn Ridge implementation')
+    print(RegRidge.intercept_)
+
+
+
+# Now plot the results
+
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
    +

    + + +

    import numpy as np
+import matplotlib.pyplot as plt
+
+from sklearn.linear_model import LinearRegression
+
+
+np.random.seed(2021)
+
+
+def fit_beta(X, y):
+    return np.linalg.pinv(X.T @ X) @ X.T @ y
+
+
+true_beta = [2, 0.5, 3.7]
+
+x = np.linspace(0, 1, 11)
+y = np.sum(
+    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
+) + 0.1 * np.random.normal(size=len(x))
+
+degree = 3
+X = np.zeros((len(x), degree))
+
+# Include the intercept in the design matrix
+for p in range(degree):
+    X[:, p] = x ** p
+
+beta = fit_beta(X, y)
+
+# Intercept is included in the design matrix
+clf = LinearRegression(fit_intercept=False).fit(X, y)
+
+print(f"True beta: {true_beta}")
+print(f"Fitted beta: {beta}")
+print(f"Sklearn fitted beta: {clf.coef_}")
+
+
+plt.figure()
+plt.scatter(x, y, label="Data")
+plt.plot(x, X @ beta, label="Fit")
+plt.plot(x, clf.predict(X), label="Sklearn (fit_intercept=False)")
+
+
+# Do not include the intercept in the design matrix
+X = np.zeros((len(x), degree - 1))
+
+for p in range(degree - 1):
+    X[:, p] = x ** (p + 1)
+
+# Intercept is not included in the design matrix
+clf = LinearRegression(fit_intercept=True).fit(X, y)
+
+# Use centered values for X and y when computing coefficients
+y_offset = np.average(y, axis=0)
+X_offset = np.average(X, axis=0)
+
+beta = fit_beta(X - X_offset, y - y_offset)
+intercept = np.mean(y_offset - X_offset @ beta)
+
+print(f"Manual intercept: {intercept}")
+print(f"Fitted beta (sans intercept): {beta}")
+print(f"Sklearn intercept: {clf.intercept_}")
+print(f"Sklearn fitted beta (sans intercept): {clf.coef_}")
+
+plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
+plt.plot(x, clf.predict(X), "--", label="Sklearn (fit_intercept=True)")
+plt.grid()
+plt.legend()
+
 plt.show()
    diff --git a/doc/pub/week38/html/week38-solarized.html b/doc/pub/week38/html/week38-solarized.html index 0c7d4a431..fb4f3fd78 100644 --- a/doc/pub/week38/html/week38-solarized.html +++ b/doc/pub/week38/html/week38-solarized.html @@ -57,7 +57,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -426,11 +429,153 @@ plt.show()











    -

    To think about

    +

    To think about, first part

    -When you are comparing your own code with for example Scikit-Learn's library, there are some minor things to keep in mind. -The example here shows how one can keep the intercept in order to compare own code. +When you are comparing your own code with for example Scikit-Learn's +library, there are some things to keep in mind. The examples +here demonstrate some of these aspects with potential pitfalls. + +

    +The discussion here focuses on the role of the intercept, how we can +set up the design matrix, what scaling we should use and other topics +which may confuse us. + +

    +Yes, it could be a bad idea to include the intercept column for the +exact reason you stated. If no transformation is applied to your data, +the intercept can be interpreted as the expected value of your target +variable when all your predictors are put to zero. Therefore, whenever +you cannot assume that the expected target variable is zero when all +your predictors are zero, it could be a bad idea to apply a model +which penalizes the intercept. Also, the analytical solution to the +ridge regression coefficients (when not shrinking $$\beta_0$$) is +derived under the assumption that both y and X are zero centered (mean +subtracted). What you are doing is correct, but you should also zero +center X (subtracting the mean of each column from the corresponding +column).  + +

    +If your predictors are of different scales, I would advice you to +standardize X by subtracting the mean of each column from the +corresponding column and dividing the column with its standard +deviation. If you dont do this, you will give an "unfair" penalization +of the parameters since their magnitude depends on the scale of their +corresponding predictor. Suppose that you have an input variable +"height". Human height might be measured in inches or meters or +kilometers. If measured in kilometers, a standard linear regression +model with this predictor would probably give a much bigger +coefficient term, than if measured in millimeters. You may see how +this could become a problem when considering the loss function for +ridge regression. + +

    +Remember that when you do any transformation to your dataset before +training, the exact same transformation has to be applied to new data +before making a prediction. In your case, this means: + +

    + + +

    #Model training:
+y_train_mean = np.mean(y_train)
+X_train_mean = np.mean(X_train,axis=0)
+X_train = X_train - X_train_mean
+y_train = y_train - y_train_mean
+
+trained_model = some_model.fit(X_train,y_train)
+
+#Model prediction:
+X_test = X_test - X_train_mean #Use mean from training data
+y_pred = trained_model(X_test)
+y_pred = y_pred + y_train_mean
+
    +

    +Here is a mathematical explanation of the zero centering: + +

    +The cost/loss function for Ridge regression is: + +$$ +C(\beta_0, \beta_1, ... , \beta_P) = \sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip}\beta_p)^2 + \lambda \sum_{p=1}^P \beta_p^2. +$$ + +

    +Notice that the intercept is left out of the \( L_2 \) regularization term. The design matrix +\( X \) does in this case not contain any intercept column. We want + +$$ +\frac{\partial L}{\partial \beta_j} = 0, +$$ + +

    +for all \( j \), so lets start with \( \beta_0 \). This means that we have + +$$ +\frac{\partial L}{\partial \beta_0} = -2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p). +$$ + +

    +We want to solve +$$ +-2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p) = 0, +$$ + +

    +which gives +$$ +\sum_{i=1}^{n} \beta_0 = \sum_{i=1}^{n}y_i - \sum_{i=1}^{n} \sum_{p=1}^P X_{ip} \beta_p, +$$ + +

    +or +$ n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}$. + +

    +If we assume that every column of \( X \) is centered, whic we can do by subtracting the mean, +

    + + +

    X = X - np.mean(X,axis=0)
+
    +

    +the sum $ \sum_{i=1}^{n} X_{ip} $ + +

    +can be rewritten as +$$ +\sum_{i=1}^{n} (X_{ip} - \frac{1}{n}\sum_{i=1}^{n} X_{ip}) = \sum_{i=1}^{n} X_{ip} - \sum_{i=1}^{n} \frac{1}{n} \sum_{i=1}^{n}X_{ip}, +$$ + +resulting in +$$ +\sum_{i=1}^{n} X_{ip} - n \frac{1}{n} \sum_{i=1}^{n}X_{ip} = 0. +$$ + +

    +Finally we have +$$ +n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}, +$$ + +or +$$ +\beta_0 = \frac{1}{n}\sum_{i=1}^{n} y_i = y_{average}. +$$ + +

    +Replacing \( y_i \) with \( y_i - \beta_0 = y_i - y_{average} \) in the loss function will give us (in vector-matrix disguise) +$$ +C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}) + \lambda \boldsymbol{\beta}^T\boldsymbol{\beta}, +$$ + +

    +which has the solution + +

    +\( \beta = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}} \). +where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - y_{average} \) +and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=1}^{n-1}X_{kj} \).

    @@ -519,6 +664,183 @@ plt.plot(np.log10(lambdas), MSERidgePredict, 'g plt.xlabel('log10(lambda)') plt.ylabel('MSE') plt.legend() +plt.show() + +

    + + +

    import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.preprocessing import StandardScaler
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 5
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+
+
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+
+
+
+
+#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable
+X_train_mean = np.mean(X_train,axis=0)
+X_train_scaled = X_train - X_train_mean #Center by removing mean from each feature
+X_test_scaled = X_test - X_train_mean
+
+y_scaler = np.mean(y_train)           #The model intercept (called y_scaler) is given by the mean of target variable (IF X is centered)
+y_train_scaled = y_train - y_scaler   #Remove the intercept from the training data.
+
+
+p = Maxpolydegree-1
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 4
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+
+lambdas = np.logspace(-4, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)
+    intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data
+    
+    ypredictOwnRidge = X_test @ OwnRidgeBeta + intercept_ #Add intercept to prediction
+    #EQUIVALENT PREDICTION:
+    ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler #Add intercept to prediction
+    print("Values for own Ridge prediction")
+    print(ypredictOwnRidge)
+
+    
+
+    RegRidge = linear_model.Ridge(lmb)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    print("Values for SL Ridge prediction")
+    print(ypredictRidge)
+
+
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+
+    print("Beta values for own Ridge implementation")
+    print(OwnRidgeBeta) #Intercept is given by mean of target variable
+    print("Beta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print('Intercept from own implementation:')
+    print(intercept_)
+    print('Intercept from Scikit-Learn Ridge implementation')
+    print(RegRidge.intercept_)
+
+
+
+# Now plot the results
+
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
    +

    + + +

    import numpy as np
+import matplotlib.pyplot as plt
+
+from sklearn.linear_model import LinearRegression
+
+
+np.random.seed(2021)
+
+
+def fit_beta(X, y):
+    return np.linalg.pinv(X.T @ X) @ X.T @ y
+
+
+true_beta = [2, 0.5, 3.7]
+
+x = np.linspace(0, 1, 11)
+y = np.sum(
+    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
+) + 0.1 * np.random.normal(size=len(x))
+
+degree = 3
+X = np.zeros((len(x), degree))
+
+# Include the intercept in the design matrix
+for p in range(degree):
+    X[:, p] = x ** p
+
+beta = fit_beta(X, y)
+
+# Intercept is included in the design matrix
+clf = LinearRegression(fit_intercept=False).fit(X, y)
+
+print(f"True beta: {true_beta}")
+print(f"Fitted beta: {beta}")
+print(f"Sklearn fitted beta: {clf.coef_}")
+
+
+plt.figure()
+plt.scatter(x, y, label="Data")
+plt.plot(x, X @ beta, label="Fit")
+plt.plot(x, clf.predict(X), label="Sklearn (fit_intercept=False)")
+
+
+# Do not include the intercept in the design matrix
+X = np.zeros((len(x), degree - 1))
+
+for p in range(degree - 1):
+    X[:, p] = x ** (p + 1)
+
+# Intercept is not included in the design matrix
+clf = LinearRegression(fit_intercept=True).fit(X, y)
+
+# Use centered values for X and y when computing coefficients
+y_offset = np.average(y, axis=0)
+X_offset = np.average(X, axis=0)
+
+beta = fit_beta(X - X_offset, y - y_offset)
+intercept = np.mean(y_offset - X_offset @ beta)
+
+print(f"Manual intercept: {intercept}")
+print(f"Fitted beta (sans intercept): {beta}")
+print(f"Sklearn intercept: {clf.intercept_}")
+print(f"Sklearn fitted beta (sans intercept): {clf.coef_}")
+
+plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
+plt.plot(x, clf.predict(X), "--", label="Sklearn (fit_intercept=True)")
+plt.grid()
+plt.legend()
+
 plt.show()

    diff --git a/doc/pub/week38/html/week38.html b/doc/pub/week38/html/week38.html index dfcb42880..54cd59e22 100644 --- a/doc/pub/week38/html/week38.html +++ b/doc/pub/week38/html/week38.html @@ -62,7 +62,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'code-example-for-cross-validation-and-k-fold-cross-validation'), - ('To think about', 2, None, 'to-think-about'), + ('To think about, first part', + 2, + None, + 'to-think-about-first-part'), ('More complicated Example: The Ising model', 2, None, @@ -431,11 +434,153 @@ plt.show()











    -

    To think about

    +

    To think about, first part

    -When you are comparing your own code with for example Scikit-Learn's library, there are some minor things to keep in mind. -The example here shows how one can keep the intercept in order to compare own code. +When you are comparing your own code with for example Scikit-Learn's +library, there are some things to keep in mind. The examples +here demonstrate some of these aspects with potential pitfalls. + +

    +The discussion here focuses on the role of the intercept, how we can +set up the design matrix, what scaling we should use and other topics +which may confuse us. + +

    +Yes, it could be a bad idea to include the intercept column for the +exact reason you stated. If no transformation is applied to your data, +the intercept can be interpreted as the expected value of your target +variable when all your predictors are put to zero. Therefore, whenever +you cannot assume that the expected target variable is zero when all +your predictors are zero, it could be a bad idea to apply a model +which penalizes the intercept. Also, the analytical solution to the +ridge regression coefficients (when not shrinking $$\beta_0$$) is +derived under the assumption that both y and X are zero centered (mean +subtracted). What you are doing is correct, but you should also zero +center X (subtracting the mean of each column from the corresponding +column).  + +

    +If your predictors are of different scales, I would advice you to +standardize X by subtracting the mean of each column from the +corresponding column and dividing the column with its standard +deviation. If you dont do this, you will give an "unfair" penalization +of the parameters since their magnitude depends on the scale of their +corresponding predictor. Suppose that you have an input variable +"height". Human height might be measured in inches or meters or +kilometers. If measured in kilometers, a standard linear regression +model with this predictor would probably give a much bigger +coefficient term, than if measured in millimeters. You may see how +this could become a problem when considering the loss function for +ridge regression. + +

    +Remember that when you do any transformation to your dataset before +training, the exact same transformation has to be applied to new data +before making a prediction. In your case, this means: + +

    + + +

    #Model training:
+y_train_mean = np.mean(y_train)
+X_train_mean = np.mean(X_train,axis=0)
+X_train = X_train - X_train_mean
+y_train = y_train - y_train_mean
+
+trained_model = some_model.fit(X_train,y_train)
+
+#Model prediction:
+X_test = X_test - X_train_mean #Use mean from training data
+y_pred = trained_model(X_test)
+y_pred = y_pred + y_train_mean
+
    +

    +Here is a mathematical explanation of the zero centering: + +

    +The cost/loss function for Ridge regression is: + +$$ +C(\beta_0, \beta_1, ... , \beta_P) = \sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip}\beta_p)^2 + \lambda \sum_{p=1}^P \beta_p^2. +$$ + +

    +Notice that the intercept is left out of the \( L_2 \) regularization term. The design matrix +\( X \) does in this case not contain any intercept column. We want + +$$ +\frac{\partial L}{\partial \beta_j} = 0, +$$ + +

    +for all \( j \), so lets start with \( \beta_0 \). This means that we have + +$$ +\frac{\partial L}{\partial \beta_0} = -2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p). +$$ + +

    +We want to solve +$$ +-2\sum_{i=1}^{n} (y_i - \beta_0 - \sum_{p=1}^P X_{ip} \beta_p) = 0, +$$ + +

    +which gives +$$ +\sum_{i=1}^{n} \beta_0 = \sum_{i=1}^{n}y_i - \sum_{i=1}^{n} \sum_{p=1}^P X_{ip} \beta_p, +$$ + +

    +or +$ n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}$. + +

    +If we assume that every column of \( X \) is centered, whic we can do by subtracting the mean, +

    + + +

    X = X - np.mean(X,axis=0)
+
    +

    +the sum $ \sum_{i=1}^{n} X_{ip} $ + +

    +can be rewritten as +$$ +\sum_{i=1}^{n} (X_{ip} - \frac{1}{n}\sum_{i=1}^{n} X_{ip}) = \sum_{i=1}^{n} X_{ip} - \sum_{i=1}^{n} \frac{1}{n} \sum_{i=1}^{n}X_{ip}, +$$ + +resulting in +$$ +\sum_{i=1}^{n} X_{ip} - n \frac{1}{n} \sum_{i=1}^{n}X_{ip} = 0. +$$ + +

    +Finally we have +$$ +n\beta_0 = \sum_{i=1}^{n} y_i - \sum_{p=1}^P\beta_p \sum_{i=1}^{n} X_{ip}, +$$ + +or +$$ +\beta_0 = \frac{1}{n}\sum_{i=1}^{n} y_i = y_{average}. +$$ + +

    +Replacing \( y_i \) with \( y_i - \beta_0 = y_i - y_{average} \) in the loss function will give us (in vector-matrix disguise) +$$ +C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}) + \lambda \boldsymbol{\beta}^T\boldsymbol{\beta}, +$$ + +

    +which has the solution + +

    +\( \beta = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}} \). +where \( \boldsymbol{\tilde{y}} = \boldsymbol{y} - y_{average} \) +and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=1}^{n-1}X_{kj} \).

    @@ -524,6 +669,183 @@ plt.plot(np..xlabel('log10(lambda)') plt.ylabel('MSE') plt.legend() +plt.show() + +

    + + +

    import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+from sklearn.preprocessing import StandardScaler
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(315)
+
+n = 100
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)
+
+Maxpolydegree = 5
+X = np.zeros((n,Maxpolydegree-1))
+
+for degree in range(1,Maxpolydegree): #No intercept column
+    X[:,degree-1] = x**(degree)
+
+
+
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+
+
+
+
+#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable
+X_train_mean = np.mean(X_train,axis=0)
+X_train_scaled = X_train - X_train_mean #Center by removing mean from each feature
+X_test_scaled = X_test - X_train_mean
+
+y_scaler = np.mean(y_train)           #The model intercept (called y_scaler) is given by the mean of target variable (IF X is centered)
+y_train_scaled = y_train - y_scaler   #Remove the intercept from the training data.
+
+
+p = Maxpolydegree-1
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 4
+MSEOwnRidgePredict = np.zeros(nlambdas)
+MSERidgePredict = np.zeros(nlambdas)
+
+lambdas = np.logspace(-4, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    OwnRidgeBeta = np.linalg.pinv(X_train_scaled.T @ X_train_scaled+lmb*I) @ X_train_scaled.T @ (y_train_scaled)
+    intercept_ = y_scaler - X_train_mean@OwnRidgeBeta #The intercept can be shifted so the model can predict on uncentered data
+    
+    ypredictOwnRidge = X_test @ OwnRidgeBeta + intercept_ #Add intercept to prediction
+    #EQUIVALENT PREDICTION:
+    ypredictOwnRidge = X_test_scaled @ OwnRidgeBeta + y_scaler #Add intercept to prediction
+    print("Values for own Ridge prediction")
+    print(ypredictOwnRidge)
+
+    
+
+    RegRidge = linear_model.Ridge(lmb)
+    RegRidge.fit(X_train,y_train)
+    ypredictRidge = RegRidge.predict(X_test)
+    print("Values for SL Ridge prediction")
+    print(ypredictRidge)
+
+
+    MSEOwnRidgePredict[i] = MSE(y_test,ypredictOwnRidge)
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+
+    print("Beta values for own Ridge implementation")
+    print(OwnRidgeBeta) #Intercept is given by mean of target variable
+    print("Beta values for Scikit-Learn Ridge implementation")
+    print(RegRidge.coef_)
+    print('Intercept from own implementation:')
+    print(intercept_)
+    print('Intercept from Scikit-Learn Ridge implementation')
+    print(RegRidge.intercept_)
+
+
+
+# Now plot the results
+
+plt.figure()
+plt.plot(np.log10(lambdas), MSEOwnRidgePredict, 'b--', label = 'MSE own Ridge Test')
+plt.plot(np.log10(lambdas), MSERidgePredict, 'g--', label = 'MSE SL Ridge Test')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
    +

    + + +

    import numpy as np
+import matplotlib.pyplot as plt
+
+from sklearn.linear_model import LinearRegression
+
+
+np.random.seed(2021)
+
+
+def fit_beta(X, y):
+    return np.linalg.pinv(X.T @ X) @ X.T @ y
+
+
+true_beta = [2, 0.5, 3.7]
+
+x = np.linspace(0, 1, 11)
+y = np.sum(
+    np.asarray([x ** p * b for p, b in enumerate(true_beta)]), axis=0
+) + 0.1 * np.random.normal(size=len(x))
+
+degree = 3
+X = np.zeros((len(x), degree))
+
+# Include the intercept in the design matrix
+for p in range(degree):
+    X[:, p] = x ** p
+
+beta = fit_beta(X, y)
+
+# Intercept is included in the design matrix
+clf = LinearRegression(fit_intercept=False).fit(X, y)
+
+print(f"True beta: {true_beta}")
+print(f"Fitted beta: {beta}")
+print(f"Sklearn fitted beta: {clf.coef_}")
+
+
+plt.figure()
+plt.scatter(x, y, label="Data")
+plt.plot(x, X @ beta, label="Fit")
+plt.plot(x, clf.predict(X), label="Sklearn (fit_intercept=False)")
+
+
+# Do not include the intercept in the design matrix
+X = np.zeros((len(x), degree - 1))
+
+for p in range(degree - 1):
+    X[:, p] = x ** (p + 1)
+
+# Intercept is not included in the design matrix
+clf = LinearRegression(fit_intercept=True).fit(X, y)
+
+# Use centered values for X and y when computing coefficients
+y_offset = np.average(y, axis=0)
+X_offset = np.average(X, axis=0)
+
+beta = fit_beta(X - X_offset, y - y_offset)
+intercept = np.mean(y_offset - X_offset @ beta)
+
+print(f"Manual intercept: {intercept}")
+print(f"Fitted beta (sans intercept): {beta}")
+print(f"Sklearn intercept: {clf.intercept_}")
+print(f"Sklearn fitted beta (sans intercept): {clf.coef_}")
+
+plt.plot(x, X @ beta + intercept, "--", label="Fit (manual intercept)")
+plt.plot(x, clf.predict(X), "--", label="Sklearn (fit_intercept=True)")
+plt.grid()
+plt.legend()
+
 plt.show()

    diff --git a/doc/pub/week38/ipynb/ipynb-week38-src.tar.gz b/doc/pub/week38/ipynb/ipynb-week38-src.tar.gz index f5c6a0d61ca517983d0eeeb892a60f6492edf9e4..41fc69dc0dc4f2b4652f80ac8dd3a033aafb0c41 100644 GIT binary patch delta 155 zcmV;M0A&Bb0l)!ZABzY87UoG|00ZsMOA5j;5C-6`dx|+h$)l}q1-oz|h+IYaS0ovk4Drs3V;2K=Xsv@wFhG& J?oa>-000VmK_>tJ delta 155 zcmV;M0A&Bb0l)!ZABzY8Zfr?m00ZsM%?iRW3