diff --git a/doc/pub/How2ReadData/ipynb/How2ReadData.ipynb b/doc/pub/How2ReadData/ipynb/How2ReadData.ipynb index f184de74e..f44a3fc0f 100644 --- a/doc/pub/How2ReadData/ipynb/How2ReadData.ipynb +++ b/doc/pub/How2ReadData/ipynb/How2ReadData.ipynb @@ -2689,9 +2689,20 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "' \\nThis is taken from the data file of the mass 2016 evaluation. \\nAll files are 3436 lines long with 124 character per line. \\n Headers are 39 lines long. \\n col 1 : Fortran character control: 1 = page feed 0 = line feed \\n format : a1,i3,i5,i5,i5,1x,a3,a4,1x,f13.5,f11.5,f11.3,f9.3,1x,a2,f11.3,f9.3,1x,i3,1x,f12.5,f11.5 \\n These formats are reflected in the pandas widths variable below, see the statement \\n widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1), \\n Pandas has also a variable header, with length 39 in this case. \\n'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "\"\"\" \n", "This is taken from the data file of the mass 2016 evaluation. \n", @@ -2717,7 +2728,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -2759,28 +2770,26 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " N Z A Element Ebinding\n", - "A \n", - "1 0 0 1 1 H 0.000000\n", - "2 1 1 1 2 H 1.112283\n", - "3 2 2 1 3 H 2.827265\n", - "4 6 2 2 4 He 7.073915\n", - "5 9 3 2 5 He 5.512132\n", - "... ... ... ... ... ...\n", - "264 3304 156 108 264 Hs 7.298375\n", - "265 3310 157 108 265 Hs 7.296247\n", - "266 3317 158 108 266 Hs 7.298273\n", - "269 3338 159 110 269 Ds 7.250154\n", - "270 3344 160 110 270 Ds 7.253775\n", - "\n", - "[267 rows x 5 columns]\n" + "A \n", + "1 0 1\n", + "2 1 2\n", + "3 2 3\n", + "4 6 4\n", + "5 9 5\n", + " ... \n", + "264 3304 264\n", + "265 3310 265\n", + "266 3317 266\n", + "269 3338 269\n", + "270 3344 270\n", + "Name: A, Length: 267, dtype: int64\n" ] } ], @@ -2790,7 +2799,9 @@ "N = Masses['N']\n", "Element = Masses['Element']\n", "Energies = Masses['Ebinding']\n", - "print(Masses)" + "#print(Masses)\n", + "xx = A\n", + "print(xx)" ] }, { diff --git a/doc/pub/week38/html/._week38-bs000.html b/doc/pub/week38/html/._week38-bs000.html index 0d1b2580c..dc41103ec 100644 --- a/doc/pub/week38/html/._week38-bs000.html +++ b/doc/pub/week38/html/._week38-bs000.html @@ -1,11 +1,11 @@
- + @@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source @@ -311,94 +218,94 @@ MathJax.Hub.Config({
@@ -446,7 +353,7 @@ MathJax.Hub.Config({ diff --git a/doc/pub/week38/html/._week38-bs003.html b/doc/pub/week38/html/._week38-bs003.html index df89ceb8e..d642a66eb 100644 --- a/doc/pub/week38/html/._week38-bs003.html +++ b/doc/pub/week38/html/._week38-bs003.html @@ -1,11 +1,11 @@
- + @@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source @@ -311,94 +218,94 @@ MathJax.Hub.Config({The expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is @@ -500,7 +407,7 @@ $$ diff --git a/doc/pub/week38/html/._week38-bs004.html b/doc/pub/week38/html/._week38-bs004.html index 20753a5ea..33f6020a8 100644 --- a/doc/pub/week38/html/._week38-bs004.html +++ b/doc/pub/week38/html/._week38-bs004.html @@ -1,11 +1,11 @@
- + @@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source @@ -311,94 +218,94 @@ MathJax.Hub.Config({When the repetitive splitting of the data set is done randomly, @@ -464,7 +371,7 @@ cross-validation (LOOCV). diff --git a/doc/pub/week38/html/._week38-bs005.html b/doc/pub/week38/html/._week38-bs005.html index 6f681210b..94b5c643a 100644 --- a/doc/pub/week38/html/._week38-bs005.html +++ b/doc/pub/week38/html/._week38-bs005.html @@ -1,11 +1,11 @@
- + @@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source @@ -311,94 +218,94 @@ MathJax.Hub.Config({For the various values of \( k \) @@ -467,7 +374,7 @@ For the various values of \( k \) diff --git a/doc/pub/week38/html/._week38-bs007.html b/doc/pub/week38/html/._week38-bs007.html index 0729c7279..ed8bcc88a 100644 --- a/doc/pub/week38/html/._week38-bs007.html +++ b/doc/pub/week38/html/._week38-bs007.html @@ -1,11 +1,11 @@
- + @@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source @@ -311,94 +218,94 @@ MathJax.Hub.Config({The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial.
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn.linear_model import Ridge
@@ -546,7 +453,7 @@ plt.show()
diff --git a/doc/pub/week38/html/._week38-bs008.html b/doc/pub/week38/html/._week38-bs008.html
index 91739634e..6e08f6a7d 100644
--- a/doc/pub/week38/html/._week38-bs008.html
+++ b/doc/pub/week38/html/._week38-bs008.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-To think about, first part
+To think about, first part
When you are comparing your own code with for example Scikit-Learn's
@@ -433,7 +340,7 @@ Thus, if we cannot assume that the expected outputs/targets are zero
when all predictors are zero (the columns in the design matrix), it
may be a bad idea to implement a model which penalizes the intercept.
Furthermore, in for example Ridge and Lasso regression, the solutions
-(when not shrinking $$\beta_0$$) for the unknown parameters
+(when not shrinking \( \beta_0 \)) for the unknown parameters
\( \boldsymbol{\beta} \) are derived under the assumption that both \( \boldsymbol{y} \) and
\( \boldsymbol{X} \) are zero centered, that is we subtract the mean values.
@@ -473,7 +380,7 @@ Furthermore, in for example Ridge and Lasso regression, the solutions
diff --git a/doc/pub/week38/html/._week38-bs009.html b/doc/pub/week38/html/._week38-bs009.html
index f47408866..08a375fec 100644
--- a/doc/pub/week38/html/._week38-bs009.html
+++ b/doc/pub/week38/html/._week38-bs009.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-More thinking
+More thinking
If our predictors represent different scales, then it is important to
@@ -482,7 +389,7 @@ This can clearly lead to problems in evaluating the cost/loss functions.
diff --git a/doc/pub/week38/html/._week38-bs010.html b/doc/pub/week38/html/._week38-bs010.html
index 6b2ca1746..3b1089e61 100644
--- a/doc/pub/week38/html/._week38-bs010.html
+++ b/doc/pub/week38/html/._week38-bs010.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Still thinking
+Still thinking
Keep in mind that when you transform your data set before training a model, the same transformation needs to be done
@@ -423,7 +330,7 @@ on your eventual new data set before making a prediction. If we translate this
-
#Model training, we compute the mean value of y and X
+#Model training, we compute the mean value of y and X
y_train_mean = np.mean(y_train)
X_train_mean = np.mean(X_train,axis=0)
X_train = X_train - X_train_mean
@@ -476,7 +383,7 @@ y_pred = y_pred
-
+
-->
diff --git a/doc/pub/week38/html/._week38-bs011.html b/doc/pub/week38/html/._week38-bs011.html
index 130fb0677..eb852aa0d 100644
--- a/doc/pub/week38/html/._week38-bs011.html
+++ b/doc/pub/week38/html/._week38-bs011.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Linear Regression code, Intercept handling first
+Linear Regression code, Intercept handling first
This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only.
@@ -422,7 +329,7 @@ This code shows a simple first-order fit to a data set using the above transform
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
@@ -531,7 +438,7 @@ plt.show()
diff --git a/doc/pub/week38/html/._week38-bs012.html b/doc/pub/week38/html/._week38-bs012.html
index 1bb7caa48..453862089 100644
--- a/doc/pub/week38/html/._week38-bs012.html
+++ b/doc/pub/week38/html/._week38-bs012.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-What does centering (subtracting the mean values) mean mathematically?
+What does centering (subtracting the mean values) mean mathematically?
Let us try to understand what this may imply mathematically when we subtract the mean values, also known as zero centering. To catch many birds with just one stone, we will focus on Ridge regression.
@@ -451,7 +358,7 @@ If we assume that every column of \( \boldsymbol{X} \) is centered, which we can
-
X = X - np.mean(X,axis=0)
+X = X - np.mean(X,axis=0)
the sum \( \sum_{i=0}^{n-1} X_{ij} \)
@@ -530,7 +437,7 @@ and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=1}^{n-1}X_{kj} \).
diff --git a/doc/pub/week38/html/._week38-bs013.html b/doc/pub/week38/html/._week38-bs013.html
index 659ea337e..741d9ab3f 100644
--- a/doc/pub/week38/html/._week38-bs013.html
+++ b/doc/pub/week38/html/._week38-bs013.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Code Examples
+Code Examples
Armed with this wisdom, we attempt first simply set the intercept eqault to False in our implementation of Ridge regression for a vanilla data set.
@@ -422,7 +329,7 @@ Armed with this wisdom, we attempt first simply set the intercept eqault to F
-
import numpy as np
+import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
@@ -534,7 +441,7 @@ Let us see how we can change this code by zero centering.
diff --git a/doc/pub/week38/html/._week38-bs014.html b/doc/pub/week38/html/._week38-bs014.html
index 01b0c6c01..03f7e06c4 100644
--- a/doc/pub/week38/html/._week38-bs014.html
+++ b/doc/pub/week38/html/._week38-bs014.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,11 +321,11 @@ MathJax.Hub.Config({
-Taking out the mean
+Taking out the mean
-
import numpy as np
+import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
@@ -542,7 +449,7 @@ plt.show()
diff --git a/doc/pub/week38/html/._week38-bs015.html b/doc/pub/week38/html/._week38-bs015.html
index cfdf84010..d05dd42d7 100644
--- a/doc/pub/week38/html/._week38-bs015.html
+++ b/doc/pub/week38/html/._week38-bs015.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-More complicated Example: The Ising model
+More complicated Example: The Ising model
The one-dimensional Ising model with nearest neighbor interaction, no
@@ -440,7 +347,7 @@ with their respective energies.
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import seaborn as sns
@@ -506,7 +413,7 @@ the coupling constant to achieve this.
diff --git a/doc/pub/week38/html/._week38-bs016.html b/doc/pub/week38/html/._week38-bs016.html
index 4d84de842..03af90f3b 100644
--- a/doc/pub/week38/html/._week38-bs016.html
+++ b/doc/pub/week38/html/._week38-bs016.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Reformulating the problem to suit regression
+Reformulating the problem to suit regression
A more general form for the one-dimensional Ising model is
@@ -455,7 +362,7 @@ We split the data in training and test data as discussed in the previous example
-
X = np.zeros((n, L ** 2))
+X = np.zeros((n, L ** 2))
for i in range(n):
X[i] = np.outer(spins[i], spins[i]).ravel()
y = energies
@@ -499,7 +406,7 @@ X_train, X_test, y_train, y_test = train_tes
diff --git a/doc/pub/week38/html/._week38-bs017.html b/doc/pub/week38/html/._week38-bs017.html
index 715f834c9..429af35b8 100644
--- a/doc/pub/week38/html/._week38-bs017.html
+++ b/doc/pub/week38/html/._week38-bs017.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Linear regression
+Linear regression
In the ordinary least squares method we choose the cost function
@@ -443,7 +350,7 @@ first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here
-
X_train_own = np.concatenate(
+X_train_own = np.concatenate(
(np.ones(len(X_train))[:, np.newaxis], X_train),
axis=1
)
@@ -455,7 +362,7 @@ X_test_own = np
-
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
return scl.inv(x.T @ x) @ (x.T @ y)
beta = ols_inv(X_train_own, y_train)
@@ -497,7 +404,7 @@ beta = ols_inv(X_train_own, y_train)
diff --git a/doc/pub/week38/html/._week38-bs018.html b/doc/pub/week38/html/._week38-bs018.html
index 8cd02d15c..442896c45 100644
--- a/doc/pub/week38/html/._week38-bs018.html
+++ b/doc/pub/week38/html/._week38-bs018.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Singular Value decomposition
+Singular Value decomposition
Doing the inversion directly turns out to be a bad idea since the matrix
@@ -456,14 +363,14 @@ linear system as an equation would reduce this down to
-
def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
u, s, v = scl.svd(x)
return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
-
beta = ols_svd(X_train_own,y_train)
+beta = ols_svd(X_train_own,y_train)
When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
@@ -471,7 +378,7 @@ When extracting the \( J \)-matrix we need to make sure that we remove the inte
-
J = beta[1:].reshape(L, L)
+J = beta[1:].reshape(L, L)
A way of looking at the coefficients in \( J \) is to plot the matrices as images.
@@ -479,7 +386,7 @@ A way of looking at the coefficients in \( J \) is to plot the matrices as image
-
fig = plt.figure(figsize=(20, 14))
+fig = plt.figure(figsize=(20, 14))
im = plt.imshow(J, **cmap_args)
plt.title("OLS", fontsize=18)
plt.xticks(fontsize=18)
@@ -536,7 +443,7 @@ In this case our matrix inversion was actually possible. The obvious question no
diff --git a/doc/pub/week38/html/._week38-bs019.html b/doc/pub/week38/html/._week38-bs019.html
index 697fe8fac..629b20061 100644
--- a/doc/pub/week38/html/._week38-bs019.html
+++ b/doc/pub/week38/html/._week38-bs019.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-The one-dimensional Ising model
+The one-dimensional Ising model
Let us bring back the Ising model again, but now with an additional
@@ -439,7 +346,7 @@ We will look at a system of \( L = 40 \) spins with a coupling constant of \( J
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import seaborn as sns
@@ -497,7 +404,7 @@ We organize the data as we did above
-
X = np.zeros((n, L ** 2))
+X = np.zeros((n, L ** 2))
for i in range(n):
X[i] = np.outer(spins[i], spins[i]).ravel()
y = energies
@@ -519,21 +426,21 @@ We will do all fitting with Scikit-Learn,
-
clf = skl.LinearRegression().fit(X_train, y_train)
+clf = skl.LinearRegression().fit(X_train, y_train)
When extracting the \( J \)-matrix we make sure to remove the intercept
-
J_sk = clf.coef_.reshape(L, L)
+J_sk = clf.coef_.reshape(L, L)
And then we plot the results
-
fig = plt.figure(figsize=(20, 14))
+fig = plt.figure(figsize=(20, 14))
im = plt.imshow(J_sk, **cmap_args)
plt.title("LinearRegression from Scikit-learn", fontsize=18)
plt.xticks(fontsize=18)
@@ -583,7 +490,7 @@ The results perfectly with our previous discussion where we used our own code.
diff --git a/doc/pub/week38/html/._week38-bs020.html b/doc/pub/week38/html/._week38-bs020.html
index e3cf5f3ed..d1215662d 100644
--- a/doc/pub/week38/html/._week38-bs020.html
+++ b/doc/pub/week38/html/._week38-bs020.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Ridge regression
+Ridge regression
Having explored the ordinary least squares we move on to ridge
@@ -433,7 +340,7 @@ $$
-
_lambda = 0.1
+_lambda = 0.1
clf_ridge = skl.Ridge(alpha=_lambda).fit(X_train, y_train)
J_ridge_sk = clf_ridge.coef_.reshape(L, L)
fig = plt.figure(figsize=(20, 14))
@@ -484,7 +391,7 @@ plt.show()
diff --git a/doc/pub/week38/html/._week38-bs021.html b/doc/pub/week38/html/._week38-bs021.html
index 92c18e7cd..d6711e37b 100644
--- a/doc/pub/week38/html/._week38-bs021.html
+++ b/doc/pub/week38/html/._week38-bs021.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-LASSO regression
+LASSO regression
In the Least Absolute Shrinkage and Selection Operator (LASSO)-method we get a third cost function.
@@ -432,7 +339,7 @@ Finding the extremal point of this cost function is not so straight-forward as i
-
clf_lasso = skl.Lasso(alpha=_lambda).fit(X_train, y_train)
+clf_lasso = skl.Lasso(alpha=_lambda).fit(X_train, y_train)
J_lasso_sk = clf_lasso.coef_.reshape(L, L)
fig = plt.figure(figsize=(20, 14))
im = plt.imshow(J_lasso_sk, **cmap_args)
@@ -487,7 +394,7 @@ constant as opposed to ridge and OLS. We get a sparse solution with
diff --git a/doc/pub/week38/html/._week38-bs022.html b/doc/pub/week38/html/._week38-bs022.html
index d7a5cdefd..66dfaf391 100644
--- a/doc/pub/week38/html/._week38-bs022.html
+++ b/doc/pub/week38/html/._week38-bs022.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Performance as function of the regularization parameter
+Performance as function of the regularization parameter
We see how the different models perform for a different set of values for \( \lambda \).
@@ -422,7 +329,7 @@ We see how the different models perform for a different set of values for \( \la
-
lambdas = np.logspace(-4, 5, 10)
+lambdas = np.logspace(-4, 5, 10)
train_errors = {
"ols_sk": np.zeros(lambdas.size),
@@ -503,7 +410,7 @@ much. Ridge is more stable over a larger range of values for
diff --git a/doc/pub/week38/html/._week38-bs023.html b/doc/pub/week38/html/._week38-bs023.html
index d2cc08158..5ae0757da 100644
--- a/doc/pub/week38/html/._week38-bs023.html
+++ b/doc/pub/week38/html/._week38-bs023.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Finding the optimal value of \( \lambda \)
+Finding the optimal value of \( \lambda \)
To determine which value of \( \lambda \) is best we plot the accuracy of
@@ -427,7 +334,7 @@ testing set that is close to the accuracy of the training set.
-
fig = plt.figure(figsize=(20, 14))
+fig = plt.figure(figsize=(20, 14))
colors = {
"ols_sk": "r",
@@ -501,7 +408,7 @@ other models for all values of \( \lambda \).
diff --git a/doc/pub/week38/html/._week38-bs024.html b/doc/pub/week38/html/._week38-bs024.html
index 47dadc10d..78c22aef5 100644
--- a/doc/pub/week38/html/._week38-bs024.html
+++ b/doc/pub/week38/html/._week38-bs024.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Logistic Regression
+Logistic Regression
In linear regression our main interest was centered on learning the
@@ -467,7 +374,7 @@ simple recipe for fitting our data.
diff --git a/doc/pub/week38/html/._week38-bs025.html b/doc/pub/week38/html/._week38-bs025.html
index c46487a47..208c29f82 100644
--- a/doc/pub/week38/html/._week38-bs025.html
+++ b/doc/pub/week38/html/._week38-bs025.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Classification problems
+Classification problems
Classification problems, however, are concerned with outcomes taking
@@ -472,7 +379,7 @@ failure etc.
diff --git a/doc/pub/week38/html/._week38-bs026.html b/doc/pub/week38/html/._week38-bs026.html
index be57adb24..60b35e953 100644
--- a/doc/pub/week38/html/._week38-bs026.html
+++ b/doc/pub/week38/html/._week38-bs026.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Optimization and Deep learning
+Optimization and Deep learning
Logistic regression will also serve as our stepping stone towards
@@ -470,7 +377,7 @@ models, as we will see later.
diff --git a/doc/pub/week38/html/._week38-bs027.html b/doc/pub/week38/html/._week38-bs027.html
index 74f3d678f..e8c3825eb 100644
--- a/doc/pub/week38/html/._week38-bs027.html
+++ b/doc/pub/week38/html/._week38-bs027.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Basics
+Basics
We consider the case where the dependent variables, also called the
@@ -476,7 +383,7 @@ $$
diff --git a/doc/pub/week38/html/._week38-bs028.html b/doc/pub/week38/html/._week38-bs028.html
index be27c1b95..038c96fba 100644
--- a/doc/pub/week38/html/._week38-bs028.html
+++ b/doc/pub/week38/html/._week38-bs028.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Linear classifier
+Linear classifier
Before moving to the logistic model, let us try to use our linear
@@ -473,7 +380,7 @@ where \( \boldsymbol{y} \) is a vector representing the possible outcomes, \( \b
diff --git a/doc/pub/week38/html/._week38-bs029.html b/doc/pub/week38/html/._week38-bs029.html
index 3a4c82f7b..69ba839b5 100644
--- a/doc/pub/week38/html/._week38-bs029.html
+++ b/doc/pub/week38/html/._week38-bs029.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Some selected properties
+Some selected properties
The main problem with our function is that it takes values on the
@@ -471,7 +378,7 @@ the probability of a given category. This leads us to the logistic function.
diff --git a/doc/pub/week38/html/._week38-bs030.html b/doc/pub/week38/html/._week38-bs030.html
index 0af859c11..7226ce297 100644
--- a/doc/pub/week38/html/._week38-bs030.html
+++ b/doc/pub/week38/html/._week38-bs030.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Simple example
+Simple example
The following example on data for coronary heart disease (CHD) as function of age may serve as an illustration. In the code here we read and plot whether a person has had CHD (output = 1) or not (output = 0). This ouput is plotted the person's against age. Clearly, the figure shows that attempting to make a standard linear regression fit may not be very meaningful.
@@ -422,7 +329,7 @@ The following example on data for coronary heart disease (CHD) as function of ag
-
# Common imports
+# Common imports
import os
import numpy as np
import pandas as pd
@@ -515,7 +422,7 @@ plt.show()
diff --git a/doc/pub/week38/html/._week38-bs031.html b/doc/pub/week38/html/._week38-bs031.html
index ab192b242..7df2d5473 100644
--- a/doc/pub/week38/html/._week38-bs031.html
+++ b/doc/pub/week38/html/._week38-bs031.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Plotting the mean value for each group
+Plotting the mean value for each group
What we could attempt however is to plot the mean value for each group.
@@ -422,7 +329,7 @@ What we could attempt however is to plot the mean value for each group.
-
agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800])
+agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800])
group = np.array([1, 2, 3, 4, 5, 6, 7, 8])
plt.plot(group, agegroupmean, "r-")
plt.axis([0,9,0, 1.0])
@@ -488,7 +395,7 @@ representing the probability for finding a value of \( y_i \) with a given
diff --git a/doc/pub/week38/html/._week38-bs032.html b/doc/pub/week38/html/._week38-bs032.html
index dc81f68e5..836d0c52d 100644
--- a/doc/pub/week38/html/._week38-bs032.html
+++ b/doc/pub/week38/html/._week38-bs032.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-The logistic function
+The logistic function
Another widely studied model, is the so-called
@@ -472,7 +379,7 @@ Note that \( 1-p(t)= p(-t) \).
diff --git a/doc/pub/week38/html/._week38-bs033.html b/doc/pub/week38/html/._week38-bs033.html
index 36ee8c2b0..f5ee53ea5 100644
--- a/doc/pub/week38/html/._week38-bs033.html
+++ b/doc/pub/week38/html/._week38-bs033.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Examples of likelihood functions used in logistic regression and nueral networks
+Examples of likelihood functions used in logistic regression and nueral networks
The following code plots the logistic function, the step function and other functions we will encounter from here and on.
@@ -422,7 +329,7 @@ The following code plots the logistic function, the step function and other func
-
"""The sigmoid function (or the logistic curve) is a
+"""The sigmoid function (or the logistic curve) is a
function that takes any real number, z, and outputs a number (0,1).
It is useful in neural networks for assigning weights on a relative scale.
The value z is the weighted sum of parameters involved in the learning algorithm."""
@@ -515,7 +422,7 @@ plt.show()
diff --git a/doc/pub/week38/html/._week38-bs034.html b/doc/pub/week38/html/._week38-bs034.html
index 2a027a1ff..78d3299ab 100644
--- a/doc/pub/week38/html/._week38-bs034.html
+++ b/doc/pub/week38/html/._week38-bs034.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Two parameters
+Two parameters
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
@@ -471,7 +378,7 @@ $$
diff --git a/doc/pub/week38/html/._week38-bs035.html b/doc/pub/week38/html/._week38-bs035.html
index fa33d4425..9201cdd2f 100644
--- a/doc/pub/week38/html/._week38-bs035.html
+++ b/doc/pub/week38/html/._week38-bs035.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Maximum likelihood
+Maximum likelihood
In order to define the total likelihood for all possible outcomes from a
@@ -472,7 +379,7 @@ $$
diff --git a/doc/pub/week38/html/._week38-bs036.html b/doc/pub/week38/html/._week38-bs036.html
index 3ce3951cb..d73ff8a15 100644
--- a/doc/pub/week38/html/._week38-bs036.html
+++ b/doc/pub/week38/html/._week38-bs036.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-The cost function rewritten
+The cost function rewritten
Reordering the logarithms, we can rewrite the cost/loss function as
@@ -470,7 +377,7 @@ in practice we often supplement the cross-entropy with additional regularization
diff --git a/doc/pub/week38/html/._week38-bs037.html b/doc/pub/week38/html/._week38-bs037.html
index 0d4a49efc..b4df33ae8 100644
--- a/doc/pub/week38/html/._week38-bs037.html
+++ b/doc/pub/week38/html/._week38-bs037.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Minimizing the cross entropy
+Minimizing the cross entropy
The cross entropy is a convex function of the weights \( \boldsymbol{\beta} \) and,
@@ -471,7 +378,7 @@ $$
diff --git a/doc/pub/week38/html/._week38-bs038.html b/doc/pub/week38/html/._week38-bs038.html
index 75f6a1403..98b3bc77f 100644
--- a/doc/pub/week38/html/._week38-bs038.html
+++ b/doc/pub/week38/html/._week38-bs038.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-A more compact expression
+A more compact expression
Let us now define a vector \( \boldsymbol{y} \) with \( n \) elements \( y_i \), an
@@ -472,7 +379,7 @@ $$
diff --git a/doc/pub/week38/html/._week38-bs039.html b/doc/pub/week38/html/._week38-bs039.html
index 52ed8b393..02db2ee2a 100644
--- a/doc/pub/week38/html/._week38-bs039.html
+++ b/doc/pub/week38/html/._week38-bs039.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Extending to more predictors
+Extending to more predictors
Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with \( p \) predictors
@@ -465,7 +372,7 @@ $$
diff --git a/doc/pub/week38/html/._week38-bs040.html b/doc/pub/week38/html/._week38-bs040.html
index 7d996ad52..7b05367d9 100644
--- a/doc/pub/week38/html/._week38-bs040.html
+++ b/doc/pub/week38/html/._week38-bs040.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Including more classes
+Including more classes
Till now we have mainly focused on two classes, the so-called binary
@@ -477,7 +384,7 @@ and the model is specified in term of \( K-1 \) so-called log-odds or
diff --git a/doc/pub/week38/html/._week38-bs041.html b/doc/pub/week38/html/._week38-bs041.html
index efa2cd448..aab36316a 100644
--- a/doc/pub/week38/html/._week38-bs041.html
+++ b/doc/pub/week38/html/._week38-bs041.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-More classes
+More classes
In our discussion of neural networks we will encounter the above again
@@ -489,7 +396,7 @@ methods.
diff --git a/doc/pub/week38/html/._week38-bs042.html b/doc/pub/week38/html/._week38-bs042.html
index 2ca873953..3c4b29a00 100644
--- a/doc/pub/week38/html/._week38-bs042.html
+++ b/doc/pub/week38/html/._week38-bs042.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -414,7 +321,7 @@ MathJax.Hub.Config({
-Friday September 24
+Friday September 24
@@ -454,7 +361,7 @@ MathJax.Hub.Config({
diff --git a/doc/pub/week38/html/week38-bs.html b/doc/pub/week38/html/week38-bs.html
index 0d1b2580c..dc41103ec 100644
--- a/doc/pub/week38/html/week38-bs.html
+++ b/doc/pub/week38/html/week38-bs.html
@@ -1,11 +1,11 @@
-
+
@@ -41,239 +41,146 @@ Automatically generated HTML file from DocOnce source
@@ -311,94 +218,94 @@ MathJax.Hub.Config({
-
Contents
@@ -469,7 +376,7 @@ MathJax.Hub.Config({
diff --git a/doc/pub/week38/html/week38-reveal.html b/doc/pub/week38/html/week38-reveal.html
index 82b585c31..a01f0bb1b 100644
--- a/doc/pub/week38/html/week38-reveal.html
+++ b/doc/pub/week38/html/week38-reveal.html
@@ -1,7 +1,7 @@
-
+
@@ -13,7 +13,7 @@
-
+
@@ -76,10 +76,10 @@ document.getElementsByTagName( 'head' )[0].appendChild( link );
.reveal .alert-block > p, .alert-block > ul {margin-bottom:1em}
/*.reveal .alert li {margin-top: 1em}*/
.reveal .alert-block p+p {margin-top:5px}
- /*.reveal .alert-notice { background-image: url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
- .reveal .alert-summary { background-image:url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
- .reveal .alert-warning { background-image: url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
- .reveal .alert-question {background-image:url(https://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
+ /*.reveal .alert-notice { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_notice.png); }
+ .reveal .alert-summary { background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_summary.png); }
+ .reveal .alert-warning { background-image: url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_warning.png); }
+ .reveal .alert-question {background-image:url(http://hplgit.github.io/doconce/bundled/html_images/small_gray_question.png); } */
@@ -159,7 +159,7 @@ MathJax.Hub.Config({
-Plans for week 38
+Plans for week 38
- Thursday: Summary of regression methods and discussion of project 1. Start Logistic Regression
@@ -169,12 +169,12 @@ MathJax.Hub.Config({
-Thursday September 23
+Thursday September 23
-Ridge and LASSO Regression, reminder
+Ridge and LASSO Regression, reminder
The expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
@@ -246,7 +246,7 @@ $$
-Various steps in cross-validation
+Various steps in cross-validation
When the repetitive splitting of the data set is done randomly,
@@ -267,7 +267,7 @@ cross-validation (LOOCV).
-How to set up the cross-validation for Ridge and/or Lasso
+How to set up the cross-validation for Ridge and/or Lasso
- Define a range of interest for the penalty parameter.
@@ -301,7 +301,7 @@ $$
-Cross-validation in brief
+Cross-validation in brief
For the various values of \( k \)
@@ -323,14 +323,14 @@ For the various values of \( k \)
-Code Example for Cross-validation and \( k \)-fold Cross-validation
+Code Example for Cross-validation and \( k \)-fold Cross-validation
The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial.
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
from sklearn.linear_model import Ridge
@@ -424,7 +424,7 @@ plt.show()
-To think about, first part
+To think about, first part
When you are comparing your own code with for example Scikit-Learn's
@@ -443,16 +443,14 @@ Thus, if we cannot assume that the expected outputs/targets are zero
when all predictors are zero (the columns in the design matrix), it
may be a bad idea to implement a model which penalizes the intercept.
Furthermore, in for example Ridge and Lasso regression, the solutions
-(when not shrinking
-$$\beta_0$$
-
) for the unknown parameters
+(when not shrinking \( \beta_0 \)) for the unknown parameters
\( \boldsymbol{\beta} \) are derived under the assumption that both \( \boldsymbol{y} \) and
\( \boldsymbol{X} \) are zero centered, that is we subtract the mean values.
-More thinking
+More thinking
If our predictors represent different scales, then it is important to
@@ -486,7 +484,7 @@ This can clearly lead to problems in evaluating the cost/loss functions.
-Still thinking
+Still thinking
Keep in mind that when you transform your data set before training a model, the same transformation needs to be done
@@ -495,7 +493,7 @@ on your eventual new data set before making a prediction. If we translate this
-
#Model training, we compute the mean value of y and X
+#Model training, we compute the mean value of y and X
y_train_mean = np.mean(y_train)
X_train_mean = np.mean(X_train,axis=0)
X_train = X_train - X_train_mean
@@ -514,7 +512,7 @@ y_pred = y_pred + y_train_mean
-Linear Regression code, Intercept handling first
+Linear Regression code, Intercept handling first
This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (code example thanks to Øyvind Sigmundson Schøyen). Here our scaling of the data is done by subtracting the mean values only.
@@ -522,7 +520,7 @@ This code shows a simple first-order fit to a data set using the above transform
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
@@ -597,7 +595,7 @@ plt.show()
-What does centering (subtracting the mean values) mean mathematically?
+What does centering (subtracting the mean values) mean mathematically?
Let us try to understand what this may imply mathematically when we subtract the mean values, also known as zero centering. To catch many birds with just one stone, we will focus on Ridge regression.
@@ -642,7 +640,7 @@ If we assume that every column of \( \boldsymbol{X} \) is centered, which we can
-
X = X - np.mean(X,axis=0)
+X = X - np.mean(X,axis=0)
the sum \( \sum_{i=0}^{n-1} X_{ij} \)
@@ -696,7 +694,7 @@ and \( \tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=1}^{n-1}X_{kj} \).
-Code Examples
+Code Examples
Armed with this wisdom, we attempt first simply set the intercept eqault to False in our implementation of Ridge regression for a vanilla data set.
@@ -704,7 +702,7 @@ Armed with this wisdom, we attempt first simply set the intercept eqault to F
-
import numpy as np
+import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
@@ -781,11 +779,11 @@ Let us see how we can change this code by zero centering.
-Taking out the mean
+Taking out the mean
-
import numpy as np
+import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
@@ -875,7 +873,7 @@ plt.show()
-More complicated Example: The Ising model
+More complicated Example: The Ising model
The one-dimensional Ising model with nearest neighbor interaction, no
@@ -903,7 +901,7 @@ with their respective energies.
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import seaborn as sns
@@ -934,7 +932,7 @@ the coupling constant to achieve this.
-Reformulating the problem to suit regression
+Reformulating the problem to suit regression
A more general form for the one-dimensional Ising model is
@@ -981,7 +979,7 @@ We split the data in training and test data as discussed in the previous example
-
X = np.zeros((n, L ** 2))
+X = np.zeros((n, L ** 2))
for i in range(n):
X[i] = np.outer(spins[i], spins[i]).ravel()
y = energies
@@ -991,7 +989,7 @@ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=
-Linear regression
+Linear regression
In the ordinary least squares method we choose the cost function
@@ -1024,7 +1022,7 @@ first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here
-
X_train_own = np.concatenate(
+X_train_own = np.concatenate(
(np.ones(len(X_train))[:, np.newaxis], X_train),
axis=1
)
@@ -1036,7 +1034,7 @@ X_test_own = np.concatenate(
-
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
return scl.inv(x.T @ x) @ (x.T @ y)
beta = ols_inv(X_train_own, y_train)
@@ -1044,7 +1042,7 @@ beta = ols_inv(X_train_own, y_train)
-Singular Value decomposition
+Singular Value decomposition
Doing the inversion directly turns out to be a bad idea since the matrix
@@ -1092,14 +1090,14 @@ linear system as an equation would reduce this down to
-
def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
u, s, v = scl.svd(x)
return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
-
beta = ols_svd(X_train_own,y_train)
+beta = ols_svd(X_train_own,y_train)
When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
@@ -1107,7 +1105,7 @@ When extracting the \( J \)-matrix we need to make sure that we remove the inte
-
J = beta[1:].reshape(L, L)
+J = beta[1:].reshape(L, L)
A way of looking at the coefficients in \( J \) is to plot the matrices as images.
@@ -1115,7 +1113,7 @@ A way of looking at the coefficients in \( J \) is to plot the matrices as image
-
fig = plt.figure(figsize=(20, 14))
+fig = plt.figure(figsize=(20, 14))
im = plt.imshow(J, **cmap_args)
plt.title("OLS", fontsize=18)
plt.xticks(fontsize=18)
@@ -1137,7 +1135,7 @@ In this case our matrix inversion was actually possible. The obvious question no
-The one-dimensional Ising model
+The one-dimensional Ising model
Let us bring back the Ising model again, but now with an additional
@@ -1164,7 +1162,7 @@ We will look at a system of \( L = 40 \) spins with a coupling constant of \( J
-
import numpy as np
+import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import seaborn as sns
@@ -1228,7 +1226,7 @@ We organize the data as we did above
-
X = np.zeros((n, L ** 2))
+X = np.zeros((n, L ** 2))
for i in range(n):
X[i] = np.outer(spins[i], spins[i]).ravel()
y = energies
@@ -1250,21 +1248,21 @@ We will do all fitting with Scikit-Learn,
-
clf = skl.LinearRegression().fit(X_train, y_train)
+clf = skl.LinearRegression().fit(X_train, y_train)
When extracting the \( J \)-matrix we make sure to remove the intercept
-
J_sk = clf.coef_.reshape(L, L)
+J_sk = clf.coef_.reshape(L, L)
And then we plot the results
-
fig = plt.figure(figsize=(20, 14))
+fig = plt.figure(figsize=(20, 14))
im = plt.imshow(J_sk, **cmap_args)
plt.title("LinearRegression from Scikit-learn", fontsize=18)
plt.xticks(fontsize=18)
@@ -1279,7 +1277,7 @@ The results perfectly with our previous discussion where we used our own code.
-Ridge regression
+Ridge regression
Having explored the ordinary least squares we move on to ridge
@@ -1300,7 +1298,7 @@ $$
-
_lambda = 0.1
+_lambda = 0.1
clf_ridge = skl.Ridge(alpha=_lambda).fit(X_train, y_train)
J_ridge_sk = clf_ridge.coef_.reshape(L, L)
fig = plt.figure(figsize=(20, 14))
@@ -1317,7 +1315,7 @@ plt.show()
-LASSO regression
+LASSO regression
In the Least Absolute Shrinkage and Selection Operator (LASSO)-method we get a third cost function.
@@ -1337,7 +1335,7 @@ Finding the extremal point of this cost function is not so straight-forward as i
-
clf_lasso = skl.Lasso(alpha=_lambda).fit(X_train, y_train)
+clf_lasso = skl.Lasso(alpha=_lambda).fit(X_train, y_train)
J_lasso_sk = clf_lasso.coef_.reshape(L, L)
fig = plt.figure(figsize=(20, 14))
im = plt.imshow(J_lasso_sk, **cmap_args)
@@ -1357,7 +1355,7 @@ constant as opposed to ridge and OLS. We get a sparse solution with
-Performance as function of the regularization parameter
+Performance as function of the regularization parameter
We see how the different models perform for a different set of values for \( \lambda \).
@@ -1365,7 +1363,7 @@ We see how the different models perform for a different set of values for \( \la
-
lambdas = np.logspace(-4, 5, 10)
+lambdas = np.logspace(-4, 5, 10)
train_errors = {
"ols_sk": np.zeros(lambdas.size),
@@ -1411,7 +1409,7 @@ much. Ridge is more stable over a larger range of values for
-Finding the optimal value of \( \lambda \)
+Finding the optimal value of \( \lambda \)
To determine which value of \( \lambda \) is best we plot the accuracy of
@@ -1424,7 +1422,7 @@ testing set that is close to the accuracy of the training set.
-
fig = plt.figure(figsize=(20, 14))
+fig = plt.figure(figsize=(20, 14))
colors = {
"ols_sk": "r",
@@ -1463,7 +1461,7 @@ other models for all values of \( \lambda \).
-Logistic Regression
+Logistic Regression
In linear regression our main interest was centered on learning the
@@ -1481,7 +1479,7 @@ simple recipe for fitting our data.
-Classification problems
+Classification problems
Classification problems, however, are concerned with outcomes taking
@@ -1504,7 +1502,7 @@ failure etc.
-Optimization and Deep learning
+Optimization and Deep learning
Logistic regression will also serve as our stepping stone towards
@@ -1525,7 +1523,7 @@ models, as we will see later.
-Basics
+Basics
We consider the case where the dependent variables, also called the
@@ -1554,7 +1552,7 @@ $$
-Linear classifier
+Linear classifier
Before moving to the logistic model, let us try to use our linear
@@ -1580,7 +1578,7 @@ where \( \boldsymbol{y} \) is a vector representing the possible outcomes, \( \b
-Some selected properties
+Some selected properties
The main problem with our function is that it takes values on the
@@ -1602,7 +1600,7 @@ the probability of a given category. This leads us to the logistic function.
-Simple example
+Simple example
The following example on data for coronary heart disease (CHD) as function of age may serve as an illustration. In the code here we read and plot whether a person has had CHD (output = 1) or not (output = 0). This ouput is plotted the person's against age. Clearly, the figure shows that attempting to make a standard linear regression fit may not be very meaningful.
@@ -1610,7 +1608,7 @@ The following example on data for coronary heart disease (CHD) as function of ag
-
# Common imports
+# Common imports
import os
import numpy as np
import pandas as pd
@@ -1669,7 +1667,7 @@ plt.show()
-Plotting the mean value for each group
+Plotting the mean value for each group
What we could attempt however is to plot the mean value for each group.
@@ -1677,7 +1675,7 @@ What we could attempt however is to plot the mean value for each group.
-
agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800])
+agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800])
group = np.array([1, 2, 3, 4, 5, 6, 7, 8])
plt.plot(group, agegroupmean, "r-")
plt.axis([0,9,0, 1.0])
@@ -1710,7 +1708,7 @@ representing the probability for finding a value of \( y_i \) with a given
-The logistic function
+The logistic function
Another widely studied model, is the so-called
@@ -1735,7 +1733,7 @@ Note that \( 1-p(t)= p(-t) \).
-Examples of likelihood functions used in logistic regression and nueral networks
+Examples of likelihood functions used in logistic regression and nueral networks
The following code plots the logistic function, the step function and other functions we will encounter from here and on.
@@ -1743,7 +1741,7 @@ The following code plots the logistic function, the step function and other func
-
"""The sigmoid function (or the logistic curve) is a
+"""The sigmoid function (or the logistic curve) is a
function that takes any real number, z, and outputs a number (0,1).
It is useful in neural networks for assigning weights on a relative scale.
The value z is the weighted sum of parameters involved in the learning algorithm."""
@@ -1802,7 +1800,7 @@ plt.show()
-Two parameters
+Two parameters
We assume now that we have two classes with \( y_i \) either \( 0 \) or \( 1 \). Furthermore we assume also that we have only two parameters \( \beta \) in our fitting of the Sigmoid function, that is we define probabilities
@@ -1828,7 +1826,7 @@ $$
-Maximum likelihood
+Maximum likelihood
In order to define the total likelihood for all possible outcomes from a
@@ -1855,7 +1853,7 @@ $$
-The cost function rewritten
+The cost function rewritten
Reordering the logarithms, we can rewrite the cost/loss function as
@@ -1880,7 +1878,7 @@ in practice we often supplement the cross-entropy with additional regularization
-Minimizing the cross entropy
+Minimizing the cross entropy
The cross entropy is a convex function of the weights \( \boldsymbol{\beta} \) and,
@@ -1906,7 +1904,7 @@ $$
-A more compact expression
+A more compact expression
Let us now define a vector \( \boldsymbol{y} \) with \( n \) elements \( y_i \), an
@@ -1933,7 +1931,7 @@ $$
-Extending to more predictors
+Extending to more predictors
Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with \( p \) predictors
@@ -1953,7 +1951,7 @@ $$
-Including more classes
+Including more classes
Till now we have mainly focused on two classes, the so-called binary
@@ -1987,7 +1985,7 @@ and the model is specified in term of \( K-1 \) so-called log-odds or
-More classes
+More classes
In our discussion of neural networks we will encounter the above again
@@ -2031,12 +2029,12 @@ methods.
-Friday September 24
+Friday September 24
-Wisconsin Cancer Data
+Wisconsin Cancer Data
We show here how we can use a simple regression case on the breast
@@ -2046,7 +2044,7 @@ classification.
-
import matplotlib.pyplot as plt
+import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
@@ -2076,7 +2074,7 @@ logreg.fit(X_train_scaled, y_train)
-Using the correlation matrix
+Using the correlation matrix
In addition to the above scores, we could also study the covariance (and the correlation matrix).
@@ -2084,7 +2082,7 @@ We use Pandas to compute the correlation matrix.
-
import matplotlib.pyplot as plt
+import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
@@ -2123,7 +2121,7 @@ plt.show()
-Discussing the correlation data
+Discussing the correlation data
In the above example we note two things. In the first plot we display
@@ -2144,14 +2142,14 @@ We constructed this matrix using pandas via the statements
-
cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
+cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
and then
-
correlation_matrix = cancerpd.corr().round(1)
+correlation_matrix = cancerpd.corr().round(1)
Diagonalizing this matrix we can in turn say something about which
@@ -2162,11 +2160,11 @@ applications. This will be discussed later this semester (Other measures in classification studies: Cancer Data again
+Other measures in classification studies: Cancer Data again
-
import matplotlib.pyplot as plt
+import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_breast_cancer
@@ -2215,7 +2213,7 @@ plt.show()
-Optimization, the central part of any Machine Learning algortithm
+Optimization, the central part of any Machine Learning algortithm
Overview Video, why do we care about gradient methods?
@@ -2233,7 +2231,7 @@ some approximative/numerical method to compute the minimum.
-Revisiting our Logistic Regression case
+Revisiting our Logistic Regression case
In our discussion on Logistic Regression we studied the
@@ -2257,7 +2255,7 @@ where \( \boldsymbol{\beta} \) are the weights we wish to extract from data, in
-The equations to solve
+The equations to solve
Our compact equations used a definition of a vector \( \boldsymbol{y} \) with \( n \)
@@ -2287,7 +2285,7 @@ This defines what is called the Hessian matrix.