From 7d18cc5cf6141b070baaac8b72fe2ea6d2183cf0 Mon Sep 17 00:00:00 2001 From: mhjensen Date: Fri, 25 Sep 2020 06:13:11 +0200 Subject: [PATCH] replacing hats with boldface --- doc/pub/week39/html/._week39-bs040.html | 9 ++-- doc/pub/week39/html/._week39-bs041.html | 4 +- doc/pub/week39/html/._week39-bs042.html | 14 +++--- doc/pub/week39/html/._week39-bs052.html | 16 +++---- doc/pub/week39/html/week39-reveal.html | 43 ++++++++++--------- doc/pub/week39/html/week39-solarized.html | 43 ++++++++++--------- doc/pub/week39/html/week39.html | 43 ++++++++++--------- doc/pub/week39/ipynb/ipynb-week39-src.tar.gz | Bin 192 -> 192 bytes doc/pub/week39/ipynb/week39.ipynb | 43 ++++++++++--------- doc/src/week39/week39.do.txt | 43 ++++++++++--------- 10 files changed, 132 insertions(+), 126 deletions(-) diff --git a/doc/pub/week39/html/._week39-bs040.html b/doc/pub/week39/html/._week39-bs040.html index a4d0d4056..0c070a07a 100644 --- a/doc/pub/week39/html/._week39-bs040.html +++ b/doc/pub/week39/html/._week39-bs040.html @@ -270,11 +270,12 @@ MathJax.Hub.Config({ import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) diff --git a/doc/pub/week39/html/._week39-bs041.html b/doc/pub/week39/html/._week39-bs041.html index cf169d4f2..390542bcf 100644 --- a/doc/pub/week39/html/._week39-bs041.html +++ b/doc/pub/week39/html/._week39-bs041.html @@ -264,13 +264,13 @@ MathJax.Hub.Config({

We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \), $$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows $$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ diff --git a/doc/pub/week39/html/._week39-bs042.html b/doc/pub/week39/html/._week39-bs042.html index 4c806df95..39aa14871 100644 --- a/doc/pub/week39/html/._week39-bs042.html +++ b/doc/pub/week39/html/._week39-bs042.html @@ -272,18 +272,18 @@ MathJax.Hub.Config({ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -292,12 +292,12 @@ eta = 0.1= 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') diff --git a/doc/pub/week39/html/._week39-bs052.html b/doc/pub/week39/html/._week39-bs052.html index 0712f84da..57a3d2b19 100644 --- a/doc/pub/week39/html/._week39-bs052.html +++ b/doc/pub/week39/html/._week39-bs052.html @@ -275,8 +275,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -291,15 +291,15 @@ Niterations = 1 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -312,7 +312,7 @@ theta = np.for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/html/week39-reveal.html b/doc/pub/week39/html/week39-reveal.html index 4befb5adc..8810406b8 100644 --- a/doc/pub/week39/html/week39-reveal.html +++ b/doc/pub/week39/html/week39-reveal.html @@ -1338,11 +1338,12 @@ plt.show() import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) @@ -1358,7 +1359,7 @@ sgdreg.fit(x,y.ravel()) We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \),

 
$$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

 
@@ -1366,7 +1367,7 @@ $$ In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows

 
$$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ @@ -1396,18 +1397,18 @@ $$ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -1416,12 +1417,12 @@ eta = 0.1 Niterations = 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1659,8 +1660,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -1675,15 +1676,15 @@ Niterations = 1000 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -1696,7 +1697,7 @@ theta = np.random.randn(2,for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html index edace6844..aa063743c 100644 --- a/doc/pub/week39/html/week39-solarized.html +++ b/doc/pub/week39/html/week39-solarized.html @@ -1283,11 +1283,12 @@ plt.show() import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) @@ -1301,13 +1302,13 @@ sgdreg.fit(x,y.ravel())

We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \), $$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows $$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ @@ -1334,18 +1335,18 @@ $$ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -1354,12 +1355,12 @@ eta = 0.1 Niterations = 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1584,8 +1585,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -1600,15 +1601,15 @@ Niterations = 1000 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -1621,7 +1622,7 @@ theta = np.random.randn(2,for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html index 05c7372cb..3eaef839d 100644 --- a/doc/pub/week39/html/week39.html +++ b/doc/pub/week39/html/week39.html @@ -1288,11 +1288,12 @@ plt.show() import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) @@ -1306,13 +1307,13 @@ sgdreg.fit(x,y.

We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \), $$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows $$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ @@ -1339,18 +1340,18 @@ $$ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -1359,12 +1360,12 @@ eta = 0.1= 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1589,8 +1590,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -1605,15 +1606,15 @@ Niterations = 1 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -1626,7 +1627,7 @@ theta = np.for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz index 3d99f0b446b67102d0a5400c58bb07cd39ebdd00..2892ce6b9eed5c46dcd5ee552bda8f36929426a2 100644 GIT binary patch literal 192 zcmV;x06+g9iwFQnZf#!x1MSaC3c@fD2H>uHia9|^nm%wT*o6y0#0#W!YGZ3slN9ak z?E`eBxG5s!+x!eO3^Rvpz1d}fy<2ZFgpeePV9Yd$rzGKAPbiInvXrqfNhu{jC}T18 zfGl^?OJ^+C!ztAnr9t_2ZWt@e4}0cU;F*8oSP28$eQzrzK`M*6QZ?KVYqknR+shmZ ug=TDl#%m`u0+&7Tq7YUn$s51b>ZJL_1pcp|aU92SzV-lWaq>g}2mk=vo>SBS literal 192 zcmV;x06+g9iwFQ=Yi(Zu1MSaC3c@fD2H>uHia9|^nx@62U>7a~5igL^sf~}MCMnw6 z+Xv`MaZ^OdxA_@n7-n|)a<$1LcXz>T2q9^bz>rzW$0YGok0|9pS;knLa-L8ilrf$K zK;}E?rL|6&{*?NN%A$HZ*N;`zhdt9P@XSB4ucd|cwsW;sptQq6>l54%Pizs%rc(u! u8tu>mgV$CX1fe<*WhtH1N}R*S=z|d#8->4q#`8SS``QCZ#E{$o2mk