diff --git a/doc/pub/week39/html/._week39-bs040.html b/doc/pub/week39/html/._week39-bs040.html index a4d0d4056..0c070a07a 100644 --- a/doc/pub/week39/html/._week39-bs040.html +++ b/doc/pub/week39/html/._week39-bs040.html @@ -270,11 +270,12 @@ MathJax.Hub.Config({ import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) diff --git a/doc/pub/week39/html/._week39-bs041.html b/doc/pub/week39/html/._week39-bs041.html index cf169d4f2..390542bcf 100644 --- a/doc/pub/week39/html/._week39-bs041.html +++ b/doc/pub/week39/html/._week39-bs041.html @@ -264,13 +264,13 @@ MathJax.Hub.Config({

We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \), $$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows $$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ diff --git a/doc/pub/week39/html/._week39-bs042.html b/doc/pub/week39/html/._week39-bs042.html index 4c806df95..39aa14871 100644 --- a/doc/pub/week39/html/._week39-bs042.html +++ b/doc/pub/week39/html/._week39-bs042.html @@ -272,18 +272,18 @@ MathJax.Hub.Config({ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -292,12 +292,12 @@ eta = 0.1= 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') diff --git a/doc/pub/week39/html/._week39-bs052.html b/doc/pub/week39/html/._week39-bs052.html index 0712f84da..57a3d2b19 100644 --- a/doc/pub/week39/html/._week39-bs052.html +++ b/doc/pub/week39/html/._week39-bs052.html @@ -275,8 +275,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -291,15 +291,15 @@ Niterations = 1 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -312,7 +312,7 @@ theta = np.for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/html/week39-reveal.html b/doc/pub/week39/html/week39-reveal.html index 4befb5adc..8810406b8 100644 --- a/doc/pub/week39/html/week39-reveal.html +++ b/doc/pub/week39/html/week39-reveal.html @@ -1338,11 +1338,12 @@ plt.show() import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) @@ -1358,7 +1359,7 @@ sgdreg.fit(x,y.ravel()) We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \),

 
$$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

 
@@ -1366,7 +1367,7 @@ $$ In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows

 
$$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ @@ -1396,18 +1397,18 @@ $$ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -1416,12 +1417,12 @@ eta = 0.1 Niterations = 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1659,8 +1660,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -1675,15 +1676,15 @@ Niterations = 1000 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -1696,7 +1697,7 @@ theta = np.random.randn(2,for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html index edace6844..aa063743c 100644 --- a/doc/pub/week39/html/week39-solarized.html +++ b/doc/pub/week39/html/week39-solarized.html @@ -1283,11 +1283,12 @@ plt.show() import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) @@ -1301,13 +1302,13 @@ sgdreg.fit(x,y.ravel())

We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \), $$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows $$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ @@ -1334,18 +1335,18 @@ $$ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -1354,12 +1355,12 @@ eta = 0.1 Niterations = 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1584,8 +1585,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -1600,15 +1601,15 @@ Niterations = 1000 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -1621,7 +1622,7 @@ theta = np.random.randn(2,for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html index 05c7372cb..3eaef839d 100644 --- a/doc/pub/week39/html/week39.html +++ b/doc/pub/week39/html/week39.html @@ -1288,11 +1288,12 @@ plt.show() import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) @@ -1306,13 +1307,13 @@ sgdreg.fit(x,y.

We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \), $$ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. $$

In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows $$ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). $$ @@ -1339,18 +1340,18 @@ $$ import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -1359,12 +1360,12 @@ eta = 0.1= 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1589,8 +1590,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -1605,15 +1606,15 @@ Niterations = 1 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -1626,7 +1627,7 @@ theta = np.for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i) diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz index 3d99f0b44..2892ce6b9 100644 Binary files a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz and b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz differ diff --git a/doc/pub/week39/ipynb/week39.ipynb b/doc/pub/week39/ipynb/week39.ipynb index b2b5ef1b4..821149be2 100644 --- a/doc/pub/week39/ipynb/week39.ipynb +++ b/doc/pub/week39/ipynb/week39.ipynb @@ -1499,11 +1499,12 @@ "import matplotlib.pyplot as plt\n", "from sklearn.linear_model import SGDRegressor\n", "\n", - "x = 2*np.random.rand(100,1)\n", - "y = 4+3*x+np.random.randn(100,1)\n", + "n = 100\n", + "x = 2*np.random.rand(n,1)\n", + "y = 4+3*x+np.random.randn(n,1)\n", "\n", - "xb = np.c_[np.ones((100,1)), x]\n", - "beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)\n", + "X = np.c_[np.ones((n,1)), x]\n", + "beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n", "print(beta_linreg)\n", "sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)\n", "sgdreg.fit(x,y.ravel())\n", @@ -1525,7 +1526,7 @@ "metadata": {}, "source": [ "$$\n", - "C_{\\text{ridge}}(\\beta) = ||X\\beta -\\mathbf{y}||^2 + \\lambda ||\\beta||^2, \\ \\lambda \\geq 0.\n", + "C_{\\text{ridge}}(\\beta) = \\frac{1}{n}||X\\beta -\\mathbf{y}||^2 + \\lambda ||\\beta||^2, \\ \\lambda \\geq 0.\n", "$$" ] }, @@ -1541,7 +1542,7 @@ "metadata": {}, "source": [ "$$\n", - "\\nabla_\\beta C_{\\text{ridge}}(\\beta) = 2\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n", + "\\nabla_\\beta C_{\\text{ridge}}(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n", "\\sum_{i=1}^{100}\\left( x_i (\\beta_0+\\beta_1x_i)-y_ix_i\\right) \\\\\n", "\\end{bmatrix} + 2\\lambda\\begin{bmatrix} \\beta_0 \\\\ \\beta_1\\end{bmatrix} = 2 (X^T(X\\beta - \\mathbf{y})+\\lambda \\beta).\n", "$$" @@ -1587,18 +1588,18 @@ "import sys\n", "\n", "# the number of datapoints\n", - "m = 100\n", + "n = 100\n", "x = 2*np.random.rand(m,1)\n", "y = 4+3*x+np.random.randn(m,1)\n", "\n", - "xb = np.c_[np.ones((m,1)), x]\n", - "XT_X = xb.T @ xb\n", + "X = np.c_[np.ones((m,1)), x]\n", + "XT_X = X.T @ X\n", "\n", "#Ridge parameter lambda\n", "lmbda = 0.001\n", "Id = lmbda* np.eye(XT_X.shape[0])\n", "\n", - "beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y\n", + "beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y\n", "print(beta_linreg)\n", "# Start plain gradient descent\n", "beta = np.random.randn(2,1)\n", @@ -1607,12 +1608,12 @@ "Niterations = 100\n", "\n", "for iter in range(Niterations):\n", - " gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta\n", + " gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta\n", " beta -= eta*gradients\n", "\n", "print(beta)\n", - "ypredict = xb @ beta\n", - "ypredict2 = xb @ beta_linreg\n", + "ypredict = X @ beta\n", + "ypredict2 = X @ beta_linreg\n", "plt.plot(x, ypredict, \"r-\")\n", "plt.plot(x, ypredict2, \"b-\")\n", "plt.plot(x, y ,'ro')\n", @@ -1877,8 +1878,8 @@ "x = 2*np.random.rand(m,1)\n", "y = 4+3*x+np.random.randn(m,1)\n", "\n", - "xb = np.c_[np.ones((m,1)), x]\n", - "theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)\n", + "X = np.c_[np.ones((m,1)), x]\n", + "theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n", "print(\"Own inversion\")\n", "print(theta_linreg)\n", "sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)\n", @@ -1893,15 +1894,15 @@ "\n", "\n", "for iter in range(Niterations):\n", - " gradients = 2.0/m*xb.T @ ((xb @ theta)-y)\n", + " gradients = 2.0/m*X.T @ ((X @ theta)-y)\n", " theta -= eta*gradients\n", - "print(\"theta frm own gd\")\n", + "print(\"theta from own gd\")\n", "print(theta)\n", "\n", "xnew = np.array([[0],[2]])\n", - "xbnew = np.c_[np.ones((2,1)), xnew]\n", - "ypredict = xbnew.dot(theta)\n", - "ypredict2 = xbnew.dot(theta_linreg)\n", + "Xnew = np.c_[np.ones((2,1)), xnew]\n", + "ypredict = Xnew.dot(theta)\n", + "ypredict2 = Xnew.dot(theta_linreg)\n", "\n", "\n", "n_epochs = 50\n", @@ -1914,7 +1915,7 @@ "for epoch in range(n_epochs):\n", " for i in range(m):\n", " random_index = np.random.randint(m)\n", - " xi = xb[random_index:random_index+1]\n", + " xi = X[random_index:random_index+1]\n", " yi = y[random_index:random_index+1]\n", " gradients = 2 * xi.T @ ((xi @ theta)-yi)\n", " eta = learning_schedule(epoch*m+i)\n", diff --git a/doc/src/week39/week39.do.txt b/doc/src/week39/week39.do.txt index 6ed5b157c..bc99fd2ec 100644 --- a/doc/src/week39/week39.do.txt +++ b/doc/src/week39/week39.do.txt @@ -913,11 +913,12 @@ import numpy as np import matplotlib.pyplot as plt from sklearn.linear_model import SGDRegressor -x = 2*np.random.rand(100,1) -y = 4+3*x+np.random.randn(100,1) +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) -xb = np.c_[np.ones((100,1)), x] -beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print(beta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) sgdreg.fit(x,y.ravel()) @@ -933,14 +934,14 @@ print(sgdreg.intercept_, sgdreg.coef_) We have also discussed Ridge regression where the loss function contains a regularized term given by the $L_2$ norm of $\beta$, !bt \[ -C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. \] !et In order to minimize $C_{\text{ridge}}(\beta)$ using GD we only have adjust the gradient as follows !bt \[ -\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ \end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). \] @@ -966,18 +967,18 @@ from matplotlib.ticker import LinearLocator, FormatStrFormatter import sys # the number of datapoints -m = 100 +n = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -XT_X = xb.T @ xb +X = np.c_[np.ones((m,1)), x] +XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 Id = lmbda* np.eye(XT_X.shape[0]) -beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) @@ -986,12 +987,12 @@ eta = 0.1 Niterations = 100 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta beta -= eta*gradients print(beta) -ypredict = xb @ beta -ypredict2 = xb @ beta_linreg +ypredict = X @ beta +ypredict2 = X @ beta_linreg plt.plot(x, ypredict, "r-") plt.plot(x, ypredict2, "b-") plt.plot(x, y ,'ro') @@ -1201,8 +1202,8 @@ m = 100 x = 2*np.random.rand(m,1) y = 4+3*x+np.random.randn(m,1) -xb = np.c_[np.ones((m,1)), x] -theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y) +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) print("Own inversion") print(theta_linreg) sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) @@ -1217,15 +1218,15 @@ Niterations = 1000 for iter in range(Niterations): - gradients = 2.0/m*xb.T @ ((xb @ theta)-y) + gradients = 2.0/m*X.T @ ((X @ theta)-y) theta -= eta*gradients -print("theta frm own gd") +print("theta from own gd") print(theta) xnew = np.array([[0],[2]]) -xbnew = np.c_[np.ones((2,1)), xnew] -ypredict = xbnew.dot(theta) -ypredict2 = xbnew.dot(theta_linreg) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) n_epochs = 50 @@ -1238,7 +1239,7 @@ theta = np.random.randn(2,1) for epoch in range(n_epochs): for i in range(m): random_index = np.random.randint(m) - xi = xb[random_index:random_index+1] + xi = X[random_index:random_index+1] yi = y[random_index:random_index+1] gradients = 2 * xi.T @ ((xi @ theta)-yi) eta = learning_schedule(epoch*m+i)