lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replacing hats with boldface

Browse Source
This commit is contained in:
mhjensen
2020-09-25 06:13:11 +02:00
parent 8566eec36d
commit 7d18cc5cf6
10 changed files with 132 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week39/html/._week39-bs040.html
+5 -4
View File
@@ -270,11 +270,12 @@ MathJax.Hub.Config({
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)), x]
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x]
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(beta_linreg)
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, eta0<span style="color: #666666">=0.1</span>)
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
doc/pub/week39/html/._week39-bs041.html
+2 -2
View File
@@ -264,13 +264,13 @@ MathJax.Hub.Config({
<p>
We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \),
$$
C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
$$
<p>
In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows
$$
\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\
\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta).
$$
doc/pub/week39/html/._week39-bs042.html
+7 -7
View File
@@ -272,18 +272,18 @@ MathJax.Hub.Config({
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sys</span>
<span style="color: #408080; font-style: italic"># the number of datapoints</span>
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
XT_X <span style="color: #666666">=</span> xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> xb
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
<span style="color: #408080; font-style: italic">#Ridge parameter lambda</span>
lmbda <span style="color: #666666">=</span> <span style="color: #666666">0.001</span>
Id <span style="color: #666666">=</span> lmbda<span style="color: #666666">*</span> np<span style="color: #666666">.</span>eye(XT_X<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>])
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(XT_X<span style="color: #666666">+</span>Id) <span style="color: #666666">@</span> xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(XT_X<span style="color: #666666">+</span>Id) <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y
<span style="color: #008000">print</span>(beta_linreg)
<span style="color: #408080; font-style: italic"># Start plain gradient descent</span>
beta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
@@ -292,12 +292,12 @@ eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span
Niterations <span style="color: #666666">=</span> <span style="color: #666666">100</span>
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> (xb <span style="color: #666666">@</span> (beta)<span style="color: #666666">-</span>y)<span style="color: #666666">+2*</span>lmbda<span style="color: #666666">*</span>beta
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>n<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> (X <span style="color: #666666">@</span> (beta)<span style="color: #666666">-</span>y)<span style="color: #666666">+2*</span>lmbda<span style="color: #666666">*</span>beta
beta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(beta)
ypredict <span style="color: #666666">=</span> xb <span style="color: #666666">@</span> beta
ypredict2 <span style="color: #666666">=</span> xb <span style="color: #666666">@</span> beta_linreg
ypredict <span style="color: #666666">=</span> X <span style="color: #666666">@</span> beta
ypredict2 <span style="color: #666666">=</span> X <span style="color: #666666">@</span> beta_linreg
plt<span style="color: #666666">.</span>plot(x, ypredict, <span style="color: #BA2121">&quot;r-&quot;</span>)
plt<span style="color: #666666">.</span>plot(x, ypredict2, <span style="color: #BA2121">&quot;b-&quot;</span>)
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">&#39;ro&#39;</span>)
doc/pub/week39/html/._week39-bs052.html
+8 -8
View File
@@ -275,8 +275,8 @@ m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Own inversion&quot;</span>)
<span style="color: #008000">print</span>(theta_linreg)
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, eta0<span style="color: #666666">=0.1</span>)
@@ -291,15 +291,15 @@ Niterations <span style="color: #666666">=</span> <span style="color: #666666">1
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xb <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta frm own gd&quot;</span>)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own gd&quot;</span>)
<span style="color: #008000">print</span>(theta)
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
xbnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
ypredict <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta)
ypredict2 <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta_linreg)
Xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
ypredict <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta)
ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta_linreg)
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
@@ -312,7 +312,7 @@ theta <span style="color: #666666">=</span> np<span style="color: #666666">.</sp
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> xb[random_index:random_index<span style="color: #666666">+1</span>]
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+1</span>]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
doc/pub/week39/html/week39-reveal.html
+22 -21
View File
@@ -1338,11 +1338,12 @@ plt.show()
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
x = <span style="color: #B452CD">2</span>*np.random.rand(<span style="color: #B452CD">100</span>,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(<span style="color: #B452CD">100</span>,<span style="color: #B452CD">1</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
xb = np.c_[np.ones((<span style="color: #B452CD">100</span>,<span style="color: #B452CD">1</span>)), x]
beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
<span style="color: #658b00">print</span>(beta_linreg)
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
sgdreg.fit(x,y.ravel())
@@ -1358,7 +1359,7 @@ sgdreg.fit(x,y.ravel())
We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \),
<p>&nbsp;<br>
$$
C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
$$
<p>&nbsp;<br>
@@ -1366,7 +1367,7 @@ $$
In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows
<p>&nbsp;<br>
$$
\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\
\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta).
$$
@@ -1396,18 +1397,18 @@ $$
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sys</span>
<span style="color: #228B22"># the number of datapoints</span>
m = <span style="color: #B452CD">100</span>
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
xb = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
XT_X = xb.T @ xb
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
<span style="color: #228B22">#Ridge parameter lambda</span>
lmbda = <span style="color: #B452CD">0.001</span>
Id = lmbda* np.eye(XT_X.shape[<span style="color: #B452CD">0</span>])
beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y
beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y
<span style="color: #658b00">print</span>(beta_linreg)
<span style="color: #228B22"># Start plain gradient descent</span>
beta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
@@ -1416,12 +1417,12 @@ eta = <span style="color: #B452CD">0.1</span>
Niterations = <span style="color: #B452CD">100</span>
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T @ (xb @ (beta)-y)+<span style="color: #B452CD">2</span>*lmbda*beta
gradients = <span style="color: #B452CD">2.0</span>/n*X.T @ (X @ (beta)-y)+<span style="color: #B452CD">2</span>*lmbda*beta
beta -= eta*gradients
<span style="color: #658b00">print</span>(beta)
ypredict = xb @ beta
ypredict2 = xb @ beta_linreg
ypredict = X @ beta
ypredict2 = X @ beta_linreg
plt.plot(x, ypredict, <span style="color: #CD5555">&quot;r-&quot;</span>)
plt.plot(x, ypredict2, <span style="color: #CD5555">&quot;b-&quot;</span>)
plt.plot(x, y ,<span style="color: #CD5555">&#39;ro&#39;</span>)
@@ -1659,8 +1660,8 @@ m = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
xb = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
@@ -1675,15 +1676,15 @@ Niterations = <span style="color: #B452CD">1000</span>
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T @ ((xb @ theta)-y)
gradients = <span style="color: #B452CD">2.0</span>/m*X.T @ ((X @ theta)-y)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta frm own gd&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
xbnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
ypredict = xbnew.dot(theta)
ypredict2 = xbnew.dot(theta_linreg)
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
n_epochs = <span style="color: #B452CD">50</span>
@@ -1696,7 +1697,7 @@ theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = np.random.randint(m)
xi = xb[random_index:random_index+<span style="color: #B452CD">1</span>]
xi = X[random_index:random_index+<span style="color: #B452CD">1</span>]
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
gradients = <span style="color: #B452CD">2</span> * xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
doc/pub/week39/html/week39-solarized.html
+22 -21
View File
@@ -1283,11 +1283,12 @@ plt.show()
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> SGDRegressor
x = <span style="color: #B452CD">2</span>*np.random.rand(<span style="color: #B452CD">100</span>,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(<span style="color: #B452CD">100</span>,<span style="color: #B452CD">1</span>)
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(n,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(n,<span style="color: #B452CD">1</span>)
xb = np.c_[np.ones((<span style="color: #B452CD">100</span>,<span style="color: #B452CD">1</span>)), x]
beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
X = np.c_[np.ones((n,<span style="color: #B452CD">1</span>)), x]
beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
<span style="color: #658b00">print</span>(beta_linreg)
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
sgdreg.fit(x,y.ravel())
@@ -1301,13 +1302,13 @@ sgdreg.fit(x,y.ravel())
<p>
We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \),
$$
C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
$$
<p>
In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows
$$
\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\
\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta).
$$
@@ -1334,18 +1335,18 @@ $$
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">sys</span>
<span style="color: #228B22"># the number of datapoints</span>
m = <span style="color: #B452CD">100</span>
n = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
xb = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
XT_X = xb.T @ xb
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
XT_X = X.T @ X
<span style="color: #228B22">#Ridge parameter lambda</span>
lmbda = <span style="color: #B452CD">0.001</span>
Id = lmbda* np.eye(XT_X.shape[<span style="color: #B452CD">0</span>])
beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y
beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y
<span style="color: #658b00">print</span>(beta_linreg)
<span style="color: #228B22"># Start plain gradient descent</span>
beta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)
@@ -1354,12 +1355,12 @@ eta = <span style="color: #B452CD">0.1</span>
Niterations = <span style="color: #B452CD">100</span>
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T @ (xb @ (beta)-y)+<span style="color: #B452CD">2</span>*lmbda*beta
gradients = <span style="color: #B452CD">2.0</span>/n*X.T @ (X @ (beta)-y)+<span style="color: #B452CD">2</span>*lmbda*beta
beta -= eta*gradients
<span style="color: #658b00">print</span>(beta)
ypredict = xb @ beta
ypredict2 = xb @ beta_linreg
ypredict = X @ beta
ypredict2 = X @ beta_linreg
plt.plot(x, ypredict, <span style="color: #CD5555">&quot;r-&quot;</span>)
plt.plot(x, ypredict2, <span style="color: #CD5555">&quot;b-&quot;</span>)
plt.plot(x, y ,<span style="color: #CD5555">&#39;ro&#39;</span>)
@@ -1584,8 +1585,8 @@ m = <span style="color: #B452CD">100</span>
x = <span style="color: #B452CD">2</span>*np.random.rand(m,<span style="color: #B452CD">1</span>)
y = <span style="color: #B452CD">4</span>+<span style="color: #B452CD">3</span>*x+np.random.randn(m,<span style="color: #B452CD">1</span>)
xb = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
X = np.c_[np.ones((m,<span style="color: #B452CD">1</span>)), x]
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;Own inversion&quot;</span>)
<span style="color: #658b00">print</span>(theta_linreg)
sgdreg = SGDRegressor(max_iter = <span style="color: #B452CD">50</span>, penalty=<span style="color: #8B008B; font-weight: bold">None</span>, eta0=<span style="color: #B452CD">0.1</span>)
@@ -1600,15 +1601,15 @@ Niterations = <span style="color: #B452CD">1000</span>
<span style="color: #8B008B; font-weight: bold">for</span> <span style="color: #658b00">iter</span> <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(Niterations):
gradients = <span style="color: #B452CD">2.0</span>/m*xb.T @ ((xb @ theta)-y)
gradients = <span style="color: #B452CD">2.0</span>/m*X.T @ ((X @ theta)-y)
theta -= eta*gradients
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta frm own gd&quot;</span>)
<span style="color: #658b00">print</span>(<span style="color: #CD5555">&quot;theta from own gd&quot;</span>)
<span style="color: #658b00">print</span>(theta)
xnew = np.array([[<span style="color: #B452CD">0</span>],[<span style="color: #B452CD">2</span>]])
xbnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
ypredict = xbnew.dot(theta)
ypredict2 = xbnew.dot(theta_linreg)
Xnew = np.c_[np.ones((<span style="color: #B452CD">2</span>,<span style="color: #B452CD">1</span>)), xnew]
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
n_epochs = <span style="color: #B452CD">50</span>
@@ -1621,7 +1622,7 @@ theta = np.random.randn(<span style="color: #B452CD">2</span>,<span style="color
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_epochs):
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(m):
random_index = np.random.randint(m)
xi = xb[random_index:random_index+<span style="color: #B452CD">1</span>]
xi = X[random_index:random_index+<span style="color: #B452CD">1</span>]
yi = y[random_index:random_index+<span style="color: #B452CD">1</span>]
gradients = <span style="color: #B452CD">2</span> * xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)
doc/pub/week39/html/week39.html
+22 -21
View File
@@ -1288,11 +1288,12 @@ plt<span style="color: #666666">.</span>show()
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> SGDRegressor
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(n,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(n,<span style="color: #666666">1</span>)
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">100</span>,<span style="color: #666666">1</span>)), x]
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((n,<span style="color: #666666">1</span>)), x]
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(beta_linreg)
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, eta0<span style="color: #666666">=0.1</span>)
sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.</span>ravel())
@@ -1306,13 +1307,13 @@ sgdreg<span style="color: #666666">.</span>fit(x,y<span style="color: #666666">.
<p>
We have also discussed Ridge regression where the loss function contains a regularized term given by the \( L_2 \) norm of \( \beta \),
$$
C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
$$
<p>
In order to minimize \( C_{\text{ridge}}(\beta) \) using GD we only have adjust the gradient as follows
$$
\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\
\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta).
$$
@@ -1339,18 +1340,18 @@ $$
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sys</span>
<span style="color: #408080; font-style: italic"># the number of datapoints</span>
m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
XT_X <span style="color: #666666">=</span> xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> xb
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
XT_X <span style="color: #666666">=</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X
<span style="color: #408080; font-style: italic">#Ridge parameter lambda</span>
lmbda <span style="color: #666666">=</span> <span style="color: #666666">0.001</span>
Id <span style="color: #666666">=</span> lmbda<span style="color: #666666">*</span> np<span style="color: #666666">.</span>eye(XT_X<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>])
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(XT_X<span style="color: #666666">+</span>Id) <span style="color: #666666">@</span> xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y
beta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(XT_X<span style="color: #666666">+</span>Id) <span style="color: #666666">@</span> X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y
<span style="color: #008000">print</span>(beta_linreg)
<span style="color: #408080; font-style: italic"># Start plain gradient descent</span>
beta <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)
@@ -1359,12 +1360,12 @@ eta <span style="color: #666666">=</span> <span style="color: #666666">0.1</span
Niterations <span style="color: #666666">=</span> <span style="color: #666666">100</span>
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> (xb <span style="color: #666666">@</span> (beta)<span style="color: #666666">-</span>y)<span style="color: #666666">+2*</span>lmbda<span style="color: #666666">*</span>beta
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>n<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> (X <span style="color: #666666">@</span> (beta)<span style="color: #666666">-</span>y)<span style="color: #666666">+2*</span>lmbda<span style="color: #666666">*</span>beta
beta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(beta)
ypredict <span style="color: #666666">=</span> xb <span style="color: #666666">@</span> beta
ypredict2 <span style="color: #666666">=</span> xb <span style="color: #666666">@</span> beta_linreg
ypredict <span style="color: #666666">=</span> X <span style="color: #666666">@</span> beta
ypredict2 <span style="color: #666666">=</span> X <span style="color: #666666">@</span> beta_linreg
plt<span style="color: #666666">.</span>plot(x, ypredict, <span style="color: #BA2121">&quot;r-&quot;</span>)
plt<span style="color: #666666">.</span>plot(x, ypredict2, <span style="color: #BA2121">&quot;b-&quot;</span>)
plt<span style="color: #666666">.</span>plot(x, y ,<span style="color: #BA2121">&#39;ro&#39;</span>)
@@ -1589,8 +1590,8 @@ m <span style="color: #666666">=</span> <span style="color: #666666">100</span>
x <span style="color: #666666">=</span> <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(m,<span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> <span style="color: #666666">4+3*</span>x<span style="color: #666666">+</span>np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(m,<span style="color: #666666">1</span>)
xb <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(xb<span style="color: #666666">.</span>T<span style="color: #666666">.</span>dot(xb))<span style="color: #666666">.</span>dot(xb<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>dot(y)
X <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((m,<span style="color: #666666">1</span>)), x]
theta_linreg <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linalg<span style="color: #666666">.</span>inv(X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> X) <span style="color: #666666">@</span> (X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> y)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Own inversion&quot;</span>)
<span style="color: #008000">print</span>(theta_linreg)
sgdreg <span style="color: #666666">=</span> SGDRegressor(max_iter <span style="color: #666666">=</span> <span style="color: #666666">50</span>, penalty<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, eta0<span style="color: #666666">=0.1</span>)
@@ -1605,15 +1606,15 @@ Niterations <span style="color: #666666">=</span> <span style="color: #666666">1
<span style="color: #008000; font-weight: bold">for</span> <span style="color: #008000">iter</span> <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(Niterations):
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>xb<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xb <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
gradients <span style="color: #666666">=</span> <span style="color: #666666">2.0/</span>m<span style="color: #666666">*</span>X<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((X <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>y)
theta <span style="color: #666666">-=</span> eta<span style="color: #666666">*</span>gradients
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta frm own gd&quot;</span>)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;theta from own gd&quot;</span>)
<span style="color: #008000">print</span>(theta)
xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">0</span>],[<span style="color: #666666">2</span>]])
xbnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
ypredict <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta)
ypredict2 <span style="color: #666666">=</span> xbnew<span style="color: #666666">.</span>dot(theta_linreg)
Xnew <span style="color: #666666">=</span> np<span style="color: #666666">.</span>c_[np<span style="color: #666666">.</span>ones((<span style="color: #666666">2</span>,<span style="color: #666666">1</span>)), xnew]
ypredict <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta)
ypredict2 <span style="color: #666666">=</span> Xnew<span style="color: #666666">.</span>dot(theta_linreg)
n_epochs <span style="color: #666666">=</span> <span style="color: #666666">50</span>
@@ -1626,7 +1627,7 @@ theta <span style="color: #666666">=</span> np<span style="color: #666666">.</sp
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n_epochs):
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m):
random_index <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randint(m)
xi <span style="color: #666666">=</span> xb[random_index:random_index<span style="color: #666666">+1</span>]
xi <span style="color: #666666">=</span> X[random_index:random_index<span style="color: #666666">+1</span>]
yi <span style="color: #666666">=</span> y[random_index:random_index<span style="color: #666666">+1</span>]
gradients <span style="color: #666666">=</span> <span style="color: #666666">2</span> <span style="color: #666666">*</span> xi<span style="color: #666666">.</span>T <span style="color: #666666">@</span> ((xi <span style="color: #666666">@</span> theta)<span style="color: #666666">-</span>yi)
eta <span style="color: #666666">=</span> learning_schedule(epoch<span style="color: #666666">*</span>m<span style="color: #666666">+</span>i)
doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/week39/ipynb/week39.ipynb
+22 -21
View File
@@ -1499,11 +1499,12 @@
"import matplotlib.pyplot as plt\n",
"from sklearn.linear_model import SGDRegressor\n",
"\n",
"x = 2*np.random.rand(100,1)\n",
"y = 4+3*x+np.random.randn(100,1)\n",
"n = 100\n",
"x = 2*np.random.rand(n,1)\n",
"y = 4+3*x+np.random.randn(n,1)\n",
"\n",
"xb = np.c_[np.ones((100,1)), x]\n",
"beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)\n",
"X = np.c_[np.ones((n,1)), x]\n",
"beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n",
"print(beta_linreg)\n",
"sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)\n",
"sgdreg.fit(x,y.ravel())\n",
@@ -1525,7 +1526,7 @@
"metadata": {},
"source": [
"$$\n",
"C_{\\text{ridge}}(\\beta) = ||X\\beta -\\mathbf{y}||^2 + \\lambda ||\\beta||^2, \\ \\lambda \\geq 0.\n",
"C_{\\text{ridge}}(\\beta) = \\frac{1}{n}||X\\beta -\\mathbf{y}||^2 + \\lambda ||\\beta||^2, \\ \\lambda \\geq 0.\n",
"$$"
]
},
@@ -1541,7 +1542,7 @@
"metadata": {},
"source": [
"$$\n",
"\\nabla_\\beta C_{\\text{ridge}}(\\beta) = 2\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n",
"\\nabla_\\beta C_{\\text{ridge}}(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n",
"\\sum_{i=1}^{100}\\left( x_i (\\beta_0+\\beta_1x_i)-y_ix_i\\right) \\\\\n",
"\\end{bmatrix} + 2\\lambda\\begin{bmatrix} \\beta_0 \\\\ \\beta_1\\end{bmatrix} = 2 (X^T(X\\beta - \\mathbf{y})+\\lambda \\beta).\n",
"$$"
@@ -1587,18 +1588,18 @@
"import sys\n",
"\n",
"# the number of datapoints\n",
"m = 100\n",
"n = 100\n",
"x = 2*np.random.rand(m,1)\n",
"y = 4+3*x+np.random.randn(m,1)\n",
"\n",
"xb = np.c_[np.ones((m,1)), x]\n",
"XT_X = xb.T @ xb\n",
"X = np.c_[np.ones((m,1)), x]\n",
"XT_X = X.T @ X\n",
"\n",
"#Ridge parameter lambda\n",
"lmbda = 0.001\n",
"Id = lmbda* np.eye(XT_X.shape[0])\n",
"\n",
"beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y\n",
"beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y\n",
"print(beta_linreg)\n",
"# Start plain gradient descent\n",
"beta = np.random.randn(2,1)\n",
@@ -1607,12 +1608,12 @@
"Niterations = 100\n",
"\n",
"for iter in range(Niterations):\n",
" gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta\n",
" gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta\n",
" beta -= eta*gradients\n",
"\n",
"print(beta)\n",
"ypredict = xb @ beta\n",
"ypredict2 = xb @ beta_linreg\n",
"ypredict = X @ beta\n",
"ypredict2 = X @ beta_linreg\n",
"plt.plot(x, ypredict, \"r-\")\n",
"plt.plot(x, ypredict2, \"b-\")\n",
"plt.plot(x, y ,'ro')\n",
@@ -1877,8 +1878,8 @@
"x = 2*np.random.rand(m,1)\n",
"y = 4+3*x+np.random.randn(m,1)\n",
"\n",
"xb = np.c_[np.ones((m,1)), x]\n",
"theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)\n",
"X = np.c_[np.ones((m,1)), x]\n",
"theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)\n",
"print(\"Own inversion\")\n",
"print(theta_linreg)\n",
"sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)\n",
@@ -1893,15 +1894,15 @@
"\n",
"\n",
"for iter in range(Niterations):\n",
" gradients = 2.0/m*xb.T @ ((xb @ theta)-y)\n",
" gradients = 2.0/m*X.T @ ((X @ theta)-y)\n",
" theta -= eta*gradients\n",
"print(\"theta frm own gd\")\n",
"print(\"theta from own gd\")\n",
"print(theta)\n",
"\n",
"xnew = np.array([[0],[2]])\n",
"xbnew = np.c_[np.ones((2,1)), xnew]\n",
"ypredict = xbnew.dot(theta)\n",
"ypredict2 = xbnew.dot(theta_linreg)\n",
"Xnew = np.c_[np.ones((2,1)), xnew]\n",
"ypredict = Xnew.dot(theta)\n",
"ypredict2 = Xnew.dot(theta_linreg)\n",
"\n",
"\n",
"n_epochs = 50\n",
@@ -1914,7 +1915,7 @@
"for epoch in range(n_epochs):\n",
" for i in range(m):\n",
" random_index = np.random.randint(m)\n",
" xi = xb[random_index:random_index+1]\n",
" xi = X[random_index:random_index+1]\n",
" yi = y[random_index:random_index+1]\n",
" gradients = 2 * xi.T @ ((xi @ theta)-yi)\n",
" eta = learning_schedule(epoch*m+i)\n",
doc/src/week39/week39.do.txt
+22 -21
View File
@@ -913,11 +913,12 @@ import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import SGDRegressor
x = 2*np.random.rand(100,1)
y = 4+3*x+np.random.randn(100,1)
n = 100
x = 2*np.random.rand(n,1)
y = 4+3*x+np.random.randn(n,1)
xb = np.c_[np.ones((100,1)), x]
beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
X = np.c_[np.ones((n,1)), x]
beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
print(beta_linreg)
sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)
sgdreg.fit(x,y.ravel())
@@ -933,14 +934,14 @@ print(sgdreg.intercept_, sgdreg.coef_)
We have also discussed Ridge regression where the loss function contains a regularized term given by the $L_2$ norm of $\beta$,
!bt
\[
C_{\text{ridge}}(\beta) = ||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0.
\]
!et
In order to minimize $C_{\text{ridge}}(\beta)$ using GD we only have adjust the gradient as follows
!bt
\[
\nabla_\beta C_{\text{ridge}}(\beta) = 2\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
\sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\
\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta).
\]
@@ -966,18 +967,18 @@ from matplotlib.ticker import LinearLocator, FormatStrFormatter
import sys
# the number of datapoints
m = 100
n = 100
x = 2*np.random.rand(m,1)
y = 4+3*x+np.random.randn(m,1)
xb = np.c_[np.ones((m,1)), x]
XT_X = xb.T @ xb
X = np.c_[np.ones((m,1)), x]
XT_X = X.T @ X
#Ridge parameter lambda
lmbda = 0.001
Id = lmbda* np.eye(XT_X.shape[0])
beta_linreg = np.linalg.inv(XT_X+Id) @ xb.T @ y
beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y
print(beta_linreg)
# Start plain gradient descent
beta = np.random.randn(2,1)
@@ -986,12 +987,12 @@ eta = 0.1
Niterations = 100
for iter in range(Niterations):
gradients = 2.0/m*xb.T @ (xb @ (beta)-y)+2*lmbda*beta
gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta
beta -= eta*gradients
print(beta)
ypredict = xb @ beta
ypredict2 = xb @ beta_linreg
ypredict = X @ beta
ypredict2 = X @ beta_linreg
plt.plot(x, ypredict, "r-")
plt.plot(x, ypredict2, "b-")
plt.plot(x, y ,'ro')
@@ -1201,8 +1202,8 @@ m = 100
x = 2*np.random.rand(m,1)
y = 4+3*x+np.random.randn(m,1)
xb = np.c_[np.ones((m,1)), x]
theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
X = np.c_[np.ones((m,1)), x]
theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y)
print("Own inversion")
print(theta_linreg)
sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1)
@@ -1217,15 +1218,15 @@ Niterations = 1000
for iter in range(Niterations):
gradients = 2.0/m*xb.T @ ((xb @ theta)-y)
gradients = 2.0/m*X.T @ ((X @ theta)-y)
theta -= eta*gradients
print("theta frm own gd")
print("theta from own gd")
print(theta)
xnew = np.array([[0],[2]])
xbnew = np.c_[np.ones((2,1)), xnew]
ypredict = xbnew.dot(theta)
ypredict2 = xbnew.dot(theta_linreg)
Xnew = np.c_[np.ones((2,1)), xnew]
ypredict = Xnew.dot(theta)
ypredict2 = Xnew.dot(theta_linreg)
n_epochs = 50
@@ -1238,7 +1239,7 @@ theta = np.random.randn(2,1)
for epoch in range(n_epochs):
for i in range(m):
random_index = np.random.randint(m)
xi = xb[random_index:random_index+1]
xi = X[random_index:random_index+1]
yi = y[random_index:random_index+1]
gradients = 2 * xi.T @ ((xi @ theta)-yi)
eta = learning_schedule(epoch*m+i)