small update on grad descent

doc/src/Splines/Splines.do.txt

@@ -478,26 +478,6 @@ when $||\nabla_\beta C(\beta_k) || \leq \epsilon = 10^{-8}$.
 
 And finally we can compare our solution for $\beta$ with the analytic result given by 
 $\beta= (X^TX)^{-1} X^T \mathbf{y}$.
-!bc pycod 
-import numpy as np
-
-"""
-The following setup is just a suggestion, feel free to write it the way you like.
-"""
-
-#Setup problem described in the exercise
-N  = 100 #Nr of datapoints
-M  = 2 #Nr of features
-x  = np.random.rand(N) #Uniformly generated x-values in [0,1]
-y  = 5*x**2 + 0.1*np.random.randn(N)
-X  = np.c_[np.ones(N),x] #Construct design matrix
-
-#Compute beta according to normal equations to compare with GD solution
-Xt_X_inv = np.linalg.inv(np.dot(X.T,X))
-Xt_y     = np.dot(X.transpose(),y)
-beta_NE = np.dot(Xt_X_inv,Xt_y)
-print(beta_NE)
-!ec
 
 !split
 ===== Gradient Descent Example =====
@@ -514,17 +494,18 @@ from matplotlib import cm
 from matplotlib.ticker import LinearLocator, FormatStrFormatter
 import sys
 
-x = 2*np.random.rand(100,1)
-y = 4+3*x+np.random.randn(100,1)
+# the number of datapoints
+m = 100
+x = 2*np.random.rand(m,1)
+y = 4+3*x+np.random.randn(m,1)
 
-xb = np.c_[np.ones((100,1)), x]
+xb = np.c_[np.ones((m,1)), x]
 beta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
 print(beta_linreg)
 beta = np.random.randn(2,1)
 
 eta = 0.1
 Niterations = 1000
-m = 100
 
 for iter in range(Niterations):
     gradients = 2.0/m*xb.T.dot(xb.dot(beta)-y)
@@ -589,42 +570,13 @@ In order to minimize $C_{\text{ridge}}(\beta)$ using GD we only have adjust the
 \]
 !et
 
-We can now extend our program to minimize $C_{\text{ridge}}(\beta)$ using gradient descent and compare with the analytical solution given by 
+We can easily extend our program to minimize $C_{\text{ridge}}(\beta)$ using gradient descent and compare with the analytical solution given by 
 !bt
 \[
-\beta_{\text{ridge}} = \left(X^T X + \lambda I_{2 \times 2} \right)^{-1} X^T \mathbf{y},
+\beta_{\text{ridge}} = \left(X^T X + \lambda I_{2 \times 2} \right)^{-1} X^T \mathbf{y}.
 \]
 !et
-for $\lambda = {0,1,10,50,100}$ ($\lambda = 0$ corresponds to ordinary least squares). 
-We can then compute $||\beta_{\text{ridge}}||$ for each $\lambda$. 
 
-!bc pycod 
-import numpy as np
-
-"""
-The following setup is just a suggestion, feel free to write it the way you like.
-"""
-
-#Setup problem described in the exercise
-N  = 100 #Nr of datapoints
-M  = 2   #Nr of features
-x  = np.random.rand(N)
-y  = 5*x**2 + 0.1*np.random.randn(N)
-
-
-#Compute analytic beta for Ridge regression 
-X    = np.c_[np.ones(N),x]
-XT_X = np.dot(X.T,X)
-
-l  = 0.1 #Ridge parameter lambda
-Id = np.eye(XT_X.shape[0])
-
-Z = np.linalg.inv(XT_X+l*Id)
-beta_ridge = np.dot(Z,np.dot(X.T,y))
-
-print(beta_ridge)
-print(np.linalg.norm(beta_ridge)) #||beta||
-!ec
 
 !split
 ===== Stochastic Gradient Descent =====