From 2d6b94571fcb98f3af16ade9179f3e32726ad309 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Wed, 3 Nov 2021 05:58:42 +0100 Subject: [PATCH] cleaning up --- doc/pub/week39/html/week39-bs.html | 69 +-- doc/pub/week39/html/week39-reveal.html | 33 +- doc/pub/week39/html/week39-solarized.html | 34 +- doc/pub/week39/html/week39.html | 34 +- doc/pub/week39/ipynb/ipynb-week39-src.tar.gz | Bin 193 -> 193 bytes doc/pub/week39/ipynb/week39.ipynb | 575 ++++++++++--------- doc/src/week39/week39.do.txt | 28 +- 7 files changed, 466 insertions(+), 307 deletions(-) diff --git a/doc/pub/week39/html/week39-bs.html b/doc/pub/week39/html/week39-bs.html index 899adc049..27c968c67 100644 --- a/doc/pub/week39/html/week39-bs.html +++ b/doc/pub/week39/html/week39-bs.html @@ -147,6 +147,10 @@ doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'gradient-descent-and-ridge'), + ('The Hessian matrix for Ridge Regression', + 2, + None, + 'the-hessian-matrix-for-ridge-regression'), ('Program example for gradient descent with Ridge Regression', 2, None, @@ -292,36 +296,37 @@ MathJax.Hub.Config({
  • Gradient Descent Example
  • And a corresponding example using scikit-learn
  • Gradient descent and Ridge
    • -
  • Program example for gradient descent with Ridge Regression
    • -
  • Using gradient descent methods, limitations
    • -
  • Challenge yourself
    • -
  • Friday October 1
    • -
  • Stochastic Gradient Descent
    • -
  • Computation of gradients
    • -
  • SGD example
    • -
  • The gradient step
    • -
  • Simple example code
    • -
  • When do we stop?
    • -
  • Slightly different approach
    • -
  • Program for stochastic gradient
    • -
  • Momentum based GD
    • -
  • More on momentum based approaches
    • -
  • Momentum parameter
    • -
  • Second moment of the gradient
    • -
  • RMS prop
    • -
  • ADAM optimizer
    • -
  • Practical tips
    • -
  • Automatic differentiation
    • -
  • Using autograd
    • -
  • Autograd with more complicated functions
    • -
  • More complicated functions using the elements of their arguments directly
    • -
  • Functions using mathematical functions from Numpy
    • -
  • More autograd
    • -
  • And with loops
    • -
  • Using recursion
    • -
  • Unsupported functions
    • -
  • The syntax a.dot(b) when finding the dot product
    • -
  • Recommended to avoid
    • +
  • The Hessian matrix for Ridge Regression
    • +
  • Program example for gradient descent with Ridge Regression
    • +
  • Using gradient descent methods, limitations
    • +
  • Challenge yourself
    • +
  • Friday October 1
    • +
  • Stochastic Gradient Descent
    • +
  • Computation of gradients
    • +
  • SGD example
    • +
  • The gradient step
    • +
  • Simple example code
    • +
  • When do we stop?
    • +
  • Slightly different approach
    • +
  • Program for stochastic gradient
    • +
  • Momentum based GD
    • +
  • More on momentum based approaches
    • +
  • Momentum parameter
    • +
  • Second moment of the gradient
    • +
  • RMS prop
    • +
  • ADAM optimizer
    • +
  • Practical tips
    • +
  • Automatic differentiation
    • +
  • Using autograd
    • +
  • Autograd with more complicated functions
    • +
  • More complicated functions using the elements of their arguments directly
    • +
  • Functions using mathematical functions from Numpy
    • +
  • More autograd
    • +
  • And with loops
    • +
  • Using recursion
    • +
  • Unsupported functions
    • +
  • The syntax a.dot(b) when finding the dot product
    • +
  • Recommended to avoid
    • @@ -351,7 +356,7 @@ MathJax.Hub.Config({
    -

    Nov 2, 2021

    +

    Nov 3, 2021


    @@ -376,7 +381,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 75
    • +
  • 76
  • »
    • diff --git a/doc/pub/week39/html/week39-reveal.html b/doc/pub/week39/html/week39-reveal.html index bd6faa596..de1877466 100644 --- a/doc/pub/week39/html/week39-reveal.html +++ b/doc/pub/week39/html/week39-reveal.html @@ -184,7 +184,7 @@ MathJax.Hub.Config({
    -

    Nov 2, 2021

    +

    Nov 3, 2021


    @@ -1760,6 +1760,26 @@ $$

     
    +

    +

    The Hessian matrix for Ridge Regression

    +

    The Hessian matrix of Ridge Regression for our simple example is given by

    +

     
    +$$ +\boldsymbol{H} \equiv \begin{bmatrix} +\frac{\partial^2 C(\beta)}{\partial \beta_0^2} & \frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} \\ +\frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} & \frac{\partial^2 C(\beta)}{\partial \beta_1^2} & \\ +\end{bmatrix} = \frac{2}{n}X^T X+2\lambda\boldsymbol{I}. +$$ +

     
    + +

    This implies that the Hessian matrix is positive definite, hence the stationary point is a +minimum. +Note that the Ridge loss function is convex, as a sum of two convex +functions. Therefore, the stationary point is a global +minimum of this function. +

    +
    +

    Program example for gradient descent with Ridge Regression

    @@ -1787,14 +1807,21 @@ XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 -Id = lmbda* np.eye(XT_X.shape[0]) +Id = n*lmbda* np.eye(XT_X.shape[0]) + +# Hessian matrix +H = (2.0/n)* XT_X+2*lmbda +# Get the eigenvalues +EigValues, EigVectors = np.linalg.eig(H) +print(f"Eigenvalues of Hessian Matrix:{EigValues}") + beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) -eta = 0.1 +eta = 1.0/np.max(EigValues) Niterations = 100 for iter in range(Niterations): diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html index add781ae1..5bdeff9da 100644 --- a/doc/pub/week39/html/week39-solarized.html +++ b/doc/pub/week39/html/week39-solarized.html @@ -174,6 +174,10 @@ div.toc p,a { 2, None, 'gradient-descent-and-ridge'), + ('The Hessian matrix for Ridge Regression', + 2, + None, + 'the-hessian-matrix-for-ridge-regression'), ('Program example for gradient descent with Ridge Regression', 2, None, @@ -278,7 +282,7 @@ MathJax.Hub.Config({
    -

    Nov 2, 2021

    +

    Nov 3, 2021


    @@ -1701,6 +1705,23 @@ $$ $$ +









    +

    The Hessian matrix for Ridge Regression

    +

    The Hessian matrix of Ridge Regression for our simple example is given by

    +$$ +\boldsymbol{H} \equiv \begin{bmatrix} +\frac{\partial^2 C(\beta)}{\partial \beta_0^2} & \frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} \\ +\frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} & \frac{\partial^2 C(\beta)}{\partial \beta_1^2} & \\ +\end{bmatrix} = \frac{2}{n}X^T X+2\lambda\boldsymbol{I}. +$$ + +

    This implies that the Hessian matrix is positive definite, hence the stationary point is a +minimum. +Note that the Ridge loss function is convex, as a sum of two convex +functions. Therefore, the stationary point is a global +minimum of this function. +

    +









    Program example for gradient descent with Ridge Regression

    @@ -1728,14 +1749,21 @@ XT_X = X.T @ X #Ridge parameter lambda lmbda = 0.001 -Id = lmbda* np.eye(XT_X.shape[0]) +Id = n*lmbda* np.eye(XT_X.shape[0]) + +# Hessian matrix +H = (2.0/n)* XT_X+2*lmbda +# Get the eigenvalues +EigValues, EigVectors = np.linalg.eig(H) +print(f"Eigenvalues of Hessian Matrix:{EigValues}") + beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) -eta = 0.1 +eta = 1.0/np.max(EigValues) Niterations = 100 for iter in range(Niterations): diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html index 5c0ddf7a9..43c6269bc 100644 --- a/doc/pub/week39/html/week39.html +++ b/doc/pub/week39/html/week39.html @@ -251,6 +251,10 @@ div.toc p,a { 2, None, 'gradient-descent-and-ridge'), + ('The Hessian matrix for Ridge Regression', + 2, + None, + 'the-hessian-matrix-for-ridge-regression'), ('Program example for gradient descent with Ridge Regression', 2, None, @@ -355,7 +359,7 @@ MathJax.Hub.Config({
    -

    Nov 2, 2021

    +

    Nov 3, 2021


    @@ -1778,6 +1782,23 @@ $$ $$ +









    +

    The Hessian matrix for Ridge Regression

    +

    The Hessian matrix of Ridge Regression for our simple example is given by

    +$$ +\boldsymbol{H} \equiv \begin{bmatrix} +\frac{\partial^2 C(\beta)}{\partial \beta_0^2} & \frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} \\ +\frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} & \frac{\partial^2 C(\beta)}{\partial \beta_1^2} & \\ +\end{bmatrix} = \frac{2}{n}X^T X+2\lambda\boldsymbol{I}. +$$ + +

    This implies that the Hessian matrix is positive definite, hence the stationary point is a +minimum. +Note that the Ridge loss function is convex, as a sum of two convex +functions. Therefore, the stationary point is a global +minimum of this function. +

    +









    Program example for gradient descent with Ridge Regression

    @@ -1805,14 +1826,21 @@ XT_X = X.#Ridge parameter lambda lmbda = 0.001 -Id = lmbda* np.eye(XT_X.shape[0]) +Id = n*lmbda* np.eye(XT_X.shape[0]) + +# Hessian matrix +H = (2.0/n)* XT_X+2*lmbda +# Get the eigenvalues +EigValues, EigVectors = np.linalg.eig(H) +print(f"Eigenvalues of Hessian Matrix:{EigValues}") + beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y print(beta_linreg) # Start plain gradient descent beta = np.random.randn(2,1) -eta = 0.1 +eta = 1.0/np.max(EigValues) Niterations = 100 for iter in range(Niterations): diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz index 0b3705d4b867195a696bf9a9ea5b0440fe7b8903..696a61d56f30e57fe06e0ffe7bcdf7066c7372a8 100644 GIT binary patch literal 193 zcmV;y06za8iwFSs7J^{_1MSaC3c@fD2H>uHia9|^nl#0wU>7a~5igL^)W+JRCMnw6 z+Xv`MaZ^OdxA_@n7-kOHdb7(ScXz>J5JHSn7&B$blqjC-3C0OviYcO)5S9YMB#Bu7 zXt|SKI_tO@PHC#MP*$jSbHi9!e%Ld=0?+&t$4Xk*?t52h1xh=SYhA-lh?=Y-+4d@j vLJK>#z=&%njR3BC;6)*w)QVri*65Srjg7)zKjV3x=Y8z~z5Rxw00;m88X{P# literal 193 zcmV;y06za8iwFSQsexet1MSbv3c@f92k@Qu6nTQtuHA-5!5usZBECS!GFRtXZ7aIB zcORf9#mf+(zssMH5RzTKTy65m-CZynLP*Lv44I|Lm?WO+5v2)GPB~#Q&lm$jSrYRA z$b2WgT&)wPKc%gYs4S{?bNyIVeb_U-0?+&t`&wF9Z#!3O1xh