update

2021-11-02 22:14:17 +01:00
parent 41bd165c9f
commit b3cdd08038
11 changed files with 18241 additions and 2266 deletions
@@ -1,6 +1,7 @@
 <!--
-Automatically generated HTML file from DocOnce source
+HTML file automatically generated from DocOnce source
 (https://github.com/doconce/doconce/)
+doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week39-bs --no_mako
 -->
 <html>
 <head>
@@ -8,24 +9,20 @@ Automatically generated HTML file from DocOnce source
 <meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 <meta name="description" content="Week 39: Optimization and  Gradient Methods">
-
 <title>Week 39: Optimization and  Gradient Methods</title>
-
 <!-- Bootstrap style: bootstrap -->
+<!-- doconce format html week39.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week39-bs --no_mako -->
 <link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
 <!-- not necessary
 <link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
 -->
-
 <style type="text/css">
-
 /* Add scrollbar to dropdown menus in bootstrap navigation bar */
 .dropdown-menu {
   height: auto;
   max-height: 400px;
   overflow-x: hidden;
 }
-
 /* Adds an invisible element before each target to offset for the navigation
   bar */
 .anchor::before {
@@ -35,8 +32,6 @@ Automatically generated HTML file from DocOnce source
  margin:-50px 0 0; /* negative fixed header height */
 }
 </style>
-
-
 </head>

 <!-- tocinfo
@@ -147,7 +142,7 @@ Automatically generated HTML file from DocOnce source
              ('And a corresponding example using _scikit-learn_',
               2,
               None,
-               'and-a-corresponding-example-using-_scikit-learn_'),
+               'and-a-corresponding-example-using-scikit-learn'),
              ('Gradient descent and Ridge',
               2,
               None,
@@ -238,8 +233,6 @@ MathJax.Hub.Config({
 </script>


-
-    
 <!-- Bootstrap navigation bar -->
 <div class="navbar navbar-default navbar-fixed-top">
  <div class="navbar-header">
@@ -250,7 +243,6 @@ MathJax.Hub.Config({
    </button>
    <a class="navbar-brand" href="week39-bs.html">Week 39: Optimization and  Gradient Methods</a>
  </div>
-
  <div class="navbar-collapse collapse navbar-responsive-collapse">
    <ul class="nav navbar-nav navbar-right">
      <li class="dropdown">
@@ -298,7 +290,7 @@ MathJax.Hub.Config({
     <!-- navigation toc: --> <li><a href="._week39-bs040.html#the-hessian-matrix" style="font-size: 80%;">The Hessian matrix</a></li>
     <!-- navigation toc: --> <li><a href="._week39-bs041.html#simple-program" style="font-size: 80%;">Simple program</a></li>
     <!-- navigation toc: --> <li><a href="._week39-bs042.html#gradient-descent-example" style="font-size: 80%;">Gradient Descent Example</a></li>
-     <!-- navigation toc: --> <li><a href="._week39-bs043.html#and-a-corresponding-example-using-_scikit-learn_" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
+     <!-- navigation toc: --> <li><a href="._week39-bs043.html#and-a-corresponding-example-using-scikit-learn" style="font-size: 80%;">And a corresponding example using <b>scikit-learn</b></a></li>
     <!-- navigation toc: --> <li><a href="._week39-bs044.html#gradient-descent-and-ridge" style="font-size: 80%;">Gradient descent and Ridge</a></li>
     <!-- navigation toc: --> <li><a href="._week39-bs045.html#program-example-for-gradient-descent-with-ridge-regression" style="font-size: 80%;">Program example for gradient descent with Ridge Regression</a></li>
     <!-- navigation toc: --> <li><a href="._week39-bs046.html#using-gradient-descent-methods-limitations" style="font-size: 80%;">Using gradient descent methods, limitations</a></li>
@@ -337,36 +329,32 @@ MathJax.Hub.Config({
  </div>
 </div>
 </div> <!-- end of navigation bar -->
-
 <div class="container">
-
 <p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
-
 <a name="part0000"></a>
 <!-- ------------------- main content ---------------------- -->
-
-
-
 <div class="jumbotron">
-<center><h1>Week 39: Optimization and  Gradient Methods</h1></center>  <!-- document title -->
+<center>
+<h1>Week 39: Optimization and  Gradient Methods</h1>
+</center>  <!-- document title -->

-<p>
 <!-- author(s): Morten Hjorth-Jensen -->
-
 <center>
 <b>Morten Hjorth-Jensen</b> [1, 2]
 </center>
-
-<p>
 <!-- institution(s) -->
+<center>
+[1] <b>Department of Physics, University of Oslo</b>
+</center>
+<center>
+[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b>
+</center>
+<br>
+<center>
+<h4>Nov 2, 2021</h4>
+</center> <!-- date -->
+<br>

-<center>[1] <b>Department of Physics, University of Oslo</b></center>
-<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
-<br>
-<p>
-<center><h4>Oct 1, 2021</h4></center> <!-- date -->
-<br>
-<p>


 <p><a href="._week39-bs001.html" class="btn btn-primary btn-lg">Read &raquo;</a></p>
@@ -392,25 +380,18 @@ MathJax.Hub.Config({
  <li><a href="._week39-bs001.html">&raquo;</a></li>
 </ul>
 <!-- ------------------- end of main content --------------- -->
-
 </div>  <!-- end container -->
 <!-- include javascript, jQuery *first* -->
 <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
 <script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
-
 <!-- Bootstrap footer
 <footer>
 <a href="https://..."><img width="250" align=right src="https://..."></a>
 </footer>
 -->
-
-
 <center style="font-size:80%">
 <!-- copyright --> &copy; 1999-2021, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
 </center>
-
-
 </body>
 </html>
-    

@@ -1148,19 +1148,19 @@ C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta|
 \]
 !et

-In order to minimize $C_{\text{ridge}}(\beta)$ using GD we only have adjust the gradient as follows 
+In order to minimize $C_{\text{ridge}}(\beta)$ using GD we adjust the gradient as follows 
 !bt
 \[
 \nabla_\beta C_{\text{ridge}}(\beta)  = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\
 \sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\
-\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta).
+\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (\frac{1}{n}X^T(X\beta - \mathbf{y})+\lambda \beta).
 \]
 !et

 We can easily extend our program to minimize $C_{\text{ridge}}(\beta)$ using gradient descent and compare with the analytical solution given by 
 !bt
 \[
-\beta_{\text{ridge}} = \left(X^T X + \lambda I_{2 \times 2} \right)^{-1} X^T \mathbf{y}.
+\beta_{\text{ridge}} = \left(X^T X + n\lambda I_{2 \times 2} \right)^{-1} X^T \mathbf{y}.
 \]
 !et