corrected some typos

2024-09-24 05:56:44 +02:00
parent 7434b6372b
commit 9c19ea9bfd
16 changed files with 4122 additions and 2775 deletions
@@ -784,11 +784,6 @@ const thebe_selector_output = ".output, .cell_output"
   The syntax a.dot(b) when finding the dot product
  </a>
 </li>
- <li class="toc-h2 nav-item toc-entry">
-  <a class="reference internal nav-link" href="#recommended-to-avoid">
-   Recommended to avoid
-  </a>
- </li>
 <li class="toc-h2 nav-item toc-entry">
  <a class="reference internal nav-link" href="#using-autograd-with-ols">
   Using Autograd with OLS
@@ -1252,11 +1247,6 @@ const thebe_selector_output = ".output, .cell_output"
   The syntax a.dot(b) when finding the dot product
  </a>
 </li>
- <li class="toc-h2 nav-item toc-entry">
-  <a class="reference internal nav-link" href="#recommended-to-avoid">
-   Recommended to avoid
-  </a>
- </li>
 <li class="toc-h2 nav-item toc-entry">
  <a class="reference internal nav-link" href="#using-autograd-with-ols">
   Using Autograd with OLS
@@ -1331,10 +1321,8 @@ doconce format html week39.do.txt --no_mako -->
 <ul class="simple">
 <li><p>Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression and discussion of gradient methods</p></li>
 <li><p>Stochastic Gradient descent with examples and automatic differentiation (theme also for next week).</p></li>
-</ul>
-<!-- * [Video of lecture](https://youtu.be/bFRVuIJroHs) -->
-<!-- * Whiteboard notes TBA at <https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesSep28.pdf> -->
-<ul class="simple">
+<li><p><a class="reference external" href="https://youtu.be/ISGpTC28Vmk">Video of lecture</a></p></li>
+<li><p><a class="reference external" href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesSeptember23.pdf">Whiteboard notes</a></p></li>
 <li><p>Readings and Videos:</p>
 <ul>
 <li><p>These lecture notes</p></li>
@@ -1354,12 +1342,9 @@ doconce format html week39.do.txt --no_mako -->
 <li><p>Exercise for week 39 on how to write the abstract and the introduction of the report and how to include references.</p></li>
 <li><p>Work on project 1, in particular resampling methods like cross-validation and bootstrap. <strong>For more discussions of project 1, chapter 5 of Goodfellow et al is a good read, in particular sections 5.1-5.5 and 5.7-5.11</strong>.</p></li>
 <li><p><a class="reference external" href="https://youtu.be/tVW1ZDmZnwM">Video on how to write scientific reports recorded during one of the lab sessions</a></p></li>
-</ul>
-<p>These sections summarize neatly what we have done till now and point to what is coming with respect to deep learning.</p>
-<ul class="simple">
 <li><p>A general guideline can be found at <a class="reference external" href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/EvaluationGrading/EvaluationForm.md">https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/EvaluationGrading/EvaluationForm.md</a>.</p></li>
 </ul>
-<!-- rett opp tyrleif --></div>
+</div>
 <div class="section" id="lecture-monday-september-23-optimization-the-central-part-of-any-machine-learning-algortithm">
 <h2>Lecture Monday September 23, Optimization, the central part of any Machine Learning algortithm<a class="headerlink" href="#lecture-monday-september-23-optimization-the-central-part-of-any-machine-learning-algortithm" title="Permalink to this headline">¶</a></h2>
 <p>The first few slides here are a repetition from last week.</p>
@@ -1797,7 +1782,7 @@ which equals</p>
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;mpl_toolkits.mplot3d.art3d.Poly3DCollection at 0x120cc1b20&gt;
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;mpl_toolkits.mplot3d.art3d.Poly3DCollection at 0x11f13fb20&gt;
 </pre></div>
 </div>
 <img alt="_images/week39_82_1.png" src="_images/week39_82_1.png" />
@@ -1855,7 +1840,7 @@ which equals</p>
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[&lt;matplotlib.lines.Line2D at 0x121cf68e0&gt;]
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[&lt;matplotlib.lines.Line2D at 0x12e13a850&gt;]
 </pre></div>
 </div>
 <img alt="_images/week39_90_1.png" src="_images/week39_90_1.png" />
@@ -2149,11 +2134,11 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.28457291 3.97480604]
-[[3.34620097]
- [3.61325173]]
-[[3.34620097]
- [3.61325173]]
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.24602146 5.15830902]
+[[3.94388948]
+ [3.14880915]]
+[[3.94388948]
+ [3.14880915]]
 </pre></div>
 </div>
 <img alt="_images/week39_153_1.png" src="_images/week39_153_1.png" />
@@ -2184,9 +2169,9 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[4.08179906]
- [3.05087018]]
-[4.03853268] [3.07469645]
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[4.1567286 ]
+ [2.83641435]]
+[4.12466453] [2.80907609]
 </pre></div>
 </div>
 </div>
@@ -2286,11 +2271,11 @@ minimum of this function.</p>
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.32932635 4.53640693]
-[[4.15709522]
- [2.9838929 ]]
-[[4.15605625]
- [2.98473694]]
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.29902518 4.33006628]
+[[3.73252708]
+ [3.18093549]]
+[[3.73265885]
+ [3.1808228 ]]
 </pre></div>
 </div>
 <img alt="_images/week39_166_1.png" src="_images/week39_166_1.png" />
@@ -3505,32 +3490,6 @@ which also computed the dot product can be used:</p>
 </div>
 </div>
 </div>
-<div class="section" id="recommended-to-avoid">
-<h2>Recommended to avoid<a class="headerlink" href="#recommended-to-avoid" title="Permalink to this headline">¶</a></h2>
-<p>The documentation recommends to avoid inplace operations such as</p>
-<div class="cell docutils container">
-<div class="cell_input docutils container">
-<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">a</span> <span class="o">+=</span> <span class="n">b</span>
-<span class="n">a</span> <span class="o">-=</span> <span class="n">b</span>
-<span class="n">a</span><span class="o">*=</span> <span class="n">b</span>
-<span class="n">a</span> <span class="o">/=</span><span class="n">b</span>
-</pre></div>
-</div>
-</div>
-<div class="cell_output docutils container">
-<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
-<span class="ne">NameError</span><span class="g g-Whitespace">                                 </span>Traceback (most recent call last)
-<span class="n">Cell</span> <span class="n">In</span><span class="p">[</span><span class="mi">27</span><span class="p">],</span> <span class="n">line</span> <span class="mi">1</span>
-<span class="ne">----&gt; </span><span class="mi">1</span> <span class="n">a</span> <span class="o">+=</span> <span class="n">b</span>
-<span class="g g-Whitespace">      </span><span class="mi">2</span> <span class="n">a</span> <span class="o">-=</span> <span class="n">b</span>
-<span class="g g-Whitespace">      </span><span class="mi">3</span> <span class="n">a</span><span class="o">*=</span> <span class="n">b</span>
-
-<span class="ne">NameError</span>: name &#39;b&#39; is not defined
-</pre></div>
-</div>
-</div>
-</div>
-</div>
 <div class="section" id="using-autograd-with-ols">
 <h2>Using Autograd with OLS<a class="headerlink" href="#using-autograd-with-ols" title="Permalink to this headline">¶</a></h2>
 <p>We conclude the part on optmization by showing how we can make codes
@@ -3590,6 +3549,18 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[3.94499279]
+ [3.03306538]]
+Eigenvalues of Hessian Matrix:[0.31248425 4.44418124]
+theta from own gd
+[[3.94499279]
+ [3.03306538]]
+</pre></div>
+</div>
+<img alt="_images/week39_263_1.png" src="_images/week39_263_1.png" />
+</div>
 </div>
 </div>
 <div class="section" id="id9">
@@ -3652,6 +3623,80 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[4.]
+ [3.]]
+Eigenvalues of Hessian Matrix:[0.30306759 4.65944342]
+0 [-20.62275788] [-25.73690387]
+1 [-0.03662823] [0.02925855]
+2 [-0.0342458] [0.02735547]
+3 [-0.03201832] [0.02557617]
+4 [-0.02993573] [0.0239126]
+5 [-0.0279886] [0.02235723]
+6 [-0.02616812] [0.02090304]
+7 [-0.02446604] [0.01954342]
+8 [-0.02287468] [0.01827225]
+9 [-0.02138683] [0.01708375]
+10 [-0.01999575] [0.01597256]
+11 [-0.01869515] [0.01493365]
+12 [-0.01747915] [0.01396231]
+13 [-0.01634224] [0.01305415]
+14 [-0.01527928] [0.01220505]
+15 [-0.01428546] [0.01141119]
+16 [-0.01335628] [0.01066897]
+17 [-0.01248754] [0.00997502]
+18 [-0.0116753] [0.0093262]
+19 [-0.01091589] [0.00871959]
+20 [-0.01020588] [0.00815244]
+21 [-0.00954206] [0.00762217]
+22 [-0.0089214] [0.0071264]
+23 [-0.00834112] [0.00666287]
+24 [-0.00779859] [0.00622949]
+25 [-0.00729134] [0.0058243]
+26 [-0.00681708] [0.00544547]
+27 [-0.00637367] [0.00509128]
+28 [-0.0059591] [0.00476012]
+29 [-0.0055715] [0.0044505]
+theta from own gd
+[[3.98281205]
+ [3.0137297 ]]
+0 [-0.00520911] [0.00416103]
+1 [-0.00487029] [0.00389038]
+2 [-0.00445186] [0.00355614]
+3 [-0.00403677] [0.00322456]
+4 [-0.00364967] [0.00291535]
+5 [-0.00329616] [0.00263296]
+6 [-0.00297571] [0.00237699]
+7 [-0.00268602] [0.00214559]
+8 [-0.00242441] [0.00193661]
+9 [-0.00218823] [0.00174795]
+10 [-0.00197505] [0.00157766]
+11 [-0.00178263] [0.00142396]
+12 [-0.00160895] [0.00128523]
+13 [-0.0014522] [0.00116001]
+14 [-0.00131072] [0.001047]
+15 [-0.00118302] [0.00094499]
+16 [-0.00106776] [0.00085292]
+17 [-0.00096373] [0.00076983]
+18 [-0.00086984] [0.00069482]
+19 [-0.00078509] [0.00062713]
+20 [-0.0007086] [0.00056603]
+21 [-0.00063957] [0.00051088]
+22 [-0.00057726] [0.00046111]
+23 [-0.00052102] [0.00041619]
+24 [-0.00047025] [0.00037564]
+25 [-0.00042444] [0.00033904]
+26 [-0.00038309] [0.00030601]
+27 [-0.00034576] [0.0002762]
+28 [-0.00031208] [0.00024929]
+29 [-0.00028167] [0.000225]
+theta from own gd wth momentum
+[[3.99916114]
+ [3.00067008]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="but-none-of-these-can-compete-with-newton-s-method">
@@ -3699,6 +3744,22 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[4.26281735]
+ [2.84478178]]
+Eigenvalues of Hessian Matrix:[0.33978889 4.51659846]
+0 [-14.67490857] [-18.08539155]
+1 [-4.46170878e-15] [-1.45781905e-14]
+2 [-1.21430643e-15] [-1.24768421e-15]
+3 [4.92661467e-16] [4.50610071e-16]
+4 [4.92661467e-16] [4.50610071e-16]
+beta from own Newton code
+[[4.26281735]
+ [2.84478178]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="including-stochastic-gradient-descent-with-autograd">
@@ -3782,6 +3843,27 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[4.0586484]
+ [3.0718316]]
+Eigenvalues of Hessian Matrix:[0.29860173 3.8931686 ]
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own gd
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[4.0586484]
+ [3.0718316]]
+</pre></div>
+</div>
+<img alt="_images/week39_269_3.png" src="_images/week39_269_3.png" />
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own sdg
+[[4.02496085]
+ [3.12081773]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="id10">
@@ -3858,6 +3940,22 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[3.96075051]
+ [3.02148021]]
+Eigenvalues of Hessian Matrix:[0.27470622 4.24106503]
+theta from own gd
+[[3.95906059]
+ [3.02296298]]
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own sdg with momentum
+[[3.95611042]
+ [2.99475306]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="similar-second-order-function-now-problem-but-now-with-adagrad">
@@ -3915,6 +4013,20 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[2.]
+ [3.]
+ [4.]]
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own AdaGrad
+[[2.00025662]
+ [2.99802696]
+ [4.00167329]]
+</pre></div>
+</div>
+</div>
 </div>
 <p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
 </div>
@@ -3979,6 +4091,20 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[2.]
+ [3.]
+ [4.]]
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own RMSprop
+[[1.99456598]
+ [2.99848815]
+ [3.99835783]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="and-finally-adam">
@@ -4047,6 +4173,20 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[2.]
+ [3.]
+ [4.]]
+</pre></div>
+</div>
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own ADAM
+[[1.99989103]
+ [3.00042093]
+ [3.99954925]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="and-logistic-regression">
@@ -4089,6 +4229,12 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Initial loss: 2.772588722239781
+Trained loss: 1.067270675787016
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="introducing-jax">
@@ -4112,6 +4258,11 @@ It provides composable transformations of Python+NumPy programs: differentiate,
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.25       0.19661197 0.10499357]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 </div>
@@ -19,8 +19,10 @@
 #   * Repetition of Logistic regression equations and classification problems and discussion of Gradient methods. Examples on how to implement Logistic Regression and discussion of gradient methods
 # 
 #   * Stochastic Gradient descent with examples and automatic differentiation (theme also for next week).
-# <!-- * [Video of lecture](https://youtu.be/bFRVuIJroHs) -->
-# <!-- * Whiteboard notes TBA at <https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesSep28.pdf> -->
+# 
+#   * [Video of lecture](https://youtu.be/ISGpTC28Vmk)
+# 
+#   * [Whiteboard notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2024/NotesSeptember23.pdf)
 # 
 #   * Readings and Videos:
 # 
@@ -46,12 +48,7 @@
 # 
 #   * [Video on how to write scientific reports recorded during one of the lab sessions](https://youtu.be/tVW1ZDmZnwM)
 # 
-# These sections summarize neatly what we have done till now and point to what is coming with respect to deep learning. 
 #   * A general guideline can be found at <https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/EvaluationGrading/EvaluationForm.md>.
-# 
-#   
-# 
-# <!-- rett opp tyrleif -->

 # ## Lecture Monday September 23, Optimization, the central part of any Machine Learning algortithm
 # 
@@ -2080,25 +2077,13 @@ print("The gradient of f9 is:",f9_alternative_grad(x))
 # w.r.t x is (b_1, b_2).


-# ## Recommended to avoid
-# The documentation recommends to avoid inplace operations such as
-
-# In[27]:
-
-
-a += b
-a -= b
-a*= b
-a /=b
-
-
 # ## Using Autograd with OLS
 # 
 # We conclude the part on optmization by showing how we can make codes
 # for linear regression and logistic regression using **autograd**. The
 # first example shows results with ordinary leats squares.

-# In[ ]:
+# In[27]:


 # Using Autograd to calculate gradients for OLS
@@ -2154,7 +2139,7 @@ plt.show()

 # ## Same code but now with momentum gradient descent

-# In[ ]:
+# In[28]:


 # Using Autograd to calculate gradients for OLS
@@ -2214,7 +2199,7 @@ print(theta)

 # ## But none of these can compete with Newton's method

-# In[ ]:
+# In[29]:


 # Using Newton's method
@@ -2260,7 +2245,7 @@ print(beta)
 # ## Including Stochastic Gradient Descent with Autograd
 # In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using **autograd**.

-# In[ ]:
+# In[30]:


 # Using Autograd to calculate gradients using SGD
@@ -2340,7 +2325,7 @@ print(theta)

 # ## Same code but now with momentum gradient descent

-# In[ ]:
+# In[31]:


 # Using Autograd to calculate gradients using SGD
@@ -2414,7 +2399,7 @@ print(theta)

 # ## Similar (second order function now) problem but now with AdaGrad

-# In[ ]:
+# In[32]:


 # Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
@@ -2471,7 +2456,7 @@ print(theta)

 # ## RMSprop for adaptive learning rate with Stochastic Gradient Descent

-# In[ ]:
+# In[33]:


 # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
@@ -2532,7 +2517,7 @@ print(theta)

 # ## And finally [ADAM](https://arxiv.org/pdf/1412.6980.pdf)

-# In[ ]:
+# In[34]:


 # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
@@ -2598,7 +2583,7 @@ print(theta)

 # ## And Logistic Regression

-# In[ ]:
+# In[35]:


 import autograd.numpy as np
@@ -2646,7 +2631,7 @@ print("Trained loss:", training_loss(weights))
 # 
 # Here's a simple example on how you can use **JAX** to compute the derivate of the logistic function.

-# In[ ]:
+# In[36]:


 import jax.numpy as jnp
@@ -2659,9 +2644,3 @@ x_small = jnp.arange(3.)
 derivative_fn = grad(sum_logistic)
 print(derivative_fn(x_small))

-
-# In[ ]:
-
-
-
-