update

2024-09-23 06:34:06 +02:00
parent 194c454152
commit da4e5d0cdd
13 changed files with 174 additions and 1290 deletions
@@ -1797,7 +1797,7 @@ which equals</p>
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;mpl_toolkits.mplot3d.art3d.Poly3DCollection at 0x1273ffac0&gt;
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;mpl_toolkits.mplot3d.art3d.Poly3DCollection at 0x120cc1b20&gt;
 </pre></div>
 </div>
 <img alt="_images/week39_82_1.png" src="_images/week39_82_1.png" />
@@ -1855,7 +1855,7 @@ which equals</p>
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[&lt;matplotlib.lines.Line2D at 0x1279bd460&gt;]
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[&lt;matplotlib.lines.Line2D at 0x121cf68e0&gt;]
 </pre></div>
 </div>
 <img alt="_images/week39_90_1.png" src="_images/week39_90_1.png" />
@@ -2149,11 +2149,11 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.39517073 4.17377547]
-[[3.7074705 ]
- [3.25970297]]
-[[3.7074705 ]
- [3.25970297]]
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.28457291 3.97480604]
+[[3.34620097]
+ [3.61325173]]
+[[3.34620097]
+ [3.61325173]]
 </pre></div>
 </div>
 <img alt="_images/week39_153_1.png" src="_images/week39_153_1.png" />
@@ -2184,9 +2184,9 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[3.95872127]
- [3.20357736]]
-[3.9712983] [3.26048873]
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[4.08179906]
+ [3.05087018]]
+[4.03853268] [3.07469645]
 </pre></div>
 </div>
 </div>
@@ -2286,11 +2286,11 @@ minimum of this function.</p>
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.25986026 4.46387965]
-[[4.18118338]
- [2.86240935]]
-[[4.18234308]
- [2.86143379]]
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.32932635 4.53640693]
+[[4.15709522]
+ [2.9838929 ]]
+[[4.15605625]
+ [2.98473694]]
 </pre></div>
 </div>
 <img alt="_images/week39_166_1.png" src="_images/week39_166_1.png" />
@@ -3590,18 +3590,6 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[3.94499279]
- [3.03306538]]
-Eigenvalues of Hessian Matrix:[0.31248425 4.44418124]
-theta from own gd
-[[3.94499279]
- [3.03306538]]
-</pre></div>
-</div>
-<img alt="_images/week39_265_1.png" src="_images/week39_265_1.png" />
-</div>
 </div>
 </div>
 <div class="section" id="id9">
@@ -3664,80 +3652,6 @@ theta from own gd
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[4.]
- [3.]]
-Eigenvalues of Hessian Matrix:[0.30306759 4.65944342]
-0 [-20.62275788] [-25.73690387]
-1 [-0.03662823] [0.02925855]
-2 [-0.0342458] [0.02735547]
-3 [-0.03201832] [0.02557617]
-4 [-0.02993573] [0.0239126]
-5 [-0.0279886] [0.02235723]
-6 [-0.02616812] [0.02090304]
-7 [-0.02446604] [0.01954342]
-8 [-0.02287468] [0.01827225]
-9 [-0.02138683] [0.01708375]
-10 [-0.01999575] [0.01597256]
-11 [-0.01869515] [0.01493365]
-12 [-0.01747915] [0.01396231]
-13 [-0.01634224] [0.01305415]
-14 [-0.01527928] [0.01220505]
-15 [-0.01428546] [0.01141119]
-16 [-0.01335628] [0.01066897]
-17 [-0.01248754] [0.00997502]
-18 [-0.0116753] [0.0093262]
-19 [-0.01091589] [0.00871959]
-20 [-0.01020588] [0.00815244]
-21 [-0.00954206] [0.00762217]
-22 [-0.0089214] [0.0071264]
-23 [-0.00834112] [0.00666287]
-24 [-0.00779859] [0.00622949]
-25 [-0.00729134] [0.0058243]
-26 [-0.00681708] [0.00544547]
-27 [-0.00637367] [0.00509128]
-28 [-0.0059591] [0.00476012]
-29 [-0.0055715] [0.0044505]
-theta from own gd
-[[3.98281205]
- [3.0137297 ]]
-0 [-0.00520911] [0.00416103]
-1 [-0.00487029] [0.00389038]
-2 [-0.00445186] [0.00355614]
-3 [-0.00403677] [0.00322456]
-4 [-0.00364967] [0.00291535]
-5 [-0.00329616] [0.00263296]
-6 [-0.00297571] [0.00237699]
-7 [-0.00268602] [0.00214559]
-8 [-0.00242441] [0.00193661]
-9 [-0.00218823] [0.00174795]
-10 [-0.00197505] [0.00157766]
-11 [-0.00178263] [0.00142396]
-12 [-0.00160895] [0.00128523]
-13 [-0.0014522] [0.00116001]
-14 [-0.00131072] [0.001047]
-15 [-0.00118302] [0.00094499]
-16 [-0.00106776] [0.00085292]
-17 [-0.00096373] [0.00076983]
-18 [-0.00086984] [0.00069482]
-19 [-0.00078509] [0.00062713]
-20 [-0.0007086] [0.00056603]
-21 [-0.00063957] [0.00051088]
-22 [-0.00057726] [0.00046111]
-23 [-0.00052102] [0.00041619]
-24 [-0.00047025] [0.00037564]
-25 [-0.00042444] [0.00033904]
-26 [-0.00038309] [0.00030601]
-27 [-0.00034576] [0.0002762]
-28 [-0.00031208] [0.00024929]
-29 [-0.00028167] [0.000225]
-theta from own gd wth momentum
-[[3.99916114]
- [3.00067008]]
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 <div class="section" id="but-none-of-these-can-compete-with-newton-s-method">
@@ -3785,22 +3699,6 @@ theta from own gd wth momentum
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[4.26281735]
- [2.84478178]]
-Eigenvalues of Hessian Matrix:[0.33978889 4.51659846]
-0 [-14.67490857] [-18.08539155]
-1 [-4.46170878e-15] [-1.45781905e-14]
-2 [-1.21430643e-15] [-1.24768421e-15]
-3 [4.92661467e-16] [4.50610071e-16]
-4 [4.92661467e-16] [4.50610071e-16]
-beta from own Newton code
-[[4.26281735]
- [2.84478178]]
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 <div class="section" id="including-stochastic-gradient-descent-with-autograd">
@@ -3884,23 +3782,6 @@ beta from own Newton code
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[4.0586484]
- [3.0718316]]
-Eigenvalues of Hessian Matrix:[0.29860173 3.8931686 ]
-theta from own gd
-[[4.0586484]
- [3.0718316]]
-</pre></div>
-</div>
-<img alt="_images/week39_271_1.png" src="_images/week39_271_1.png" />
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own sdg
-[[4.02496085]
- [3.12081773]]
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 <div class="section" id="id10">
@@ -3977,20 +3858,6 @@ theta from own gd
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[3.96075051]
- [3.02148021]]
-Eigenvalues of Hessian Matrix:[0.27470622 4.24106503]
-theta from own gd
-[[3.95906059]
- [3.02296298]]
-theta from own sdg with momentum
-[[3.95611042]
- [2.99475306]]
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 <div class="section" id="similar-second-order-function-now-problem-but-now-with-adagrad">
@@ -4048,18 +3915,6 @@ theta from own sdg with momentum
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[2.]
- [3.]
- [4.]]
-theta from own AdaGrad
-[[2.00025662]
- [2.99802696]
- [4.00167329]]
-</pre></div>
-</div>
-</div>
 </div>
 <p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
 </div>
@@ -4124,18 +3979,6 @@ theta from own AdaGrad
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[2.]
- [3.]
- [4.]]
-theta from own RMSprop
-[[1.99456598]
- [2.99848815]
- [3.99835783]]
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 <div class="section" id="and-finally-adam">
@@ -4204,18 +4047,6 @@ theta from own RMSprop
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
-[[2.]
- [3.]
- [4.]]
-theta from own ADAM
-[[1.99989103]
- [3.00042093]
- [3.99954925]]
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 <div class="section" id="and-logistic-regression">
@@ -4258,12 +4089,6 @@ theta from own ADAM
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Initial loss: 2.772588722239781
-Trained loss: 1.067270675787016
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 <div class="section" id="introducing-jax">
@@ -4287,11 +4112,6 @@ It provides composable transformations of Python+NumPy programs: differentiate,
 </pre></div>
 </div>
 </div>
-<div class="cell_output docutils container">
-<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.25       0.19661197 0.10499357]
-</pre></div>
-</div>
-</div>
 </div>
 </div>
 </div>
@@ -2098,7 +2098,7 @@ a /=b
 # for linear regression and logistic regression using **autograd**. The
 # first example shows results with ordinary leats squares.

-# In[28]:
+# In[ ]:


 # Using Autograd to calculate gradients for OLS
@@ -2154,7 +2154,7 @@ plt.show()

 # ## Same code but now with momentum gradient descent

-# In[29]:
+# In[ ]:


 # Using Autograd to calculate gradients for OLS
@@ -2214,7 +2214,7 @@ print(theta)

 # ## But none of these can compete with Newton's method

-# In[30]:
+# In[ ]:


 # Using Newton's method
@@ -2260,7 +2260,7 @@ print(beta)
 # ## Including Stochastic Gradient Descent with Autograd
 # In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using **autograd**.

-# In[31]:
+# In[ ]:


 # Using Autograd to calculate gradients using SGD
@@ -2340,7 +2340,7 @@ print(theta)

 # ## Same code but now with momentum gradient descent

-# In[32]:
+# In[ ]:


 # Using Autograd to calculate gradients using SGD
@@ -2414,7 +2414,7 @@ print(theta)

 # ## Similar (second order function now) problem but now with AdaGrad

-# In[33]:
+# In[ ]:


 # Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent
@@ -2471,7 +2471,7 @@ print(theta)

 # ## RMSprop for adaptive learning rate with Stochastic Gradient Descent

-# In[34]:
+# In[ ]:


 # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
@@ -2532,7 +2532,7 @@ print(theta)

 # ## And finally [ADAM](https://arxiv.org/pdf/1412.6980.pdf)

-# In[35]:
+# In[ ]:


 # Using Autograd to calculate gradients using RMSprop  and Stochastic Gradient descent
@@ -2598,7 +2598,7 @@ print(theta)

 # ## And Logistic Regression

-# In[36]:
+# In[ ]:


 import autograd.numpy as np
@@ -2646,7 +2646,7 @@ print("Trained loss:", training_loss(weights))
 # 
 # Here's a simple example on how you can use **JAX** to compute the derivate of the logistic function.

-# In[37]:
+# In[ ]:


 import jax.numpy as jnp