corrected week 39

2024-09-23 06:17:13 +02:00
parent 9b1d005b9b
commit 194c454152
31 changed files with 5369 additions and 4915 deletions
@@ -800,8 +800,8 @@ const thebe_selector_output = ".output, .cell_output"
  </a>
 </li>
 <li class="toc-h2 nav-item toc-entry">
-  <a class="reference internal nav-link" href="#but-noen-of-these-can-compete-with-newton-s-method">
-   But noen of these can compete with Newton’s method
+  <a class="reference internal nav-link" href="#but-none-of-these-can-compete-with-newton-s-method">
+   But none of these can compete with Newton’s method
  </a>
 </li>
 <li class="toc-h2 nav-item toc-entry">
@@ -1268,8 +1268,8 @@ const thebe_selector_output = ".output, .cell_output"
  </a>
 </li>
 <li class="toc-h2 nav-item toc-entry">
-  <a class="reference internal nav-link" href="#but-noen-of-these-can-compete-with-newton-s-method">
-   But noen of these can compete with Newton’s method
+  <a class="reference internal nav-link" href="#but-none-of-these-can-compete-with-newton-s-method">
+   But none of these can compete with Newton’s method
  </a>
 </li>
 <li class="toc-h2 nav-item toc-entry">
@@ -1788,7 +1788,7 @@ which equals</p>
    <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">2</span><span class="o">*</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">6</span><span class="o">*</span><span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">]])</span>

 <span class="n">fig</span> <span class="o">=</span> <span class="n">pt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
-<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">gca</span><span class="p">(</span><span class="n">projection</span><span class="o">=</span><span class="s2">&quot;3d&quot;</span><span class="p">)</span>
+<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">add_subplot</span><span class="p">(</span><span class="n">projection</span> <span class="o">=</span> <span class="s1">&#39;3d&#39;</span><span class="p">)</span>

 <span class="n">xmesh</span><span class="p">,</span> <span class="n">ymesh</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mgrid</span><span class="p">[</span><span class="o">-</span><span class="mi">3</span><span class="p">:</span><span class="mi">3</span><span class="p">:</span><span class="mi">50</span><span class="n">j</span><span class="p">,</span><span class="o">-</span><span class="mi">3</span><span class="p">:</span><span class="mi">3</span><span class="p">:</span><span class="mi">50</span><span class="n">j</span><span class="p">]</span>
 <span class="n">fmesh</span> <span class="o">=</span> <span class="n">f</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">xmesh</span><span class="p">,</span> <span class="n">ymesh</span><span class="p">]))</span>
@@ -1797,21 +1797,10 @@ which equals</p>
 </div>
 </div>
 <div class="cell_output docutils container">
-<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
-<span class="ne">TypeError</span><span class="g g-Whitespace">                                 </span>Traceback (most recent call last)
-<span class="n">Cell</span> <span class="n">In</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">line</span> <span class="mi">18</span>
-<span class="g g-Whitespace">     </span><span class="mi">15</span>     <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">2</span><span class="o">*</span><span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">6</span><span class="o">*</span><span class="n">x</span><span class="p">[</span><span class="mi">1</span><span class="p">]])</span>
-<span class="g g-Whitespace">     </span><span class="mi">17</span> <span class="n">fig</span> <span class="o">=</span> <span class="n">pt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
-<span class="ne">---&gt; </span><span class="mi">18</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">gca</span><span class="p">(</span><span class="n">projection</span><span class="o">=</span><span class="s2">&quot;3d&quot;</span><span class="p">)</span>
-<span class="g g-Whitespace">     </span><span class="mi">20</span> <span class="n">xmesh</span><span class="p">,</span> <span class="n">ymesh</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mgrid</span><span class="p">[</span><span class="o">-</span><span class="mi">3</span><span class="p">:</span><span class="mi">3</span><span class="p">:</span><span class="mi">50</span><span class="n">j</span><span class="p">,</span><span class="o">-</span><span class="mi">3</span><span class="p">:</span><span class="mi">3</span><span class="p">:</span><span class="mi">50</span><span class="n">j</span><span class="p">]</span>
-<span class="g g-Whitespace">     </span><span class="mi">21</span> <span class="n">fmesh</span> <span class="o">=</span> <span class="n">f</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">xmesh</span><span class="p">,</span> <span class="n">ymesh</span><span class="p">]))</span>
-
-<span class="ne">TypeError</span>: gca() got an unexpected keyword argument &#39;projection&#39;
-</pre></div>
-</div>
-<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;Figure size 640x480 with 0 Axes&gt;
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;mpl_toolkits.mplot3d.art3d.Poly3DCollection at 0x1273ffac0&gt;
 </pre></div>
 </div>
+<img alt="_images/week39_82_1.png" src="_images/week39_82_1.png" />
 </div>
 </div>
 <p>And then as countor plot</p>
@@ -1823,6 +1812,9 @@ which equals</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<img alt="_images/week39_84_0.png" src="_images/week39_84_0.png" />
+</div>
 </div>
 <p>Find guesses</p>
 <div class="cell docutils container">
@@ -1846,6 +1838,11 @@ which equals</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[ 0.69230769 -0.38461539]
+</pre></div>
+</div>
+</div>
 </div>
 <p>What happened?</p>
 <div class="cell docutils container">
@@ -1857,6 +1854,12 @@ which equals</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[&lt;matplotlib.lines.Line2D at 0x1279bd460&gt;]
+</pre></div>
+</div>
+<img alt="_images/week39_90_1.png" src="_images/week39_90_1.png" />
+</div>
 </div>
 <p>Note that we did only one iteration here. We can easily add more using our previous guesses.</p>
 </div>
@@ -2018,7 +2021,8 @@ desirable properties such as:</p>
 <p>We revisit an example similar to what we had in the first homework set. We had a function  of the type</p>
 <div class="cell docutils container">
 <div class="cell_input docutils container">
-<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">m</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
+<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">m</span> <span class="o">=</span> <span class="mi">100</span>
+<span class="n">x</span> <span class="o">=</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">m</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
 <span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">m</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
 </pre></div>
 </div>
@@ -2144,6 +2148,16 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.39517073 4.17377547]
+[[3.7074705 ]
+ [3.25970297]]
+[[3.7074705 ]
+ [3.25970297]]
+</pre></div>
+</div>
+<img alt="_images/week39_153_1.png" src="_images/week39_153_1.png" />
+</div>
 </div>
 </div>
 <div class="section" id="and-a-corresponding-example-using-scikit-learn">
@@ -2169,6 +2183,13 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[3.95872127]
+ [3.20357736]]
+[3.9712983] [3.26048873]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="gradient-descent-and-ridge">
@@ -2264,6 +2285,16 @@ minimum of this function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Eigenvalues of Hessian Matrix:[0.25986026 4.46387965]
+[[4.18118338]
+ [2.86240935]]
+[[4.18234308]
+ [2.86143379]]
+</pre></div>
+</div>
+<img alt="_images/week39_166_1.png" src="_images/week39_166_1.png" />
+</div>
 </div>
 </div>
 <div class="section" id="using-gradient-descent-methods-limitations">
@@ -2340,6 +2371,41 @@ minimum of this function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&gt;0 f([0.74724774]) = 0.55838
+&gt;1 f([0.59779819]) = 0.35736
+&gt;2 f([0.47823856]) = 0.22871
+&gt;3 f([0.38259084]) = 0.14638
+&gt;4 f([0.30607268]) = 0.09368
+&gt;5 f([0.24485814]) = 0.05996
+&gt;6 f([0.19588651]) = 0.03837
+&gt;7 f([0.15670921]) = 0.02456
+&gt;8 f([0.12536737]) = 0.01572
+&gt;9 f([0.10029389]) = 0.01006
+&gt;10 f([0.08023512]) = 0.00644
+&gt;11 f([0.06418809]) = 0.00412
+&gt;12 f([0.05135047]) = 0.00264
+&gt;13 f([0.04108038]) = 0.00169
+&gt;14 f([0.0328643]) = 0.00108
+&gt;15 f([0.02629144]) = 0.00069
+&gt;16 f([0.02103315]) = 0.00044
+&gt;17 f([0.01682652]) = 0.00028
+&gt;18 f([0.01346122]) = 0.00018
+&gt;19 f([0.01076897]) = 0.00012
+&gt;20 f([0.00861518]) = 0.00007
+&gt;21 f([0.00689214]) = 0.00005
+&gt;22 f([0.00551372]) = 0.00003
+&gt;23 f([0.00441097]) = 0.00002
+&gt;24 f([0.00352878]) = 0.00001
+&gt;25 f([0.00282302]) = 0.00001
+&gt;26 f([0.00225842]) = 0.00001
+&gt;27 f([0.00180673]) = 0.00000
+&gt;28 f([0.00144539]) = 0.00000
+&gt;29 f([0.00115631]) = 0.00000
+</pre></div>
+</div>
+<img alt="_images/week39_169_1.png" src="_images/week39_169_1.png" />
+</div>
 </div>
 </div>
 <div class="section" id="same-code-but-now-with-momentum-gradient-descent">
@@ -2412,6 +2478,41 @@ minimum of this function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&gt;0 f([0.74724774]) = 0.55838
+&gt;1 f([0.54175461]) = 0.29350
+&gt;2 f([0.37175575]) = 0.13820
+&gt;3 f([0.24640494]) = 0.06072
+&gt;4 f([0.15951871]) = 0.02545
+&gt;5 f([0.1015491]) = 0.01031
+&gt;6 f([0.0638484]) = 0.00408
+&gt;7 f([0.03976851]) = 0.00158
+&gt;8 f([0.02459084]) = 0.00060
+&gt;9 f([0.01511937]) = 0.00023
+&gt;10 f([0.00925406]) = 0.00009
+&gt;11 f([0.00564365]) = 0.00003
+&gt;12 f([0.0034318]) = 0.00001
+&gt;13 f([0.00208188]) = 0.00000
+&gt;14 f([0.00126053]) = 0.00000
+&gt;15 f([0.00076202]) = 0.00000
+&gt;16 f([0.00046006]) = 0.00000
+&gt;17 f([0.00027746]) = 0.00000
+&gt;18 f([0.00016719]) = 0.00000
+&gt;19 f([0.00010067]) = 0.00000
+&gt;20 f([6.05804744e-05]) = 0.00000
+&gt;21 f([3.64373635e-05]) = 0.00000
+&gt;22 f([2.19069576e-05]) = 0.00000
+&gt;23 f([1.31664443e-05]) = 0.00000
+&gt;24 f([7.91100141e-06]) = 0.00000
+&gt;25 f([4.75216828e-06]) = 0.00000
+&gt;26 f([2.85408468e-06]) = 0.00000
+&gt;27 f([1.71384267e-06]) = 0.00000
+&gt;28 f([1.02900153e-06]) = 0.00000
+&gt;29 f([6.17748881e-07]) = 0.00000
+</pre></div>
+</div>
+<img alt="_images/week39_171_1.png" src="_images/week39_171_1.png" />
+</div>
 </div>
 </div>
 <div class="section" id="overview-video-on-stochastic-gradient-descent">
@@ -2607,6 +2708,11 @@ function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>gamma_j after 500 epochs: 9.97108e-05
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="code-with-a-number-of-minibatches-which-varies">
@@ -2683,6 +2789,21 @@ function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[4.08687303]
+ [3.00141727]]
+Eigenvalues of Hessian Matrix:[0.29653038 4.0474753 ]
+theta from own gd
+[[4.08687303]
+ [3.00141727]]
+theta from own sdg
+[[4.014934 ]
+ [3.0042992]]
+</pre></div>
+</div>
+<img alt="_images/week39_193_1.png" src="_images/week39_193_1.png" />
+</div>
 </div>
 </div>
 <div class="section" id="replace-or-not">
@@ -3011,6 +3132,12 @@ f'(x) = \cos\left(2\pi x + x^2\right)\left(2\pi + 2x\right)
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<img alt="_images/week39_236_0.png" src="_images/week39_236_0.png" />
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The max absolute difference is: 1.77636e-15
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="using-autograd">
@@ -3042,6 +3169,12 @@ experiment with other, possibly more complicated, functions as well.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The gradient of f1 evaluated at a = 1 using autograd is: 3
+The gradient of f1 evaluated at a = 1 by finding the analytic expression is: 3
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="autograd-with-more-complicated-functions">
@@ -3087,6 +3220,17 @@ being differentiated with respect to.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Evaluating at x1 = 1, x2 = 3
+------------------------------
+The derivative of f2 w.r.t x1: 12
+The analytical derivative of f2 w.r.t x1: 12
+
+The derivative of f2 w.r.t x2: -4
+The analytical derivative of f2 w.r.t x2: -4
+</pre></div>
+</div>
+</div>
 </div>
 <p>Note that the grad function will not produce the true gradient of the function. The true gradient of a function with two or more variables will produce a vector, where each element is the function differentiated w.r.t a variable.</p>
 </div>
@@ -3114,6 +3258,12 @@ being differentiated with respect to.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The computed gradient of f3 is:  [ 2.  3.  5.  7. 88.]
+The analytical gradient of f3 is:  [ 2.  3.  5.  7. 88.]
+</pre></div>
+</div>
+</div>
 </div>
 <p>Note that in this case, when sending an array as input argument, the
 output from Autograd is another array. This is the true gradient of
@@ -3146,6 +3296,12 @@ could expect form a gradient-evaluting function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The computed derivative of f4 at x = 2.7 is: 13.8759
+The analytical gradient of f4 at x = 2.7 is: 13.8759
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="more-autograd">
@@ -3169,6 +3325,11 @@ could expect form a gradient-evaluting function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The computed derivative of f5 at x = 2.7 is: 5.4
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="and-with-loops">
@@ -3202,6 +3363,12 @@ could expect form a gradient-evaluting function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The computed derivative of f6_for at x = 0.5 is: 3.95703
+The computed derivative of f6_while at x = 0.5 is: 3.95703
+</pre></div>
+</div>
+</div>
 </div>
 <div class="cell docutils container">
 <div class="cell_input docutils container">
@@ -3217,6 +3384,11 @@ could expect form a gradient-evaluting function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The analytical derivative of f6 at x = 0.5 is: 3.95703
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="using-recursion">
@@ -3253,6 +3425,12 @@ could expect form a gradient-evaluting function.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The computed derivative of f7 at n = 2 is: 1
+The analytical derivative of f7 at n = 2 is: 1
+</pre></div>
+</div>
+</div>
 </div>
 <p>Note that if n is equal to zero or one, Autograd will give an error message. This message appears when the output is independent on input.</p>
 </div>
@@ -3268,16 +3446,16 @@ could expect form a gradient-evaluting function.</p>
    <span class="n">x</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3</span>
    <span class="k">return</span> <span class="n">x</span><span class="o">*</span><span class="mi">2</span>

-<span class="n">f8_grad</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">f8</span><span class="p">)</span>
+<span class="c1">#f8_grad = grad(f8)</span>

-<span class="n">x</span> <span class="o">=</span> <span class="mf">8.4</span>
+<span class="c1">#x = 8.4</span>

-<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The derivative of f8 is:&quot;</span><span class="p">,</span><span class="n">f8_grad</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
+<span class="c1">#print(&quot;The derivative of f8 is:&quot;,f8_grad(x))</span>
 </pre></div>
 </div>
 </div>
 </div>
-<p>Here, Autograd tells us that an ‘ArrayBox’ does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible.</p>
+<p>Here, running this code, Autograd tells us that an ‘ArrayBox’ does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible.</p>
 </div>
 <div class="section" id="the-syntax-a-dot-b-when-finding-the-dot-product">
 <h2>The syntax a.dot(b) when finding the dot product<a class="headerlink" href="#the-syntax-a-dot-b-when-finding-the-dot-product" title="Permalink to this headline">¶</a></h2>
@@ -3289,11 +3467,11 @@ could expect form a gradient-evaluting function.</p>
    <span class="n">b</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">1.0</span><span class="p">,</span><span class="mf">2.0</span><span class="p">])</span>
    <span class="k">return</span> <span class="n">a</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">b</span><span class="p">)</span>

-<span class="n">f9_grad</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">f9</span><span class="p">)</span>
+<span class="c1">#f9_grad = grad(f9)</span>

-<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">1.0</span><span class="p">,</span><span class="mf">0.0</span><span class="p">])</span>
+<span class="c1">#x = np.array([1.0,0.0])</span>

-<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The derivative of f9 is:&quot;</span><span class="p">,</span><span class="n">f9_grad</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
+<span class="c1">#print(&quot;The derivative of f9 is:&quot;,f9_grad(x))</span>
 </pre></div>
 </div>
 </div>
@@ -3320,6 +3498,11 @@ which also computed the dot product can be used:</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The gradient of f9 is: [1. 2.]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="recommended-to-avoid">
@@ -3334,6 +3517,18 @@ which also computed the dot product can be used:</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
+<span class="ne">NameError</span><span class="g g-Whitespace">                                 </span>Traceback (most recent call last)
+<span class="n">Cell</span> <span class="n">In</span><span class="p">[</span><span class="mi">27</span><span class="p">],</span> <span class="n">line</span> <span class="mi">1</span>
+<span class="ne">----&gt; </span><span class="mi">1</span> <span class="n">a</span> <span class="o">+=</span> <span class="n">b</span>
+<span class="g g-Whitespace">      </span><span class="mi">2</span> <span class="n">a</span> <span class="o">-=</span> <span class="n">b</span>
+<span class="g g-Whitespace">      </span><span class="mi">3</span> <span class="n">a</span><span class="o">*=</span> <span class="n">b</span>
+
+<span class="ne">NameError</span>: name &#39;b&#39; is not defined
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="using-autograd-with-ols">
@@ -3395,6 +3590,18 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[3.94499279]
+ [3.03306538]]
+Eigenvalues of Hessian Matrix:[0.31248425 4.44418124]
+theta from own gd
+[[3.94499279]
+ [3.03306538]]
+</pre></div>
+</div>
+<img alt="_images/week39_265_1.png" src="_images/week39_265_1.png" />
+</div>
 </div>
 </div>
 <div class="section" id="id9">
@@ -3457,10 +3664,84 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[4.]
+ [3.]]
+Eigenvalues of Hessian Matrix:[0.30306759 4.65944342]
+0 [-20.62275788] [-25.73690387]
+1 [-0.03662823] [0.02925855]
+2 [-0.0342458] [0.02735547]
+3 [-0.03201832] [0.02557617]
+4 [-0.02993573] [0.0239126]
+5 [-0.0279886] [0.02235723]
+6 [-0.02616812] [0.02090304]
+7 [-0.02446604] [0.01954342]
+8 [-0.02287468] [0.01827225]
+9 [-0.02138683] [0.01708375]
+10 [-0.01999575] [0.01597256]
+11 [-0.01869515] [0.01493365]
+12 [-0.01747915] [0.01396231]
+13 [-0.01634224] [0.01305415]
+14 [-0.01527928] [0.01220505]
+15 [-0.01428546] [0.01141119]
+16 [-0.01335628] [0.01066897]
+17 [-0.01248754] [0.00997502]
+18 [-0.0116753] [0.0093262]
+19 [-0.01091589] [0.00871959]
+20 [-0.01020588] [0.00815244]
+21 [-0.00954206] [0.00762217]
+22 [-0.0089214] [0.0071264]
+23 [-0.00834112] [0.00666287]
+24 [-0.00779859] [0.00622949]
+25 [-0.00729134] [0.0058243]
+26 [-0.00681708] [0.00544547]
+27 [-0.00637367] [0.00509128]
+28 [-0.0059591] [0.00476012]
+29 [-0.0055715] [0.0044505]
+theta from own gd
+[[3.98281205]
+ [3.0137297 ]]
+0 [-0.00520911] [0.00416103]
+1 [-0.00487029] [0.00389038]
+2 [-0.00445186] [0.00355614]
+3 [-0.00403677] [0.00322456]
+4 [-0.00364967] [0.00291535]
+5 [-0.00329616] [0.00263296]
+6 [-0.00297571] [0.00237699]
+7 [-0.00268602] [0.00214559]
+8 [-0.00242441] [0.00193661]
+9 [-0.00218823] [0.00174795]
+10 [-0.00197505] [0.00157766]
+11 [-0.00178263] [0.00142396]
+12 [-0.00160895] [0.00128523]
+13 [-0.0014522] [0.00116001]
+14 [-0.00131072] [0.001047]
+15 [-0.00118302] [0.00094499]
+16 [-0.00106776] [0.00085292]
+17 [-0.00096373] [0.00076983]
+18 [-0.00086984] [0.00069482]
+19 [-0.00078509] [0.00062713]
+20 [-0.0007086] [0.00056603]
+21 [-0.00063957] [0.00051088]
+22 [-0.00057726] [0.00046111]
+23 [-0.00052102] [0.00041619]
+24 [-0.00047025] [0.00037564]
+25 [-0.00042444] [0.00033904]
+26 [-0.00038309] [0.00030601]
+27 [-0.00034576] [0.0002762]
+28 [-0.00031208] [0.00024929]
+29 [-0.00028167] [0.000225]
+theta from own gd wth momentum
+[[3.99916114]
+ [3.00067008]]
+</pre></div>
 </div>
 </div>
-<div class="section" id="but-noen-of-these-can-compete-with-newton-s-method">
-<h2>But noen of these can compete with Newton’s method<a class="headerlink" href="#but-noen-of-these-can-compete-with-newton-s-method" title="Permalink to this headline">¶</a></h2>
+</div>
+</div>
+<div class="section" id="but-none-of-these-can-compete-with-newton-s-method">
+<h2>But none of these can compete with Newton’s method<a class="headerlink" href="#but-none-of-these-can-compete-with-newton-s-method" title="Permalink to this headline">¶</a></h2>
 <div class="cell docutils container">
 <div class="cell_input docutils container">
 <div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Using Newton&#39;s method</span>
@@ -3504,6 +3785,22 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[4.26281735]
+ [2.84478178]]
+Eigenvalues of Hessian Matrix:[0.33978889 4.51659846]
+0 [-14.67490857] [-18.08539155]
+1 [-4.46170878e-15] [-1.45781905e-14]
+2 [-1.21430643e-15] [-1.24768421e-15]
+3 [4.92661467e-16] [4.50610071e-16]
+4 [4.92661467e-16] [4.50610071e-16]
+beta from own Newton code
+[[4.26281735]
+ [2.84478178]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="including-stochastic-gradient-descent-with-autograd">
@@ -3587,6 +3884,23 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[4.0586484]
+ [3.0718316]]
+Eigenvalues of Hessian Matrix:[0.29860173 3.8931686 ]
+theta from own gd
+[[4.0586484]
+ [3.0718316]]
+</pre></div>
+</div>
+<img alt="_images/week39_271_1.png" src="_images/week39_271_1.png" />
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>theta from own sdg
+[[4.02496085]
+ [3.12081773]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="id10">
@@ -3663,6 +3977,20 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[3.96075051]
+ [3.02148021]]
+Eigenvalues of Hessian Matrix:[0.27470622 4.24106503]
+theta from own gd
+[[3.95906059]
+ [3.02296298]]
+theta from own sdg with momentum
+[[3.95611042]
+ [2.99475306]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="similar-second-order-function-now-problem-but-now-with-adagrad">
@@ -3720,6 +4048,18 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[2.]
+ [3.]
+ [4.]]
+theta from own AdaGrad
+[[2.00025662]
+ [2.99802696]
+ [4.00167329]]
+</pre></div>
+</div>
+</div>
 </div>
 <p>Running this code we note an almost perfect agreement with the results from matrix inversion.</p>
 </div>
@@ -3784,6 +4124,18 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[2.]
+ [3.]
+ [4.]]
+theta from own RMSprop
+[[1.99456598]
+ [2.99848815]
+ [3.99835783]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="and-finally-adam">
@@ -3852,6 +4204,18 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
+[[2.]
+ [3.]
+ [4.]]
+theta from own ADAM
+[[1.99989103]
+ [3.00042093]
+ [3.99954925]]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="and-logistic-regression">
@@ -3894,6 +4258,12 @@ first example shows results with ordinary leats squares.</p>
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Initial loss: 2.772588722239781
+Trained loss: 1.067270675787016
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 <div class="section" id="introducing-jax">
@@ -3917,6 +4287,11 @@ It provides composable transformations of Python+NumPy programs: differentiate,
 </pre></div>
 </div>
 </div>
+<div class="cell_output docutils container">
+<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.25       0.19661197 0.10499357]
+</pre></div>
+</div>
+</div>
 </div>
 </div>
 </div>
@@ -532,7 +532,7 @@ def df(x):
    return np.array([2*x[0], 6*x[1]])

 fig = pt.figure()
-ax = fig.gca(projection="3d")
+ax = fig.add_subplot(projection = '3d')

 xmesh, ymesh = np.mgrid[-3:3:50j,-3:3:50j]
 fmesh = f(np.array([xmesh, ymesh]))
@@ -755,6 +755,7 @@ pt.plot(it_array.T[0], it_array.T[1], "x-")
 # In[6]:


+m = 100
 x = 2*np.random.rand(m,1)
 y = 4+3*x+np.random.randn(m,1)

@@ -2029,14 +2030,14 @@ def f8(x): # Assume x is an array
    x[2] = 3
    return x*2

-f8_grad = grad(f8)
+#f8_grad = grad(f8)

-x = 8.4
+#x = 8.4

-print("The derivative of f8 is:",f8_grad(x))
+#print("The derivative of f8 is:",f8_grad(x))


-# Here, Autograd tells us that an 'ArrayBox' does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible.
+# Here, running this code, Autograd tells us that an 'ArrayBox' does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible.

 # ## The syntax a.dot(b) when finding the dot product

@@ -2049,11 +2050,11 @@ def f9(a): # Assume a is an array with 2 elements
    b = np.array([1.0,2.0])
    return a.dot(b)

-f9_grad = grad(f9)
+#f9_grad = grad(f9)

-x = np.array([1.0,0.0])
+#x = np.array([1.0,0.0])

-print("The derivative of f9 is:",f9_grad(x))
+#print("The derivative of f9 is:",f9_grad(x))


 # Here we are told that the 'dot' function does not belong to Autograd's
@@ -2211,7 +2212,7 @@ print("theta from own gd wth momentum")
 print(theta)


-# ## But noen of these can compete with Newton's method
+# ## But none of these can compete with Newton's method

 # In[30]:

@@ -2658,3 +2659,9 @@ x_small = jnp.arange(3.)
 derivative_fn = grad(sum_logistic)
 print(derivative_fn(x_small))

+
+# In[ ]:
+
+
+
+