update
This commit is contained in:
Morten Hjorth-Jensen
@@ -7,8 +7,8 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>7. Optimization, the central part of any Machine Learning algortithm — Applied Data Analysis and Machine Learning</title>
|
||||
|
||||
<link href="_static/css/theme.css" rel="stylesheet" />
|
||||
<link href="_static/css/index.c5995385ac14fb8791e8eb36b4908be2.css" rel="stylesheet" />
|
||||
<link href="_static/css/theme.css" rel="stylesheet">
|
||||
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
|
||||
|
||||
|
||||
<link rel="stylesheet"
|
||||
@@ -31,31 +31,37 @@
|
||||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||||
|
||||
<link rel="preload" as="script" href="_static/js/index.1c5a1a01449ed65a7b51.js">
|
||||
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
|
||||
|
||||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||||
<script src="_static/jquery.js"></script>
|
||||
<script src="_static/underscore.js"></script>
|
||||
<script src="_static/doctools.js"></script>
|
||||
<script src="_static/togglebutton.js"></script>
|
||||
<script src="_static/clipboard.min.js"></script>
|
||||
<script src="_static/copybutton.js"></script>
|
||||
<script>let toggleHintShow = 'Click to show';</script>
|
||||
<script>let toggleHintHide = 'Click to hide';</script>
|
||||
<script>let toggleOpenOnPrint = 'true';</script>
|
||||
<script src="_static/togglebutton.js"></script>
|
||||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||||
<script src="_static/sphinx-book-theme.12a9622fbb08dcb3a2a40b2c02b83a57.js"></script>
|
||||
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
|
||||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||||
const thebe_selector = ".thebe,.cell"
|
||||
const thebe_selector_input = "pre"
|
||||
const thebe_selector_output = ".output, .cell_output"
|
||||
</script>
|
||||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||||
<script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||||
<link rel="index" title="Index" href="genindex.html" />
|
||||
<link rel="search" title="Search" href="search.html" />
|
||||
<link rel="next" title="8. Support Vector Machines, overarching aims" href="chapter5.html" />
|
||||
<link rel="prev" title="6. Logistic Regression" href="chapter4.html" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta name="docsearch:language" content="en" />
|
||||
<meta name="docsearch:language" content="None">
|
||||
|
||||
|
||||
<!-- Google Analytics -->
|
||||
|
||||
</head>
|
||||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||||
@@ -94,7 +100,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading">
|
||||
<p aria-level="2" class="caption" role="heading">
|
||||
<span class="caption-text">
|
||||
About the course
|
||||
</span>
|
||||
@@ -116,7 +122,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading">
|
||||
<p aria-level="2" class="caption" role="heading">
|
||||
<span class="caption-text">
|
||||
Review of Statistics with Resampling Techniques and Linear Algebra
|
||||
</span>
|
||||
@@ -133,7 +139,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading">
|
||||
<p aria-level="2" class="caption" role="heading">
|
||||
<span class="caption-text">
|
||||
From Regression to Support Vector Machines
|
||||
</span>
|
||||
@@ -170,7 +176,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading">
|
||||
<p aria-level="2" class="caption" role="heading">
|
||||
<span class="caption-text">
|
||||
Decision Trees, Ensemble Methods and Boosting
|
||||
</span>
|
||||
@@ -187,7 +193,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading">
|
||||
<p aria-level="2" class="caption" role="heading">
|
||||
<span class="caption-text">
|
||||
Dimensionality Reduction
|
||||
</span>
|
||||
@@ -204,7 +210,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<p class="caption" role="heading">
|
||||
<p aria-level="2" class="caption" role="heading">
|
||||
<span class="caption-text">
|
||||
Deep Learning Methods
|
||||
</span>
|
||||
@@ -281,7 +287,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
data-placement="left">.ipynb</button></a>
|
||||
<!-- Download PDF via print -->
|
||||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||||
onClick="window.print()" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||||
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -299,7 +305,7 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</div>
|
||||
|
||||
<!-- Table of contents -->
|
||||
<div class="d-none d-md-block col-md-2 bd-toc show">
|
||||
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
|
||||
|
||||
<div class="tocsection onthispage pt-5 pb-3">
|
||||
<i class="fas fa-list"></i> Contents
|
||||
@@ -407,7 +413,118 @@ const thebe_selector_output = ".output, .cell_output"
|
||||
</div>
|
||||
<div id="main-content" class="row">
|
||||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||||
|
||||
<!-- Table of contents that is only displayed when printing the page -->
|
||||
<div id="jb-print-docs-body" class="onlyprint">
|
||||
<h1>Optimization, the central part of any Machine Learning algortithm</h1>
|
||||
<!-- Table of contents -->
|
||||
<div id="print-main-content">
|
||||
<div id="jb-print-toc">
|
||||
|
||||
<div>
|
||||
<h2> Contents </h2>
|
||||
</div>
|
||||
<nav aria-label="Page">
|
||||
<ul class="visible nav section-nav flex-column">
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#steepest-descent">
|
||||
7.1. Steepest descent
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#convex-functions">
|
||||
7.2. Convex functions
|
||||
</a>
|
||||
<ul class="nav section-nav flex-column">
|
||||
<li class="toc-h3 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#some-simple-problems">
|
||||
7.2.1. Some simple problems
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#standard-steepest-descent">
|
||||
7.3. Standard steepest descent
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#conjugate-gradient-method">
|
||||
7.4. Conjugate gradient method
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#revisiting-our-linear-regression-solvers">
|
||||
7.5. Revisiting our Linear Regression Solvers
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#using-gradient-descent-methods-limitations">
|
||||
7.6. Using gradient descent methods, limitations
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#stochastic-gradient-descent-sgd">
|
||||
7.7. Stochastic Gradient Descent (SGD)
|
||||
</a>
|
||||
<ul class="nav section-nav flex-column">
|
||||
<li class="toc-h3 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#program-for-stochastic-gradient">
|
||||
7.7.1. Program for stochastic gradient
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#momentum-based-gd">
|
||||
7.8. Momentum based GD
|
||||
</a>
|
||||
<ul class="nav section-nav flex-column">
|
||||
<li class="toc-h3 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#rms-prop">
|
||||
7.8.1. RMS prop
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h3 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#adam-optimizer">
|
||||
7.8.2. ADAM optimizer
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#practical-tips">
|
||||
7.9. Practical tips
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#automatic-differentiation">
|
||||
7.10. Automatic differentiation
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h2 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#using-autograd-with-ols">
|
||||
7.11. Using Autograd with OLS
|
||||
</a>
|
||||
<ul class="nav section-nav flex-column">
|
||||
<li class="toc-h3 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#including-stochastic-gradient-descent-with-autograd">
|
||||
7.11.1. Including Stochastic Gradient Descent with Autograd
|
||||
</a>
|
||||
</li>
|
||||
<li class="toc-h3 nav-item toc-entry">
|
||||
<a class="reference internal nav-link" href="#and-logistic-regression">
|
||||
7.11.2. And Logistic Regression
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
|
||||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||||
@@ -782,11 +899,11 @@ which equals</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-1-7f2b3a6174c2>:18: MatplotlibDeprecationWarning: Calling gca() with keyword arguments was deprecated in Matplotlib 3.4. Starting two minor releases later, gca() will take no keyword arguments. The gca() function should only be used to get the current axes, or if no axes exist, create new axes with default keyword arguments. To create a new axes with non-default arguments, use plt.axes() or plt.subplot().
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94582/483257001.py:18: MatplotlibDeprecationWarning: Calling gca() with keyword arguments was deprecated in Matplotlib 3.4. Starting two minor releases later, gca() will take no keyword arguments. The gca() function should only be used to get the current axes, or if no axes exist, create new axes with default keyword arguments. To create a new axes with non-default arguments, use plt.axes() or plt.subplot().
|
||||
ax = fig.gca(projection="3d")
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><mpl_toolkits.mplot3d.art3d.Poly3DCollection at 0x11da42d90>
|
||||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><mpl_toolkits.mplot3d.art3d.Poly3DCollection at 0x128ee8850>
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapteroptimization_61_2.png" src="_images/chapteroptimization_61_2.png" />
|
||||
@@ -844,7 +961,7 @@ which equals</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[<matplotlib.lines.Line2D at 0x11e031e50>]
|
||||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[<matplotlib.lines.Line2D at 0x12946f2e0>]
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapteroptimization_69_1.png" src="_images/chapteroptimization_69_1.png" />
|
||||
@@ -1101,11 +1218,11 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.29588901 4.59955801]
|
||||
[[4.21985165]
|
||||
[2.77627886]]
|
||||
[[4.21985165]
|
||||
[2.77627886]]
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.34158665 3.94915262]
|
||||
[[3.97117751]
|
||||
[3.11850274]]
|
||||
[[3.97117751]
|
||||
[3.11850274]]
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapteroptimization_123_1.png" src="_images/chapteroptimization_123_1.png" />
|
||||
@@ -1134,9 +1251,9 @@ when <span class="math notranslate nohighlight">\(||\nabla_\beta C(\beta_k) || \
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[3.54163136]
|
||||
[3.31866499]]
|
||||
[3.59833875] [3.3594821]
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[4.40754621]
|
||||
[2.78752269]]
|
||||
[4.37713991] [2.77711437]
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1207,10 +1324,10 @@ C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta|
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[4.00673393]
|
||||
[2.84569271]]
|
||||
[[3.98017611]
|
||||
[2.86623151]]
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[3.94107596]
|
||||
[2.96620033]]
|
||||
[[3.96670977]
|
||||
[2.94212937]]
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapteroptimization_132_1.png" src="_images/chapteroptimization_132_1.png" />
|
||||
@@ -1460,15 +1577,15 @@ function.</p>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Own inversion
|
||||
[[3.99519225]
|
||||
[3.01509543]]
|
||||
Eigenvalues of Hessian Matrix:[0.29229741 4.7643536 ]
|
||||
[[3.95446837]
|
||||
[3.16961682]]
|
||||
Eigenvalues of Hessian Matrix:[0.31447174 4.32459186]
|
||||
theta from own gd
|
||||
[[3.99519225]
|
||||
[3.01509543]]
|
||||
[[3.95446837]
|
||||
[3.16961682]]
|
||||
theta from own sdg
|
||||
[[3.98756882]
|
||||
[3.02625928]]
|
||||
[[3.91682433]
|
||||
[3.13655438]]
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapteroptimization_148_1.png" src="_images/chapteroptimization_148_1.png" />
|
||||
@@ -2078,56 +2195,56 @@ The analytical derivative of f7 at n = 2 is: 1
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span>---------------------------------------------------------------------------
|
||||
TypeError Traceback (most recent call last)
|
||||
<ipython-input-22-cabc613b8702> in <module>
|
||||
9 x = 8.4
|
||||
10
|
||||
---> 11 print("The derivative of f8 is:",f8_grad(x))
|
||||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
|
||||
<span class="ne">TypeError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||||
<span class="nn">Input In [22],</span> in <span class="ni"><cell line: 11></span><span class="nt">()</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">7</span> <span class="n">f8_grad</span> <span class="o">=</span> <span class="n">grad</span><span class="p">(</span><span class="n">f8</span><span class="p">)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">x</span> <span class="o">=</span> <span class="mf">8.4</span>
|
||||
<span class="ne">---> </span><span class="mi">11</span> <span class="nb">print</span><span class="p">(</span><span class="s2">"The derivative of f8 is:"</span><span class="p">,</span><span class="n">f8_grad</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
|
||||
|
||||
~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py in nary_f(*args, **kwargs)
|
||||
18 else:
|
||||
19 x = tuple(args[i] for i in argnum)
|
||||
---> 20 return unary_operator(unary_f, x, *nary_op_args, **nary_op_kwargs)
|
||||
21 return nary_f
|
||||
22 return nary_operator
|
||||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:20,</span> in <span class="ni">unary_to_nary.<locals>.nary_operator.<locals>.nary_f</span><span class="nt">(*args, **kwargs)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">18</span> <span class="k">else</span><span class="p">:</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">19</span> <span class="n">x</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">args</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">argnum</span><span class="p">)</span>
|
||||
<span class="ne">---> </span><span class="mi">20</span> <span class="k">return</span> <span class="n">unary_operator</span><span class="p">(</span><span class="n">unary_f</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="o">*</span><span class="n">nary_op_args</span><span class="p">,</span> <span class="o">**</span><span class="n">nary_op_kwargs</span><span class="p">)</span>
|
||||
|
||||
~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/differential_operators.py in grad(fun, x)
|
||||
23 arguments as `fun`, but returns the gradient instead. The function `fun`
|
||||
24 should be scalar-valued. The gradient has the same type as the argument."""
|
||||
---> 25 vjp, ans = _make_vjp(fun, x)
|
||||
26 if not vspace(ans).size == 1:
|
||||
27 raise TypeError("Grad only applies to real scalar-output functions. "
|
||||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/differential_operators.py:25,</span> in <span class="ni">grad</span><span class="nt">(fun, x)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">18</span> <span class="nd">@unary_to_nary</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">19</span> <span class="k">def</span> <span class="nf">grad</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">20</span> <span class="sd">"""</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">21</span><span class="sd"> Returns a function which computes the gradient of `fun` with respect to</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">22</span><span class="sd"> positional argument number `argnum`. The returned function takes the same</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">23</span><span class="sd"> arguments as `fun`, but returns the gradient instead. The function `fun`</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">24</span><span class="sd"> should be scalar-valued. The gradient has the same type as the argument."""</span>
|
||||
<span class="ne">---> </span><span class="mi">25</span> <span class="n">vjp</span><span class="p">,</span> <span class="n">ans</span> <span class="o">=</span> <span class="n">_make_vjp</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">26</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">vspace</span><span class="p">(</span><span class="n">ans</span><span class="p">)</span><span class="o">.</span><span class="n">size</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">27</span> <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">"Grad only applies to real scalar-output functions. "</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">28</span> <span class="s2">"Try jacobian, elementwise_grad or holomorphic_grad."</span><span class="p">)</span>
|
||||
|
||||
~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py in make_vjp(fun, x)
|
||||
8 def make_vjp(fun, x):
|
||||
9 start_node = VJPNode.new_root()
|
||||
---> 10 end_value, end_node = trace(start_node, fun, x)
|
||||
11 if end_node is None:
|
||||
12 def vjp(g): return vspace(x).zeros()
|
||||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/core.py:10,</span> in <span class="ni">make_vjp</span><span class="nt">(fun, x)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">def</span> <span class="nf">make_vjp</span><span class="p">(</span><span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">start_node</span> <span class="o">=</span> <span class="n">VJPNode</span><span class="o">.</span><span class="n">new_root</span><span class="p">()</span>
|
||||
<span class="ne">---> </span><span class="mi">10</span> <span class="n">end_value</span><span class="p">,</span> <span class="n">end_node</span> <span class="o">=</span> <span class="n">trace</span><span class="p">(</span><span class="n">start_node</span><span class="p">,</span> <span class="n">fun</span><span class="p">,</span> <span class="n">x</span><span class="p">)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="k">if</span> <span class="n">end_node</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="k">def</span> <span class="nf">vjp</span><span class="p">(</span><span class="n">g</span><span class="p">):</span> <span class="k">return</span> <span class="n">vspace</span><span class="p">(</span><span class="n">x</span><span class="p">)</span><span class="o">.</span><span class="n">zeros</span><span class="p">()</span>
|
||||
|
||||
~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py in trace(start_node, fun, x)
|
||||
8 with trace_stack.new_trace() as t:
|
||||
9 start_box = new_box(x, t, start_node)
|
||||
---> 10 end_box = fun(start_box)
|
||||
11 if isbox(end_box) and end_box._trace == start_box._trace:
|
||||
12 return end_box._value, end_box._node
|
||||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/tracer.py:10,</span> in <span class="ni">trace</span><span class="nt">(start_node, fun, x)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="k">with</span> <span class="n">trace_stack</span><span class="o">.</span><span class="n">new_trace</span><span class="p">()</span> <span class="k">as</span> <span class="n">t</span><span class="p">:</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">start_box</span> <span class="o">=</span> <span class="n">new_box</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="n">start_node</span><span class="p">)</span>
|
||||
<span class="ne">---> </span><span class="mi">10</span> <span class="n">end_box</span> <span class="o">=</span> <span class="n">fun</span><span class="p">(</span><span class="n">start_box</span><span class="p">)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="k">if</span> <span class="n">isbox</span><span class="p">(</span><span class="n">end_box</span><span class="p">)</span> <span class="ow">and</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_trace</span> <span class="o">==</span> <span class="n">start_box</span><span class="o">.</span><span class="n">_trace</span><span class="p">:</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">12</span> <span class="k">return</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_value</span><span class="p">,</span> <span class="n">end_box</span><span class="o">.</span><span class="n">_node</span>
|
||||
|
||||
~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py in unary_f(x)
|
||||
13 else:
|
||||
14 subargs = subvals(args, zip(argnum, x))
|
||||
---> 15 return fun(*subargs, **kwargs)
|
||||
16 if isinstance(argnum, int):
|
||||
17 x = args[argnum]
|
||||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/autograd/wrap_util.py:15,</span> in <span class="ni">unary_to_nary.<locals>.nary_operator.<locals>.nary_f.<locals>.unary_f</span><span class="nt">(x)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">13</span> <span class="k">else</span><span class="p">:</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">14</span> <span class="n">subargs</span> <span class="o">=</span> <span class="n">subvals</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="nb">zip</span><span class="p">(</span><span class="n">argnum</span><span class="p">,</span> <span class="n">x</span><span class="p">))</span>
|
||||
<span class="ne">---> </span><span class="mi">15</span> <span class="k">return</span> <span class="n">fun</span><span class="p">(</span><span class="o">*</span><span class="n">subargs</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||||
|
||||
<ipython-input-22-cabc613b8702> in f8(x)
|
||||
2 from autograd import grad
|
||||
3 def f8(x): # Assume x is an array
|
||||
----> 4 x[2] = 3
|
||||
5 return x*2
|
||||
6
|
||||
<span class="nn">Input In [22],</span> in <span class="ni">f8</span><span class="nt">(x)</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="k">def</span> <span class="nf">f8</span><span class="p">(</span><span class="n">x</span><span class="p">):</span> <span class="c1"># Assume x is an array</span>
|
||||
<span class="ne">----> </span><span class="mi">4</span> <span class="n">x</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3</span>
|
||||
<span class="g g-Whitespace"> </span><span class="mi">5</span> <span class="k">return</span> <span class="n">x</span><span class="o">*</span><span class="mi">2</span>
|
||||
|
||||
TypeError: 'ArrayBox' object does not support item assignment
|
||||
<span class="ne">TypeError</span>: 'ArrayBox' object does not support item assignment
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -2395,54 +2512,42 @@ first example shows results with ordinary leats squares.</p>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<div class='prev-next-bottom'>
|
||||
|
||||
<div id="prev">
|
||||
<a class="left-prev" href="chapter4.html" title="previous page">
|
||||
<i class="prevnext-label fas fa-angle-left"></i>
|
||||
<div class="prevnext-info">
|
||||
<p class="prevnext-label">previous</p>
|
||||
<p class="prevnext-title"><span class="section-number">6. </span>Logistic Regression</p>
|
||||
</div>
|
||||
</a>
|
||||
<!-- Previous / next buttons -->
|
||||
<div class='prev-next-area'>
|
||||
<a class='left-prev' id="prev-link" href="chapter4.html" title="previous page">
|
||||
<i class="fas fa-angle-left"></i>
|
||||
<div class="prev-next-info">
|
||||
<p class="prev-next-subtitle">previous</p>
|
||||
<p class="prev-next-title"><span class="section-number">6. </span>Logistic Regression</p>
|
||||
</div>
|
||||
</a>
|
||||
<a class='right-next' id="next-link" href="chapter5.html" title="next page">
|
||||
<div class="prev-next-info">
|
||||
<p class="prev-next-subtitle">next</p>
|
||||
<p class="prev-next-title"><span class="section-number">8. </span>Support Vector Machines, overarching aims</p>
|
||||
</div>
|
||||
<div id="next">
|
||||
<a class="right-next" href="chapter5.html" title="next page">
|
||||
<div class="prevnext-info">
|
||||
<p class="prevnext-label">next</p>
|
||||
<p class="prevnext-title"><span class="section-number">8. </span>Support Vector Machines, overarching aims</p>
|
||||
</div>
|
||||
<i class="prevnext-label fas fa-angle-right"></i>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<i class="fas fa-angle-right"></i>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer class="footer">
|
||||
<div class="container">
|
||||
<p>
|
||||
|
||||
By Morten Hjorth-Jensen<br/>
|
||||
|
||||
© Copyright 2021.<br/>
|
||||
</p>
|
||||
</div>
|
||||
</footer>
|
||||
<p>
|
||||
|
||||
By Morten Hjorth-Jensen<br/>
|
||||
|
||||
© Copyright 2021.<br/>
|
||||
</p>
|
||||
</footer>
|
||||
</main>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="_static/js/index.1c5a1a01449ed65a7b51.js"></script>
|
||||
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user