lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2022-10-04 17:50:07 +02:00
parent ca80391647
commit e48ba29bba
225 changed files with 6376 additions and 4975 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/LectureNotes/_build/html/chapter10.html
+292 -130
View File
@@ -7,8 +7,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>14. Building a Feed Forward Neural Network &#8212; Applied Data Analysis and Machine Learning</title>
<link href="_static/css/theme.css" rel="stylesheet" />
<link href="_static/css/index.c5995385ac14fb8791e8eb36b4908be2.css" rel="stylesheet" />
<link href="_static/css/theme.css" rel="stylesheet">
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
<link rel="stylesheet"
@@ -31,31 +31,37 @@
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
<link rel="preload" as="script" href="_static/js/index.1c5a1a01449ed65a7b51.js">
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
<script src="_static/jquery.js"></script>
<script src="_static/underscore.js"></script>
<script src="_static/doctools.js"></script>
<script src="_static/togglebutton.js"></script>
<script src="_static/clipboard.min.js"></script>
<script src="_static/copybutton.js"></script>
<script>let toggleHintShow = 'Click to show';</script>
<script>let toggleHintHide = 'Click to hide';</script>
<script>let toggleOpenOnPrint = 'true';</script>
<script src="_static/togglebutton.js"></script>
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
<script src="_static/sphinx-book-theme.12a9622fbb08dcb3a2a40b2c02b83a57.js"></script>
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
const thebe_selector = ".thebe,.cell"
const thebe_selector_input = "pre"
const thebe_selector_output = ".output, .cell_output"
</script>
<script async="async" src="_static/sphinx-thebe.js"></script>
<script async="async" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="15. Solving Differential Equations with Deep Learning" href="chapter11.html" />
<link rel="prev" title="13. Neural networks" href="chapter9.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en" />
<meta name="docsearch:language" content="None">
<!-- Google Analytics -->
</head>
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
@@ -94,7 +100,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
About the course
</span>
@@ -116,7 +122,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Review of Statistics with Resampling Techniques and Linear Algebra
</span>
@@ -133,7 +139,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
From Regression to Support Vector Machines
</span>
@@ -170,7 +176,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Decision Trees, Ensemble Methods and Boosting
</span>
@@ -187,7 +193,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Dimensionality Reduction
</span>
@@ -204,7 +210,7 @@ const thebe_selector_output = ".output, .cell_output"
</a>
</li>
</ul>
<p class="caption" role="heading">
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Deep Learning Methods
</span>
@@ -281,7 +287,7 @@ const thebe_selector_output = ".output, .cell_output"
data-placement="left">.ipynb</button></a>
<!-- Download PDF via print -->
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
onClick="window.print()" data-toggle="tooltip" data-placement="left">.pdf</button>
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
</div>
</div>
@@ -299,7 +305,7 @@ const thebe_selector_output = ".output, .cell_output"
</div>
<!-- Table of contents -->
<div class="d-none d-md-block col-md-2 bd-toc show">
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
<div class="tocsection onthispage pt-5 pb-3">
<i class="fas fa-list"></i> Contents
@@ -463,7 +469,174 @@ const thebe_selector_output = ".output, .cell_output"
</div>
<div id="main-content" class="row">
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
<!-- Table of contents that is only displayed when printing the page -->
<div id="jb-print-docs-body" class="onlyprint">
<h1>Building a Feed Forward Neural Network</h1>
<!-- Table of contents -->
<div id="print-main-content">
<div id="jb-print-toc">
<div>
<h2> Contents </h2>
</div>
<nav aria-label="Page">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#defining-the-cost-function">
14.1. Defining the cost function
</a>
<ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#example-binary-classification-problem">
14.1.1. Example: binary classification problem
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#the-softmax-function">
14.1.2. The Softmax function
</a>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#developing-a-code-for-doing-neural-networks-with-back-propagation">
14.2. Developing a code for doing neural networks with back propagation
</a>
<ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#collect-and-pre-process-data">
14.2.1. Collect and pre-process data
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#train-and-test-datasets">
14.2.2. Train and test datasets
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#define-model-and-architecture">
14.2.3. Define model and architecture
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#layers">
14.2.4. Layers
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#feed-forward-pass">
14.2.5. Feed-forward pass
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#choose-cost-function-and-optimizer">
14.2.6. Choose cost function and optimizer
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#optimizing-the-cost-function">
14.2.7. Optimizing the cost function
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#regularization">
14.2.8. Regularization
</a>
</li>
<li class="toc-h3 nav-item toc-entry">
<a class="reference internal nav-link" href="#matrix-multiplication">
14.2.9. Matrix multiplication
</a>
</li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#improving-performance">
14.3. Improving performance
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#evaluate-model-performance-on-test-data">
14.4. Evaluate model performance on test data
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#adjust-hyperparameters">
14.5. Adjust hyperparameters
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#visualization">
14.6. Visualization
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#scikit-learn-implementation">
14.7. scikit-learn implementation
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#id1">
14.8. Visualization
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#building-neural-networks-in-tensorflow-and-keras">
14.9. Building neural networks in Tensorflow and Keras
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#the-breast-cancer-data-now-with-keras">
14.10. The Breast Cancer Data, now with Keras
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#fine-tuning-neural-network-hyperparameters">
14.11. Fine-tuning neural network hyperparameters
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#which-activation-function-should-i-use">
14.12. Which activation function should I use?
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#the-relu-function-family">
14.13. The RELU function family
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#batch-normalization">
14.14. Batch Normalization
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#dropout">
14.15. Dropout
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#gradient-clipping">
14.16. Gradient Clipping
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#a-top-down-perspective-on-neural-networks">
14.17. A top-down perspective on Neural networks
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#limitations-of-supervised-learning-with-deep-networks">
14.18. Limitations of supervised learning with deep networks
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div>
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
@@ -1101,7 +1274,7 @@ the <em>Hadamard product</em>, meaning element-wise multiplication.</p>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Old accuracy on training data: 0.1440501043841336
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1435,7 +1608,7 @@ Lambda = 10.0
Accuracy score on test set: 0.19166666666666668
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1444,7 +1617,7 @@ Lambda = 1e-05
Accuracy score on test set: 0.10555555555555556
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1453,7 +1626,7 @@ Lambda = 0.0001
Accuracy score on test set: 0.08611111111111111
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1462,7 +1635,7 @@ Lambda = 0.001
Accuracy score on test set: 0.10555555555555556
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1471,7 +1644,7 @@ Lambda = 0.01
Accuracy score on test set: 0.08888888888888889
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1480,7 +1653,7 @@ Lambda = 0.1
Accuracy score on test set: 0.08611111111111111
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1489,7 +1662,7 @@ Lambda = 1.0
Accuracy score on test set: 0.08888888888888889
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1498,11 +1671,11 @@ Lambda = 10.0
Accuracy score on test set: 0.09166666666666666
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1511,11 +1684,11 @@ Lambda = 1e-05
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1524,11 +1697,11 @@ Lambda = 0.0001
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1537,11 +1710,11 @@ Lambda = 0.001
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1550,11 +1723,11 @@ Lambda = 0.01
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1563,7 +1736,7 @@ Lambda = 0.1
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -1572,11 +1745,11 @@ Lambda = 1.0
Accuracy score on test set: 0.10555555555555556
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1585,11 +1758,11 @@ Lambda = 10.0
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1598,11 +1771,11 @@ Lambda = 1e-05
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1611,11 +1784,11 @@ Lambda = 0.0001
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1624,11 +1797,11 @@ Lambda = 0.001
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1637,11 +1810,11 @@ Lambda = 0.01
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1650,11 +1823,11 @@ Lambda = 0.1
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1663,11 +1836,11 @@ Lambda = 1.0
Accuracy score on test set: 0.07777777777777778
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-6-2572e3a4b38d&gt;:43: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:43: RuntimeWarning: overflow encountered in exp
exp_term = np.exp(self.z_o)
&lt;ipython-input-6-2572e3a4b38d&gt;:44: RuntimeWarning: invalid value encountered in true_divide
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/1630775253.py:44: RuntimeWarning: invalid value encountered in true_divide
self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
</pre></div>
</div>
@@ -1720,15 +1893,15 @@ Accuracy score on test set: 0.07777777777777778
</div>
</div>
<div class="cell_output docutils container">
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
&lt;ipython-input-4-16b8e3cda33a&gt;:4: RuntimeWarning: overflow encountered in exp
/var/folders/td/3yk470mj5p931p9dtkk0y6jw0000gn/T/ipykernel_94478/953065564.py:4: RuntimeWarning: overflow encountered in exp
return 1/(1 + np.exp(-x))
</pre></div>
</div>
@@ -2018,47 +2191,48 @@ Accuracy score on test set: 0.9055555555555556
Learning rate = 0.1
Lambda = 0.1
Accuracy score on test set: 0.8805555555555555
Learning rate = 0.1
Lambda = 1.0
Accuracy score on test set: 0.8722222222222222
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 0.1
Lambda = 1.0
Accuracy score on test set: 0.8722222222222222
Learning rate = 0.1
Lambda = 10.0
Accuracy score on test set: 0.8666666666666667
Learning rate = 1.0
Lambda = 1e-05
Accuracy score on test set: 0.08611111111111111
Learning rate = 1.0
Lambda = 0.0001
Accuracy score on test set: 0.10555555555555556
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 1.0
Lambda = 0.0001
Accuracy score on test set: 0.10555555555555556
Learning rate = 1.0
Lambda = 0.001
Accuracy score on test set: 0.10555555555555556
Learning rate = 1.0
Lambda = 0.01
Accuracy score on test set: 0.17777777777777778
Learning rate = 1.0
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 1.0
Lambda = 0.1
Accuracy score on test set: 0.08333333333333333
Learning rate = 1.0
Lambda = 1.0
Accuracy score on test set: 0.08888888888888889
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 1.0
Learning rate = 1.0
Lambda = 10.0
Accuracy score on test set: 0.09444444444444444
Learning rate = 10.0
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 10.0
Lambda = 1e-05
Accuracy score on test set: 0.17222222222222222
@@ -2069,22 +2243,22 @@ Accuracy score on test set: 0.11666666666666667
Learning rate = 10.0
Lambda = 0.001
Accuracy score on test set: 0.10555555555555556
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 10.0
Learning rate = 10.0
Lambda = 0.01
Accuracy score on test set: 0.1388888888888889
Learning rate = 10.0
Lambda = 0.1
Accuracy score on test set: 0.11388888888888889
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 10.0
Lambda = 1.0
Accuracy score on test set: 0.10555555555555556
Lambda = 0.1
Accuracy score on test set: 0.11388888888888889
Learning rate = 10.0
Lambda = 1.0
Accuracy score on test set: 0.10555555555555556
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 10.0
Lambda = 10.0
Accuracy score on test set: 0.09444444444444444
</pre></div>
@@ -2176,7 +2350,7 @@ and/or if you use <strong>anaconda</strong>, just write (or install from the gra
</div>
</div>
<div class="cell_output docutils container">
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt"> File</span><span class="nn"> &quot;&lt;ipython-input-12-364418e97433&gt;&quot;</span><span class="gt">, line </span><span class="mi">1</span>
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span> <span class="n">Input</span> <span class="n">In</span> <span class="p">[</span><span class="mi">12</span><span class="p">]</span>
<span class="n">conda</span> <span class="n">create</span> <span class="o">-</span><span class="n">n</span> <span class="n">tf</span> <span class="n">tensorflow</span>
<span class="o">^</span>
<span class="ne">SyntaxError</span>: invalid syntax
@@ -2774,54 +2948,42 @@ features).</p>
</div>
<div class='prev-next-bottom'>
<div id="prev">
<a class="left-prev" href="chapter9.html" title="previous page">
<i class="prevnext-label fas fa-angle-left"></i>
<div class="prevnext-info">
<p class="prevnext-label">previous</p>
<p class="prevnext-title"><span class="section-number">13. </span>Neural networks</p>
</div>
</a>
<!-- Previous / next buttons -->
<div class='prev-next-area'>
<a class='left-prev' id="prev-link" href="chapter9.html" title="previous page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title"><span class="section-number">13. </span>Neural networks</p>
</div>
</a>
<a class='right-next' id="next-link" href="chapter11.html" title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title"><span class="section-number">15. </span>Solving Differential Equations with Deep Learning</p>
</div>
<div id="next">
<a class="right-next" href="chapter11.html" title="next page">
<div class="prevnext-info">
<p class="prevnext-label">next</p>
<p class="prevnext-title"><span class="section-number">15. </span>Solving Differential Equations with Deep Learning</p>
</div>
<i class="prevnext-label fas fa-angle-right"></i>
</a>
</div>
<i class="fas fa-angle-right"></i>
</a>
</div>
</div>
</div>
<footer class="footer">
<div class="container">
<p>
By Morten Hjorth-Jensen<br/>
&copy; Copyright 2021.<br/>
</p>
</div>
</footer>
<p>
By Morten Hjorth-Jensen<br/>
&copy; Copyright 2021.<br/>
</p>
</footer>
</main>
</div>
</div>
<script src="_static/js/index.1c5a1a01449ed65a7b51.js"></script>
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
</body>
</html>