Morten Hjorth-Jensen
1887 lines
188 KiB
HTML
1887 lines
188 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>17. Recurrent neural networks: Overarching view — Applied Data Analysis and Machine Learning</title>
|
||
|
||
<link href="_static/css/theme.css" rel="stylesheet">
|
||
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-book-theme.css?digest=c3fdc42140077d1ad13ad2f1588a4309" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||
|
||
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
|
||
|
||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||
<script src="_static/jquery.js"></script>
|
||
<script src="_static/underscore.js"></script>
|
||
<script src="_static/doctools.js"></script>
|
||
<script src="_static/clipboard.min.js"></script>
|
||
<script src="_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||
const thebe_selector = ".thebe,.cell"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output, .cell_output"
|
||
</script>
|
||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="prev" title="16. Convolutional Neural Networks" href="chapter12.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta name="docsearch:language" content="None">
|
||
|
||
|
||
<!-- Google Analytics -->
|
||
|
||
</head>
|
||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||
|
||
<div class="container-fluid" id="banner"></div>
|
||
|
||
|
||
|
||
<div class="container-xl">
|
||
<div class="row">
|
||
|
||
<div class="col-12 col-md-3 bd-sidebar site-navigation show" id="site-navigation">
|
||
|
||
<div class="navbar-brand-box">
|
||
<a class="navbar-brand text-wrap" href="index.html">
|
||
|
||
<!-- `logo` is deprecated in Sphinx 4.0, so remove this when we stop supporting 3 -->
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo" alt="logo">
|
||
|
||
|
||
<h1 class="site-logo" id="site-title">Applied Data Analysis and Machine Learning</h1>
|
||
|
||
</a>
|
||
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
|
||
<i class="icon fas fa-search"></i>
|
||
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search this book..." aria-label="Search this book..." autocomplete="off" >
|
||
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item active">
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
About the course
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="schedule.html">
|
||
Teaching schedule with links to material
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="teachers.html">
|
||
Teachers and Grading
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="textbooks.html">
|
||
Textbooks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Review of Statistics with Resampling Techniques and Linear Algebra
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="statistics.html">
|
||
1. Elements of Probability Theory and Statistical Data Analysis
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="linalg.html">
|
||
2. Linear Algebra, Handling of Arrays and more Python Features
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
From Regression to Support Vector Machines
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter1.html">
|
||
3. Linear Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter2.html">
|
||
4. Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter3.html">
|
||
5. Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter4.html">
|
||
6. Logistic Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapteroptimization.html">
|
||
7. Optimization, the central part of any Machine Learning algortithm
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter5.html">
|
||
8. Support Vector Machines, overarching aims
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Decision Trees, Ensemble Methods and Boosting
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter6.html">
|
||
9. Decision trees, overarching aims
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter7.html">
|
||
10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Dimensionality Reduction
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter8.html">
|
||
11. Basic ideas of the Principal Component Analysis (PCA)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="clustering.html">
|
||
12. Clustering and Unsupervised Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Deep Learning Methods
|
||
</span>
|
||
</p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter9.html">
|
||
13. Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter10.html">
|
||
14. Building a Feed Forward Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter11.html">
|
||
15. Solving Differential Equations with Deep Learning
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter12.html">
|
||
16. Convolutional Neural Networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1 current active">
|
||
<a class="current reference internal" href="#">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav> <!-- To handle the deprecated key -->
|
||
|
||
<div class="navbar_extra_footer">
|
||
Powered by <a href="https://jupyterbook.org">Jupyter Book</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
|
||
|
||
<div class="topbar container-xl fixed-top">
|
||
<div class="topbar-contents row">
|
||
<div class="col-12 col-md-3 bd-topbar-whitespace site-navigation show"></div>
|
||
<div class="col pl-md-4 topbar-main">
|
||
|
||
<button id="navbar-toggler" class="navbar-toggler ml-0" type="button" data-toggle="collapse"
|
||
data-toggle="tooltip" data-placement="bottom" data-target=".site-navigation" aria-controls="navbar-menu"
|
||
aria-expanded="true" aria-label="Toggle navigation" aria-controls="site-navigation"
|
||
title="Toggle navigation" data-toggle="tooltip" data-placement="left">
|
||
<i class="fas fa-bars"></i>
|
||
<i class="fas fa-arrow-left"></i>
|
||
<i class="fas fa-arrow-up"></i>
|
||
</button>
|
||
|
||
|
||
<div class="dropdown-buttons-trigger">
|
||
<button id="dropdown-buttons-trigger" class="btn btn-secondary topbarbtn" aria-label="Download this page"><i
|
||
class="fas fa-download"></i></button>
|
||
|
||
<div class="dropdown-buttons">
|
||
<!-- ipynb file if we had a myst markdown file -->
|
||
|
||
<!-- Download raw file -->
|
||
<a class="dropdown-buttons" href="_sources/chapter13.ipynb"><button type="button"
|
||
class="btn btn-secondary topbarbtn" title="Download source file" data-toggle="tooltip"
|
||
data-placement="left">.ipynb</button></a>
|
||
<!-- Download PDF via print -->
|
||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Source interaction buttons -->
|
||
|
||
<!-- Full screen (wrap in <a> to have style consistency -->
|
||
|
||
<a class="full-screen-button"><button type="button" class="btn btn-secondary topbarbtn" data-toggle="tooltip"
|
||
data-placement="bottom" onclick="toggleFullScreen()" aria-label="Fullscreen mode"
|
||
title="Fullscreen mode"><i
|
||
class="fas fa-expand"></i></button></a>
|
||
|
||
<!-- Launch buttons -->
|
||
|
||
</div>
|
||
|
||
<!-- Table of contents -->
|
||
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
|
||
|
||
<div class="tocsection onthispage pt-5 pb-3">
|
||
<i class="fas fa-list"></i> Contents
|
||
</div>
|
||
<nav id="bd-toc-nav" aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#a-simple-example">
|
||
17.1. A simple example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#an-extrapolation-example">
|
||
17.2. An extrapolation example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#predicting-new-points-with-a-trained-recurrent-neural-network">
|
||
17.3. Predicting New Points With A Trained Recurrent Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#other-types-of-recurrent-neural-networks">
|
||
17.4. Other Types of Recurrent Neural Networks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#generative-models">
|
||
18. Generative Models
|
||
</a>
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#generative-adversarial-networks">
|
||
18.1. Generative Adversarial Networks
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#writing-our-first-generative-adversarial-network">
|
||
18.2. Writing Our First Generative Adversarial Network
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mnist-and-gans">
|
||
18.2.1. MNIST and GANs
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="main-content" class="row">
|
||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||
<!-- Table of contents that is only displayed when printing the page -->
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Recurrent neural networks: Overarching view</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#a-simple-example">
|
||
17.1. A simple example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#an-extrapolation-example">
|
||
17.2. An extrapolation example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#predicting-new-points-with-a-trained-recurrent-neural-network">
|
||
17.3. Predicting New Points With A Trained Recurrent Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#other-types-of-recurrent-neural-networks">
|
||
17.4. Other Types of Recurrent Neural Networks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h1 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#generative-models">
|
||
18. Generative Models
|
||
</a>
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#generative-adversarial-networks">
|
||
18.1. Generative Adversarial Networks
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#writing-our-first-generative-adversarial-network">
|
||
18.2. Writing Our First Generative Adversarial Network
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mnist-and-gans">
|
||
18.2.1. MNIST and GANs
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html chapter13.do.txt --><div class="tex2jax_ignore mathjax_ignore section" id="recurrent-neural-networks-overarching-view">
|
||
<h1><span class="section-number">17. </span>Recurrent neural networks: Overarching view<a class="headerlink" href="#recurrent-neural-networks-overarching-view" title="Permalink to this headline">¶</a></h1>
|
||
<p>Till now our focus has been, including convolutional neural networks
|
||
as well, on feedforward neural networks. The output or the activations
|
||
flow only in one direction, from the input layer to the output layer.</p>
|
||
<p>A recurrent neural network (RNN) looks very much like a feedforward
|
||
neural network, except that it also has connections pointing
|
||
backward.</p>
|
||
<p>RNNs are used to analyze time series data such as stock prices, and
|
||
tell you when to buy or sell. In autonomous driving systems, they can
|
||
anticipate car trajectories and help avoid accidents. More generally,
|
||
they can work on sequences of arbitrary lengths, rather than on
|
||
fixed-sized inputs like all the nets we have discussed so far. For
|
||
example, they can take sentences, documents, or audio samples as
|
||
input, making them extremely useful for natural language processing
|
||
systems such as automatic translation and speech-to-text.</p>
|
||
<p>More to text to be added</p>
|
||
<div class="section" id="a-simple-example">
|
||
<h2><span class="section-number">17.1. </span>A simple example<a class="headerlink" href="#a-simple-example" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
|
||
|
||
<span class="c1"># Start importing packages</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">datasets</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">models</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.layers</span> <span class="kn">import</span> <span class="n">Input</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.models</span> <span class="kn">import</span> <span class="n">Model</span><span class="p">,</span> <span class="n">Sequential</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.layers</span> <span class="kn">import</span> <span class="n">Dense</span><span class="p">,</span> <span class="n">SimpleRNN</span><span class="p">,</span> <span class="n">LSTM</span><span class="p">,</span> <span class="n">GRU</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">optimizers</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">regularizers</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.utils</span> <span class="kn">import</span> <span class="n">to_categorical</span>
|
||
|
||
|
||
|
||
<span class="c1"># convert into dataset matrix</span>
|
||
<span class="k">def</span> <span class="nf">convertToMatrix</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">step</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">Y</span> <span class="o">=</span><span class="p">[],</span> <span class="p">[]</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">-</span><span class="n">step</span><span class="p">):</span>
|
||
<span class="n">d</span><span class="o">=</span><span class="n">i</span><span class="o">+</span><span class="n">step</span>
|
||
<span class="n">X</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="n">d</span><span class="p">,])</span>
|
||
<span class="n">Y</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">d</span><span class="p">,])</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">X</span><span class="p">),</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">Y</span><span class="p">)</span>
|
||
|
||
<span class="n">step</span> <span class="o">=</span> <span class="mi">4</span>
|
||
<span class="n">N</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
<span class="n">Tp</span> <span class="o">=</span> <span class="mi">800</span>
|
||
|
||
<span class="n">t</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="n">N</span><span class="p">)</span>
|
||
<span class="n">x</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">sin</span><span class="p">(</span><span class="mf">0.02</span><span class="o">*</span><span class="n">t</span><span class="p">)</span><span class="o">+</span><span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">N</span><span class="p">)</span>
|
||
<span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">df</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="n">values</span><span class="o">=</span><span class="n">df</span><span class="o">.</span><span class="n">values</span>
|
||
<span class="n">train</span><span class="p">,</span><span class="n">test</span> <span class="o">=</span> <span class="n">values</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="n">Tp</span><span class="p">,:],</span> <span class="n">values</span><span class="p">[</span><span class="n">Tp</span><span class="p">:</span><span class="n">N</span><span class="p">,:]</span>
|
||
|
||
<span class="c1"># add step elements into train and test</span>
|
||
<span class="n">test</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">test</span><span class="p">,</span><span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">test</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,],</span><span class="n">step</span><span class="p">))</span>
|
||
<span class="n">train</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">train</span><span class="p">,</span><span class="n">np</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="n">train</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,],</span><span class="n">step</span><span class="p">))</span>
|
||
|
||
<span class="n">trainX</span><span class="p">,</span><span class="n">trainY</span> <span class="o">=</span><span class="n">convertToMatrix</span><span class="p">(</span><span class="n">train</span><span class="p">,</span><span class="n">step</span><span class="p">)</span>
|
||
<span class="n">testX</span><span class="p">,</span><span class="n">testY</span> <span class="o">=</span><span class="n">convertToMatrix</span><span class="p">(</span><span class="n">test</span><span class="p">,</span><span class="n">step</span><span class="p">)</span>
|
||
<span class="n">trainX</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">trainX</span><span class="p">,</span> <span class="p">(</span><span class="n">trainX</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">1</span><span class="p">,</span> <span class="n">trainX</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]))</span>
|
||
<span class="n">testX</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">testX</span><span class="p">,</span> <span class="p">(</span><span class="n">testX</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">1</span><span class="p">,</span> <span class="n">testX</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]))</span>
|
||
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">Sequential</span><span class="p">()</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">SimpleRNN</span><span class="p">(</span><span class="n">units</span><span class="o">=</span><span class="mi">32</span><span class="p">,</span> <span class="n">input_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="n">step</span><span class="p">),</span> <span class="n">activation</span><span class="o">=</span><span class="s2">"relu"</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Dense</span><span class="p">(</span><span class="mi">8</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s2">"relu"</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Dense</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="s1">'mean_squared_error'</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s1">'rmsprop'</span><span class="p">)</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
|
||
|
||
<span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">trainX</span><span class="p">,</span><span class="n">trainY</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="mi">16</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">trainPredict</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">trainX</span><span class="p">)</span>
|
||
<span class="n">testPredict</span><span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">testX</span><span class="p">)</span>
|
||
<span class="n">predicted</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="n">trainPredict</span><span class="p">,</span><span class="n">testPredict</span><span class="p">),</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
|
||
<span class="n">trainScore</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span><span class="n">trainX</span><span class="p">,</span> <span class="n">trainY</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">trainScore</span><span class="p">)</span>
|
||
|
||
<span class="n">index</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">index</span><span class="o">.</span><span class="n">values</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">index</span><span class="p">,</span><span class="n">df</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">index</span><span class="p">,</span><span class="n">predicted</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">axvline</span><span class="p">(</span><span class="n">df</span><span class="o">.</span><span class="n">index</span><span class="p">[</span><span class="n">Tp</span><span class="p">],</span> <span class="n">c</span><span class="o">=</span><span class="s2">"r"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
|
||
<span class="ne">ModuleNotFoundError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||
<span class="nn">Input In [1],</span> in <span class="ni"><cell line: 7></span><span class="nt">()</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">5</span> <span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">6</span> <span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="ne">----> </span><span class="mi">7</span> <span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">datasets</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">models</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="kn">from</span> <span class="nn">tensorflow.keras.layers</span> <span class="kn">import</span> <span class="n">Input</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">51</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">49</span> <span class="kn">from</span> <span class="nn">._api.v2</span> <span class="kn">import</span> <span class="n">autograph</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">50</span> <span class="kn">from</span> <span class="nn">._api.v2</span> <span class="kn">import</span> <span class="n">bitwise</span>
|
||
<span class="ne">---> </span><span class="mi">51</span> <span class="kn">from</span> <span class="nn">._api.v2</span> <span class="kn">import</span> <span class="n">compat</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">52</span> <span class="kn">from</span> <span class="nn">._api.v2</span> <span class="kn">import</span> <span class="n">config</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">53</span> <span class="kn">from</span> <span class="nn">._api.v2</span> <span class="kn">import</span> <span class="n">data</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">_api</span><span class="o">/</span><span class="n">v2</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">37</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="sd">"""Compatibility functions.</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span><span class="sd"> </span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">5</span><span class="sd"> The `tf.compat` module contains two sets of compatibility functions.</span>
|
||
<span class="sd"> (...)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">32</span><span class="sd"> </span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">33</span><span class="sd"> """</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">35</span> <span class="kn">import</span> <span class="nn">sys</span> <span class="k">as</span> <span class="nn">_sys</span>
|
||
<span class="ne">---> </span><span class="mi">37</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">v1</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">38</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">v2</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">39</span> <span class="kn">from</span> <span class="nn">tensorflow.python.compat.compat</span> <span class="kn">import</span> <span class="n">forward_compatibility_horizon</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">_api</span><span class="o">/</span><span class="n">v2</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="n">v1</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">30</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">28</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">autograph</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">29</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">bitwise</span>
|
||
<span class="ne">---> </span><span class="mi">30</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">compat</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">31</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">config</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">32</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">data</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">_api</span><span class="o">/</span><span class="n">v2</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="n">v1</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">37</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="sd">"""Compatibility functions.</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span><span class="sd"> </span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">5</span><span class="sd"> The `tf.compat` module contains two sets of compatibility functions.</span>
|
||
<span class="sd"> (...)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">32</span><span class="sd"> </span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">33</span><span class="sd"> """</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">35</span> <span class="kn">import</span> <span class="nn">sys</span> <span class="k">as</span> <span class="nn">_sys</span>
|
||
<span class="ne">---> </span><span class="mi">37</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">v1</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">38</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">v2</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">39</span> <span class="kn">from</span> <span class="nn">tensorflow.python.compat.compat</span> <span class="kn">import</span> <span class="n">forward_compatibility_horizon</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">_api</span><span class="o">/</span><span class="n">v2</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="n">v1</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="n">v1</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">47</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">45</span> <span class="kn">from</span> <span class="nn">tensorflow._api.v2.compat.v1</span> <span class="kn">import</span> <span class="n">layers</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">46</span> <span class="kn">from</span> <span class="nn">tensorflow._api.v2.compat.v1</span> <span class="kn">import</span> <span class="n">linalg</span>
|
||
<span class="ne">---> </span><span class="mi">47</span> <span class="kn">from</span> <span class="nn">tensorflow._api.v2.compat.v1</span> <span class="kn">import</span> <span class="n">lite</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">48</span> <span class="kn">from</span> <span class="nn">tensorflow._api.v2.compat.v1</span> <span class="kn">import</span> <span class="n">logging</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">49</span> <span class="kn">from</span> <span class="nn">tensorflow._api.v2.compat.v1</span> <span class="kn">import</span> <span class="n">lookup</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">_api</span><span class="o">/</span><span class="n">v2</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="n">v1</span><span class="o">/</span><span class="n">lite</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">9</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">6</span> <span class="kn">import</span> <span class="nn">sys</span> <span class="k">as</span> <span class="nn">_sys</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">8</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">constants</span>
|
||
<span class="ne">----> </span><span class="mi">9</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">experimental</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">10</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python.lite</span> <span class="kn">import</span> <span class="n">Interpreter</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python.lite</span> <span class="kn">import</span> <span class="n">OpHint</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">_api</span><span class="o">/</span><span class="n">v2</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="n">v1</span><span class="o">/</span><span class="n">lite</span><span class="o">/</span><span class="n">experimental</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">8</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="sd">"""Public API for tf.lite.experimental namespace.</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span><span class="sd"> """</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">6</span> <span class="kn">import</span> <span class="nn">sys</span> <span class="k">as</span> <span class="nn">_sys</span>
|
||
<span class="ne">----> </span><span class="mi">8</span> <span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">authoring</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python.analyzer</span> <span class="kn">import</span> <span class="n">ModelAnalyzer</span> <span class="k">as</span> <span class="n">Analyzer</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">10</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python.lite</span> <span class="kn">import</span> <span class="n">OpResolverType</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">_api</span><span class="o">/</span><span class="n">v2</span><span class="o">/</span><span class="n">compat</span><span class="o">/</span><span class="n">v1</span><span class="o">/</span><span class="n">lite</span><span class="o">/</span><span class="n">experimental</span><span class="o">/</span><span class="n">authoring</span><span class="o">/</span><span class="fm">__init__</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">8</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">3</span> <span class="sd">"""Public API for tf.lite.experimental.authoring namespace.</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span><span class="sd"> """</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">6</span> <span class="kn">import</span> <span class="nn">sys</span> <span class="k">as</span> <span class="nn">_sys</span>
|
||
<span class="ne">----> </span><span class="mi">8</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python.authoring.authoring</span> <span class="kn">import</span> <span class="n">compatible</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">lite</span><span class="o">/</span><span class="n">python</span><span class="o">/</span><span class="n">authoring</span><span class="o">/</span><span class="n">authoring</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">43</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">39</span> <span class="kn">import</span> <span class="nn">functools</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">42</span> <span class="c1"># pylint: disable=g-import-not-at-top</span>
|
||
<span class="ne">---> </span><span class="mi">43</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python</span> <span class="kn">import</span> <span class="n">convert</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">44</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python</span> <span class="kn">import</span> <span class="n">lite</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">45</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python.metrics</span> <span class="kn">import</span> <span class="n">converter_error_data_pb2</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">lite</span><span class="o">/</span><span class="n">python</span><span class="o">/</span><span class="n">convert</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">29</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">26</span> <span class="kn">import</span> <span class="nn">six</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">28</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python</span> <span class="kn">import</span> <span class="n">lite_constants</span>
|
||
<span class="ne">---> </span><span class="mi">29</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python</span> <span class="kn">import</span> <span class="n">util</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">30</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python</span> <span class="kn">import</span> <span class="n">wrap_toco</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">31</span> <span class="kn">from</span> <span class="nn">tensorflow.lite.python.convert_phase</span> <span class="kn">import</span> <span class="n">Component</span>
|
||
|
||
<span class="n">File</span> <span class="o">~/</span><span class="n">miniforge3</span><span class="o">/</span><span class="n">envs</span><span class="o">/</span><span class="n">myenv</span><span class="o">/</span><span class="n">lib</span><span class="o">/</span><span class="n">python3</span><span class="mf">.9</span><span class="o">/</span><span class="n">site</span><span class="o">-</span><span class="n">packages</span><span class="o">/</span><span class="n">tensorflow</span><span class="o">/</span><span class="n">lite</span><span class="o">/</span><span class="n">python</span><span class="o">/</span><span class="n">util</span><span class="o">.</span><span class="n">py</span><span class="p">:</span><span class="mi">26</span><span class="p">,</span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">23</span> <span class="kn">import</span> <span class="nn">six</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">24</span> <span class="kn">from</span> <span class="nn">six.moves</span> <span class="kn">import</span> <span class="nb">range</span>
|
||
<span class="ne">---> </span><span class="mi">26</span> <span class="kn">import</span> <span class="nn">flatbuffers</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">27</span> <span class="kn">from</span> <span class="nn">tensorflow.core.protobuf</span> <span class="kn">import</span> <span class="n">config_pb2</span> <span class="k">as</span> <span class="n">_config_pb2</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">28</span> <span class="kn">from</span> <span class="nn">tensorflow.core.protobuf</span> <span class="kn">import</span> <span class="n">graph_debug_info_pb2</span>
|
||
|
||
<span class="ne">ModuleNotFoundError</span>: No module named 'flatbuffers'
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="an-extrapolation-example">
|
||
<h2><span class="section-number">17.2. </span>An extrapolation example<a class="headerlink" href="#an-extrapolation-example" title="Permalink to this headline">¶</a></h2>
|
||
<p>The following code provides an example of how recurrent neural
|
||
networks can be used to extrapolate to unknown values of physics data
|
||
sets. Specifically, the data sets used in this program come from
|
||
a quantum mechanical many-body calculation of energies as functions of the number of particles.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># For matrices and calculations</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="c1"># For machine learning (backend for keras)</span>
|
||
<span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
|
||
<span class="c1"># User-friendly machine learning library</span>
|
||
<span class="c1"># Front end for TensorFlow</span>
|
||
<span class="kn">import</span> <span class="nn">tensorflow.keras</span>
|
||
<span class="c1"># Different methods from Keras needed to create an RNN</span>
|
||
<span class="c1"># This is not necessary but it shortened function calls </span>
|
||
<span class="c1"># that need to be used in the code.</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">datasets</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">models</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.layers</span> <span class="kn">import</span> <span class="n">Input</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">regularizers</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.models</span> <span class="kn">import</span> <span class="n">Model</span><span class="p">,</span> <span class="n">Sequential</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.layers</span> <span class="kn">import</span> <span class="n">Dense</span><span class="p">,</span> <span class="n">SimpleRNN</span><span class="p">,</span> <span class="n">LSTM</span><span class="p">,</span> <span class="n">GRU</span>
|
||
<span class="c1"># For timing the code</span>
|
||
<span class="kn">from</span> <span class="nn">timeit</span> <span class="kn">import</span> <span class="n">default_timer</span> <span class="k">as</span> <span class="n">timer</span>
|
||
<span class="c1"># For plotting</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
|
||
<span class="c1"># The data set</span>
|
||
<span class="n">datatype</span><span class="o">=</span><span class="s1">'VaryDimension'</span>
|
||
<span class="n">X_tot</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">42</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">y_tot</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="o">-</span><span class="mf">0.03077640549</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.08336233266</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.1446729567</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.2116753732</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.2830637392</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.3581341341</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.436462435</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.5177783846</span><span class="p">,</span>
|
||
<span class="o">-</span><span class="mf">0.6019067271</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.6887363571</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.7782028952</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.8702784034</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.9649652536</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.062292565</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.16231451</span><span class="p">,</span>
|
||
<span class="o">-</span><span class="mf">1.265109911</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.370782966</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.479465113</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.591317992</span><span class="p">,</span> <span class="o">-</span><span class="mf">1.70653767</span><span class="p">])</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The way the recurrent neural networks are trained in this program
|
||
differs from how machine learning algorithms are usually trained.
|
||
Typically a machine learning algorithm is trained by learning the
|
||
relationship between the x data and the y data. In this program, the
|
||
recurrent neural network will be trained to recognize the relationship
|
||
in a sequence of y values. This is type of data formatting is
|
||
typically used time series forcasting, but it can also be used in any
|
||
extrapolation (time series forecasting is just a specific type of
|
||
extrapolation along the time axis). This method of data formatting
|
||
does not use the x data and assumes that the y data are evenly spaced.</p>
|
||
<p>For a standard machine learning algorithm, the training data has the
|
||
form of (x,y) so the machine learning algorithm learns to assiciate a
|
||
y value with a given x value. This is useful when the test data has x
|
||
values within the same range as the training data. However, for this
|
||
application, the x values of the test data are outside of the x values
|
||
of the training data and the traditional method of training a machine
|
||
learning algorithm does not work as well. For this reason, the
|
||
recurrent neural network is trained on sequences of y values of the
|
||
form ((y1, y2), y3), so that the network is concerned with learning
|
||
the pattern of the y data and not the relation between the x and y
|
||
data. As long as the pattern of y data outside of the training region
|
||
stays relatively stable compared to what was inside the training
|
||
region, this method of training can produce accurate extrapolations to
|
||
y values far removed from the training data set.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># FORMAT_DATA</span>
|
||
<span class="k">def</span> <span class="nf">format_data</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">length_of_sequence</span> <span class="o">=</span> <span class="mi">2</span><span class="p">):</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> Inputs:</span>
|
||
<span class="sd"> data(a numpy array): the data that will be the inputs to the recurrent neural</span>
|
||
<span class="sd"> network</span>
|
||
<span class="sd"> length_of_sequence (an int): the number of elements in one iteration of the</span>
|
||
<span class="sd"> sequence patter. For a function approximator use length_of_sequence = 2.</span>
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> rnn_input (a 3D numpy array): the input data for the recurrent neural network. Its</span>
|
||
<span class="sd"> dimensions are length of data - length of sequence, length of sequence, </span>
|
||
<span class="sd"> dimnsion of data</span>
|
||
<span class="sd"> rnn_output (a numpy array): the training data for the neural network</span>
|
||
<span class="sd"> Formats data to be used in a recurrent neural network.</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="n">X</span><span class="p">,</span> <span class="n">Y</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[]</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span><span class="o">-</span><span class="n">length_of_sequence</span><span class="p">):</span>
|
||
<span class="c1"># Get the next length_of_sequence elements</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="n">i</span><span class="o">+</span><span class="n">length_of_sequence</span><span class="p">]</span>
|
||
<span class="c1"># Get the element that immediately follows that</span>
|
||
<span class="n">b</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="n">i</span><span class="o">+</span><span class="n">length_of_sequence</span><span class="p">]</span>
|
||
<span class="c1"># Reshape so that each data point is contained in its own array</span>
|
||
<span class="n">a</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">reshape</span> <span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">a</span><span class="p">),</span> <span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">X</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">a</span><span class="p">)</span>
|
||
<span class="n">Y</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">b</span><span class="p">)</span>
|
||
<span class="n">rnn_input</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">rnn_output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">Y</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">rnn_input</span><span class="p">,</span> <span class="n">rnn_output</span>
|
||
|
||
|
||
<span class="c1"># ## Defining the Recurrent Neural Network Using Keras</span>
|
||
<span class="c1"># </span>
|
||
<span class="c1"># The following method defines a simple recurrent neural network in keras consisting of one input layer, one hidden layer, and one output layer.</span>
|
||
|
||
<span class="k">def</span> <span class="nf">rnn</span><span class="p">(</span><span class="n">length_of_sequences</span><span class="p">,</span> <span class="n">batch_size</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">stateful</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> Inputs:</span>
|
||
<span class="sd"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||
<span class="sd"> when the data is formatted</span>
|
||
<span class="sd"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||
<span class="sd"> method</span>
|
||
<span class="sd"> Builds and compiles a recurrent neural network with one hidden layer and returns the model.</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Number of neurons in the input and output layers</span>
|
||
<span class="n">in_out_neurons</span> <span class="o">=</span> <span class="mi">1</span>
|
||
<span class="c1"># Number of neurons in the hidden layer</span>
|
||
<span class="n">hidden_neurons</span> <span class="o">=</span> <span class="mi">200</span>
|
||
<span class="c1"># Define the input layer</span>
|
||
<span class="n">inp</span> <span class="o">=</span> <span class="n">Input</span><span class="p">(</span><span class="n">batch_shape</span><span class="o">=</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span>
|
||
<span class="n">length_of_sequences</span><span class="p">,</span>
|
||
<span class="n">in_out_neurons</span><span class="p">))</span>
|
||
<span class="c1"># Define the hidden layer as a simple RNN layer with a set number of neurons and add it to </span>
|
||
<span class="c1"># the network immediately after the input layer</span>
|
||
<span class="n">rnn</span> <span class="o">=</span> <span class="n">SimpleRNN</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="p">,</span>
|
||
<span class="n">return_sequences</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||
<span class="n">stateful</span> <span class="o">=</span> <span class="n">stateful</span><span class="p">,</span>
|
||
<span class="n">name</span><span class="o">=</span><span class="s2">"RNN"</span><span class="p">)(</span><span class="n">inp</span><span class="p">)</span>
|
||
<span class="c1"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||
<span class="c1">#and add it to the network immediately after the hidden layer.</span>
|
||
<span class="n">dens</span> <span class="o">=</span> <span class="n">Dense</span><span class="p">(</span><span class="n">in_out_neurons</span><span class="p">,</span><span class="n">name</span><span class="o">=</span><span class="s2">"dense"</span><span class="p">)(</span><span class="n">rnn</span><span class="p">)</span>
|
||
<span class="c1"># Create the machine learning model starting with the input layer and ending with the </span>
|
||
<span class="c1"># output layer</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">Model</span><span class="p">(</span><span class="n">inputs</span><span class="o">=</span><span class="p">[</span><span class="n">inp</span><span class="p">],</span><span class="n">outputs</span><span class="o">=</span><span class="p">[</span><span class="n">dens</span><span class="p">])</span>
|
||
<span class="c1"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||
<span class="c1"># function and an Adams optimizer.</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="s2">"mean_squared_error"</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s2">"adam"</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">model</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="predicting-new-points-with-a-trained-recurrent-neural-network">
|
||
<h2><span class="section-number">17.3. </span>Predicting New Points With A Trained Recurrent Neural Network<a class="headerlink" href="#predicting-new-points-with-a-trained-recurrent-neural-network" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">test_rnn</span> <span class="p">(</span><span class="n">x1</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">plot_min</span><span class="p">,</span> <span class="n">plot_max</span><span class="p">):</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> Inputs:</span>
|
||
<span class="sd"> x1 (a list or numpy array): The complete x component of the data set</span>
|
||
<span class="sd"> y_test (a list or numpy array): The complete y component of the data set</span>
|
||
<span class="sd"> plot_min (an int or float): the smallest x value used in the training data</span>
|
||
<span class="sd"> plot_max (an int or float): the largest x valye used in the training data</span>
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> None.</span>
|
||
<span class="sd"> Uses a trained recurrent neural network model to predict future points in the </span>
|
||
<span class="sd"> series. Computes the MSE of the predicted data set from the true data set, saves</span>
|
||
<span class="sd"> the predicted data set to a csv file, and plots the predicted and true data sets w</span>
|
||
<span class="sd"> while also displaying the data range used for training.</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Add the training data as the first dim points in the predicted data array as these</span>
|
||
<span class="c1"># are known values.</span>
|
||
<span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_test</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
|
||
<span class="c1"># Generate the first input to the trained recurrent neural network using the last two </span>
|
||
<span class="c1"># points of the training data. Based on how the network was trained this means that it</span>
|
||
<span class="c1"># will predict the first point in the data set after the training data. All of the </span>
|
||
<span class="c1"># brackets are necessary for Tensorflow.</span>
|
||
<span class="n">next_input</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[[</span><span class="n">y_test</span><span class="p">[</span><span class="n">dim</span><span class="o">-</span><span class="mi">2</span><span class="p">]],</span> <span class="p">[</span><span class="n">y_test</span><span class="p">[</span><span class="n">dim</span><span class="o">-</span><span class="mi">1</span><span class="p">]]]])</span>
|
||
<span class="c1"># Save the very last point in the training data set. This will be used later.</span>
|
||
<span class="n">last</span> <span class="o">=</span> <span class="p">[</span><span class="n">y_test</span><span class="p">[</span><span class="n">dim</span><span class="o">-</span><span class="mi">1</span><span class="p">]]</span>
|
||
|
||
<span class="c1"># Iterate until the complete data set is created.</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span> <span class="p">(</span><span class="n">dim</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">y_test</span><span class="p">)):</span>
|
||
<span class="c1"># Predict the next point in the data set using the previous two points.</span>
|
||
<span class="nb">next</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">next_input</span><span class="p">)</span>
|
||
<span class="c1"># Append just the number of the predicted data set</span>
|
||
<span class="n">y_pred</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">next</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span>
|
||
<span class="c1"># Create the input that will be used to predict the next data point in the data set.</span>
|
||
<span class="n">next_input</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="n">last</span><span class="p">,</span> <span class="nb">next</span><span class="p">[</span><span class="mi">0</span><span class="p">]]],</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
|
||
<span class="n">last</span> <span class="o">=</span> <span class="nb">next</span>
|
||
|
||
<span class="c1"># Print the mean squared error between the known data set and the predicted data set.</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'MSE: '</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">subtract</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">))</span><span class="o">.</span><span class="n">mean</span><span class="p">())</span>
|
||
<span class="c1"># Save the predicted data set as a csv file for later use</span>
|
||
<span class="n">name</span> <span class="o">=</span> <span class="n">datatype</span> <span class="o">+</span> <span class="s1">'Predicted'</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="n">dim</span><span class="p">)</span><span class="o">+</span><span class="s1">'.csv'</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">savetxt</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">','</span><span class="p">)</span>
|
||
<span class="c1"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||
<span class="c1"># for the training data.</span>
|
||
<span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">()</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x1</span><span class="p">,</span> <span class="n">y_test</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"true"</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x1</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="s1">'g-.'</span><span class="p">,</span><span class="n">label</span><span class="o">=</span><span class="s2">"predicted"</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="c1"># Created a red region to represent the points used in the training data.</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">axvspan</span><span class="p">(</span><span class="n">plot_min</span><span class="p">,</span> <span class="n">plot_max</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s1">'red'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Check to make sure the data set is complete</span>
|
||
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">X_tot</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">y_tot</span><span class="p">)</span>
|
||
|
||
<span class="c1"># This is the number of points that will be used in as the training data</span>
|
||
<span class="n">dim</span><span class="o">=</span><span class="mi">12</span>
|
||
|
||
<span class="c1"># Separate the training data from the whole data set</span>
|
||
<span class="n">X_train</span> <span class="o">=</span> <span class="n">X_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
<span class="n">y_train</span> <span class="o">=</span> <span class="n">y_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
|
||
|
||
<span class="c1"># Generate the training data for the RNN, using a sequence of 2</span>
|
||
<span class="n">rnn_input</span><span class="p">,</span> <span class="n">rnn_training</span> <span class="o">=</span> <span class="n">format_data</span><span class="p">(</span><span class="n">y_train</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||
<span class="c1"># machine learning model</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">(</span><span class="n">length_of_sequences</span> <span class="o">=</span> <span class="n">rnn_input</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Start the timer. Want to time training+testing</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="c1"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||
<span class="c1"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||
<span class="n">hist</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">rnn_input</span><span class="p">,</span> <span class="n">rnn_training</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="mi">150</span><span class="p">,</span>
|
||
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span><span class="n">validation_split</span><span class="o">=</span><span class="mf">0.05</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">label</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"loss"</span><span class="p">,</span><span class="s2">"val_loss"</span><span class="p">]:</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="n">label</span><span class="p">],</span><span class="n">label</span><span class="o">=</span><span class="n">label</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">"loss"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">"epoch"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">"The final validation loss: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="s2">"val_loss"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Use the trained neural network to predict more points of the data set</span>
|
||
<span class="n">test_rnn</span><span class="p">(</span><span class="n">X_tot</span><span class="p">,</span> <span class="n">y_tot</span><span class="p">,</span> <span class="n">X_tot</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">X_tot</span><span class="p">[</span><span class="n">dim</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="c1"># Stop the timer and calculate the total time needed.</span>
|
||
<span class="n">end</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Time: '</span><span class="p">,</span> <span class="n">end</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Changing the size of the recurrent neural network and its parameters
|
||
can drastically change the results you get from the model. The below
|
||
code takes the simple recurrent neural network from above and adds a
|
||
second hidden layer, changes the number of neurons in the hidden
|
||
layer, and explicitly declares the activation function of the hidden
|
||
layers to be a sigmoid function. The loss function and optimizer can
|
||
also be changed but are kept the same as the above network. These
|
||
parameters can be tuned to provide the optimal result from the
|
||
network. For some ideas on how to improve the performance of a
|
||
<a class="reference external" href="https://danijar.com/tips-for-training-recurrent-neural-networks">recurrent neural network</a>.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">rnn_2layers</span><span class="p">(</span><span class="n">length_of_sequences</span><span class="p">,</span> <span class="n">batch_size</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">stateful</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> Inputs:</span>
|
||
<span class="sd"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||
<span class="sd"> when the data is formatted</span>
|
||
<span class="sd"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||
<span class="sd"> method</span>
|
||
<span class="sd"> Builds and compiles a recurrent neural network with two hidden layers and returns the model.</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Number of neurons in the input and output layers</span>
|
||
<span class="n">in_out_neurons</span> <span class="o">=</span> <span class="mi">1</span>
|
||
<span class="c1"># Number of neurons in the hidden layer, increased from the first network</span>
|
||
<span class="n">hidden_neurons</span> <span class="o">=</span> <span class="mi">500</span>
|
||
<span class="c1"># Define the input layer</span>
|
||
<span class="n">inp</span> <span class="o">=</span> <span class="n">Input</span><span class="p">(</span><span class="n">batch_shape</span><span class="o">=</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span>
|
||
<span class="n">length_of_sequences</span><span class="p">,</span>
|
||
<span class="n">in_out_neurons</span><span class="p">))</span>
|
||
<span class="c1"># Create two hidden layers instead of one hidden layer. Explicitly set the activation</span>
|
||
<span class="c1"># function to be the sigmoid function (the default value is hyperbolic tangent)</span>
|
||
<span class="n">rnn1</span> <span class="o">=</span> <span class="n">SimpleRNN</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="p">,</span>
|
||
<span class="n">return_sequences</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="c1"># This needs to be True if another hidden layer is to follow</span>
|
||
<span class="n">stateful</span> <span class="o">=</span> <span class="n">stateful</span><span class="p">,</span> <span class="n">activation</span> <span class="o">=</span> <span class="s1">'sigmoid'</span><span class="p">,</span>
|
||
<span class="n">name</span><span class="o">=</span><span class="s2">"RNN1"</span><span class="p">)(</span><span class="n">inp</span><span class="p">)</span>
|
||
<span class="n">rnn2</span> <span class="o">=</span> <span class="n">SimpleRNN</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="p">,</span>
|
||
<span class="n">return_sequences</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">activation</span> <span class="o">=</span> <span class="s1">'sigmoid'</span><span class="p">,</span>
|
||
<span class="n">stateful</span> <span class="o">=</span> <span class="n">stateful</span><span class="p">,</span>
|
||
<span class="n">name</span><span class="o">=</span><span class="s2">"RNN2"</span><span class="p">)(</span><span class="n">rnn1</span><span class="p">)</span>
|
||
<span class="c1"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||
<span class="c1">#and add it to the network immediately after the hidden layer.</span>
|
||
<span class="n">dens</span> <span class="o">=</span> <span class="n">Dense</span><span class="p">(</span><span class="n">in_out_neurons</span><span class="p">,</span><span class="n">name</span><span class="o">=</span><span class="s2">"dense"</span><span class="p">)(</span><span class="n">rnn2</span><span class="p">)</span>
|
||
<span class="c1"># Create the machine learning model starting with the input layer and ending with the </span>
|
||
<span class="c1"># output layer</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">Model</span><span class="p">(</span><span class="n">inputs</span><span class="o">=</span><span class="p">[</span><span class="n">inp</span><span class="p">],</span><span class="n">outputs</span><span class="o">=</span><span class="p">[</span><span class="n">dens</span><span class="p">])</span>
|
||
<span class="c1"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||
<span class="c1"># function and an Adams optimizer.</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="s2">"mean_squared_error"</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s2">"adam"</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">model</span>
|
||
|
||
<span class="c1"># Check to make sure the data set is complete</span>
|
||
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">X_tot</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">y_tot</span><span class="p">)</span>
|
||
|
||
<span class="c1"># This is the number of points that will be used in as the training data</span>
|
||
<span class="n">dim</span><span class="o">=</span><span class="mi">12</span>
|
||
|
||
<span class="c1"># Separate the training data from the whole data set</span>
|
||
<span class="n">X_train</span> <span class="o">=</span> <span class="n">X_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
<span class="n">y_train</span> <span class="o">=</span> <span class="n">y_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
|
||
|
||
<span class="c1"># Generate the training data for the RNN, using a sequence of 2</span>
|
||
<span class="n">rnn_input</span><span class="p">,</span> <span class="n">rnn_training</span> <span class="o">=</span> <span class="n">format_data</span><span class="p">(</span><span class="n">y_train</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||
<span class="c1"># machine learning model</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">rnn_2layers</span><span class="p">(</span><span class="n">length_of_sequences</span> <span class="o">=</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Start the timer. Want to time training+testing</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="c1"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||
<span class="c1"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||
<span class="n">hist</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">rnn_input</span><span class="p">,</span> <span class="n">rnn_training</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="mi">150</span><span class="p">,</span>
|
||
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span><span class="n">validation_split</span><span class="o">=</span><span class="mf">0.05</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||
<span class="c1"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||
<span class="c1"># being overtrained.</span>
|
||
<span class="k">for</span> <span class="n">label</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"loss"</span><span class="p">,</span><span class="s2">"val_loss"</span><span class="p">]:</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="n">label</span><span class="p">],</span><span class="n">label</span><span class="o">=</span><span class="n">label</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">"loss"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">"epoch"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">"The final validation loss: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="s2">"val_loss"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Use the trained neural network to predict more points of the data set</span>
|
||
<span class="n">test_rnn</span><span class="p">(</span><span class="n">X_tot</span><span class="p">,</span> <span class="n">y_tot</span><span class="p">,</span> <span class="n">X_tot</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">X_tot</span><span class="p">[</span><span class="n">dim</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="c1"># Stop the timer and calculate the total time needed.</span>
|
||
<span class="n">end</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Time: '</span><span class="p">,</span> <span class="n">end</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="other-types-of-recurrent-neural-networks">
|
||
<h2><span class="section-number">17.4. </span>Other Types of Recurrent Neural Networks<a class="headerlink" href="#other-types-of-recurrent-neural-networks" title="Permalink to this headline">¶</a></h2>
|
||
<p>Besides a simple recurrent neural network layer, there are two other
|
||
commonly used types of recurrent neural network layers: Long Short
|
||
Term Memory (LSTM) and Gated Recurrent Unit (GRU). For a short
|
||
introduction to these layers see <a class="reference external" href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b">https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</a>
|
||
and <a class="reference external" href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b">https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</a>.</p>
|
||
<p>The first network created below is similar to the previous network,
|
||
but it replaces the SimpleRNN layers with LSTM layers. The second
|
||
network below has two hidden layers made up of GRUs, which are
|
||
preceeded by two dense (feeddorward) neural network layers. These
|
||
dense layers “preprocess” the data before it reaches the recurrent
|
||
layers. This architecture has been shown to improve the performance
|
||
of recurrent neural networks (see the link above and also
|
||
<a class="reference external" href="https://arxiv.org/pdf/1807.02857.pdf">https://arxiv.org/pdf/1807.02857.pdf</a>.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">lstm_2layers</span><span class="p">(</span><span class="n">length_of_sequences</span><span class="p">,</span> <span class="n">batch_size</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">stateful</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> Inputs:</span>
|
||
<span class="sd"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||
<span class="sd"> when the data is formatted</span>
|
||
<span class="sd"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||
<span class="sd"> method</span>
|
||
<span class="sd"> Builds and compiles a recurrent neural network with two LSTM hidden layers and returns the model.</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Number of neurons on the input/output layer and the number of neurons in the hidden layer</span>
|
||
<span class="n">in_out_neurons</span> <span class="o">=</span> <span class="mi">1</span>
|
||
<span class="n">hidden_neurons</span> <span class="o">=</span> <span class="mi">250</span>
|
||
<span class="c1"># Input Layer</span>
|
||
<span class="n">inp</span> <span class="o">=</span> <span class="n">Input</span><span class="p">(</span><span class="n">batch_shape</span><span class="o">=</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span>
|
||
<span class="n">length_of_sequences</span><span class="p">,</span>
|
||
<span class="n">in_out_neurons</span><span class="p">))</span>
|
||
<span class="c1"># Hidden layers (in this case they are LSTM layers instead if SimpleRNN layers)</span>
|
||
<span class="n">rnn</span><span class="o">=</span> <span class="n">LSTM</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="p">,</span>
|
||
<span class="n">return_sequences</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||
<span class="n">stateful</span> <span class="o">=</span> <span class="n">stateful</span><span class="p">,</span>
|
||
<span class="n">name</span><span class="o">=</span><span class="s2">"RNN"</span><span class="p">,</span> <span class="n">use_bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'tanh'</span><span class="p">)(</span><span class="n">inp</span><span class="p">)</span>
|
||
<span class="n">rnn1</span> <span class="o">=</span> <span class="n">LSTM</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="p">,</span>
|
||
<span class="n">return_sequences</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||
<span class="n">stateful</span> <span class="o">=</span> <span class="n">stateful</span><span class="p">,</span>
|
||
<span class="n">name</span><span class="o">=</span><span class="s2">"RNN1"</span><span class="p">,</span> <span class="n">use_bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'tanh'</span><span class="p">)(</span><span class="n">rnn</span><span class="p">)</span>
|
||
<span class="c1"># Output layer</span>
|
||
<span class="n">dens</span> <span class="o">=</span> <span class="n">Dense</span><span class="p">(</span><span class="n">in_out_neurons</span><span class="p">,</span><span class="n">name</span><span class="o">=</span><span class="s2">"dense"</span><span class="p">)(</span><span class="n">rnn1</span><span class="p">)</span>
|
||
<span class="c1"># Define the midel</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">Model</span><span class="p">(</span><span class="n">inputs</span><span class="o">=</span><span class="p">[</span><span class="n">inp</span><span class="p">],</span><span class="n">outputs</span><span class="o">=</span><span class="p">[</span><span class="n">dens</span><span class="p">])</span>
|
||
<span class="c1"># Compile the model</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="s1">'mean_squared_error'</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s1">'adam'</span><span class="p">)</span>
|
||
<span class="c1"># Return the model</span>
|
||
<span class="k">return</span> <span class="n">model</span>
|
||
|
||
<span class="k">def</span> <span class="nf">dnn2_gru2</span><span class="p">(</span><span class="n">length_of_sequences</span><span class="p">,</span> <span class="n">batch_size</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">stateful</span> <span class="o">=</span> <span class="kc">False</span><span class="p">):</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> Inputs:</span>
|
||
<span class="sd"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||
<span class="sd"> when the data is formatted</span>
|
||
<span class="sd"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||
<span class="sd"> Returns:</span>
|
||
<span class="sd"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||
<span class="sd"> method</span>
|
||
<span class="sd"> Builds and compiles a recurrent neural network with four hidden layers (two dense followed by</span>
|
||
<span class="sd"> two GRU layers) and returns the model.</span>
|
||
<span class="sd"> """</span>
|
||
<span class="c1"># Number of neurons on the input/output layers and hidden layers</span>
|
||
<span class="n">in_out_neurons</span> <span class="o">=</span> <span class="mi">1</span>
|
||
<span class="n">hidden_neurons</span> <span class="o">=</span> <span class="mi">250</span>
|
||
<span class="c1"># Input layer</span>
|
||
<span class="n">inp</span> <span class="o">=</span> <span class="n">Input</span><span class="p">(</span><span class="n">batch_shape</span><span class="o">=</span><span class="p">(</span><span class="n">batch_size</span><span class="p">,</span>
|
||
<span class="n">length_of_sequences</span><span class="p">,</span>
|
||
<span class="n">in_out_neurons</span><span class="p">))</span>
|
||
<span class="c1"># Hidden Dense (feedforward) layers</span>
|
||
<span class="n">dnn</span> <span class="o">=</span> <span class="n">Dense</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="o">/</span><span class="mi">2</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s1">'dnn'</span><span class="p">)(</span><span class="n">inp</span><span class="p">)</span>
|
||
<span class="n">dnn1</span> <span class="o">=</span> <span class="n">Dense</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="o">/</span><span class="mi">2</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="s1">'dnn1'</span><span class="p">)(</span><span class="n">dnn</span><span class="p">)</span>
|
||
<span class="c1"># Hidden GRU layers</span>
|
||
<span class="n">rnn1</span> <span class="o">=</span> <span class="n">GRU</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="p">,</span>
|
||
<span class="n">return_sequences</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||
<span class="n">stateful</span> <span class="o">=</span> <span class="n">stateful</span><span class="p">,</span>
|
||
<span class="n">name</span><span class="o">=</span><span class="s2">"RNN1"</span><span class="p">,</span> <span class="n">use_bias</span><span class="o">=</span><span class="kc">True</span><span class="p">)(</span><span class="n">dnn1</span><span class="p">)</span>
|
||
<span class="n">rnn</span> <span class="o">=</span> <span class="n">GRU</span><span class="p">(</span><span class="n">hidden_neurons</span><span class="p">,</span>
|
||
<span class="n">return_sequences</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||
<span class="n">stateful</span> <span class="o">=</span> <span class="n">stateful</span><span class="p">,</span>
|
||
<span class="n">name</span><span class="o">=</span><span class="s2">"RNN"</span><span class="p">,</span> <span class="n">use_bias</span><span class="o">=</span><span class="kc">True</span><span class="p">)(</span><span class="n">rnn1</span><span class="p">)</span>
|
||
<span class="c1"># Output layer</span>
|
||
<span class="n">dens</span> <span class="o">=</span> <span class="n">Dense</span><span class="p">(</span><span class="n">in_out_neurons</span><span class="p">,</span><span class="n">name</span><span class="o">=</span><span class="s2">"dense"</span><span class="p">)(</span><span class="n">rnn</span><span class="p">)</span>
|
||
<span class="c1"># Define the model</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">Model</span><span class="p">(</span><span class="n">inputs</span><span class="o">=</span><span class="p">[</span><span class="n">inp</span><span class="p">],</span><span class="n">outputs</span><span class="o">=</span><span class="p">[</span><span class="n">dens</span><span class="p">])</span>
|
||
<span class="c1"># Compile the mdoel</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="s1">'mean_squared_error'</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="s1">'adam'</span><span class="p">)</span>
|
||
<span class="c1"># Return the model</span>
|
||
<span class="k">return</span> <span class="n">model</span>
|
||
|
||
<span class="c1"># Check to make sure the data set is complete</span>
|
||
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">X_tot</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">y_tot</span><span class="p">)</span>
|
||
|
||
<span class="c1"># This is the number of points that will be used in as the training data</span>
|
||
<span class="n">dim</span><span class="o">=</span><span class="mi">12</span>
|
||
|
||
<span class="c1"># Separate the training data from the whole data set</span>
|
||
<span class="n">X_train</span> <span class="o">=</span> <span class="n">X_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
<span class="n">y_train</span> <span class="o">=</span> <span class="n">y_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
|
||
|
||
<span class="c1"># Generate the training data for the RNN, using a sequence of 2</span>
|
||
<span class="n">rnn_input</span><span class="p">,</span> <span class="n">rnn_training</span> <span class="o">=</span> <span class="n">format_data</span><span class="p">(</span><span class="n">y_train</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||
<span class="c1"># machine learning model</span>
|
||
<span class="c1"># Change the method name to reflect which network you want to use</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">dnn2_gru2</span><span class="p">(</span><span class="n">length_of_sequences</span> <span class="o">=</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Start the timer. Want to time training+testing</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="c1"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||
<span class="c1"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||
<span class="n">hist</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">rnn_input</span><span class="p">,</span> <span class="n">rnn_training</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="mi">150</span><span class="p">,</span>
|
||
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span><span class="n">validation_split</span><span class="o">=</span><span class="mf">0.05</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||
<span class="c1"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||
<span class="c1"># being overtrained.</span>
|
||
<span class="k">for</span> <span class="n">label</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"loss"</span><span class="p">,</span><span class="s2">"val_loss"</span><span class="p">]:</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="n">label</span><span class="p">],</span><span class="n">label</span><span class="o">=</span><span class="n">label</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">"loss"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">"epoch"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">"The final validation loss: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="s2">"val_loss"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Use the trained neural network to predict more points of the data set</span>
|
||
<span class="n">test_rnn</span><span class="p">(</span><span class="n">X_tot</span><span class="p">,</span> <span class="n">y_tot</span><span class="p">,</span> <span class="n">X_tot</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">X_tot</span><span class="p">[</span><span class="n">dim</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="c1"># Stop the timer and calculate the total time needed.</span>
|
||
<span class="n">end</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Time: '</span><span class="p">,</span> <span class="n">end</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># ### Training Recurrent Neural Networks in the Standard Way (i.e. learning the relationship between the X and Y data)</span>
|
||
<span class="c1"># </span>
|
||
<span class="c1"># Finally, comparing the performace of a recurrent neural network using the standard data formatting to the performance of the network with time sequence data formatting shows the benefit of this type of data formatting with extrapolation.</span>
|
||
|
||
<span class="c1"># Check to make sure the data set is complete</span>
|
||
<span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">X_tot</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">y_tot</span><span class="p">)</span>
|
||
|
||
<span class="c1"># This is the number of points that will be used in as the training data</span>
|
||
<span class="n">dim</span><span class="o">=</span><span class="mi">12</span>
|
||
|
||
<span class="c1"># Separate the training data from the whole data set</span>
|
||
<span class="n">X_train</span> <span class="o">=</span> <span class="n">X_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
<span class="n">y_train</span> <span class="o">=</span> <span class="n">y_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">]</span>
|
||
|
||
<span class="c1"># Reshape the data for Keras specifications</span>
|
||
<span class="n">X_train</span> <span class="o">=</span> <span class="n">X_train</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="n">dim</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">y_train</span> <span class="o">=</span> <span class="n">y_train</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="n">dim</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>
|
||
|
||
|
||
<span class="c1"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||
<span class="c1"># machine learning model</span>
|
||
<span class="c1"># Set the sequence length to 1 for regular data formatting </span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">rnn</span><span class="p">(</span><span class="n">length_of_sequences</span> <span class="o">=</span> <span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Start the timer. Want to time training+testing</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="c1"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||
<span class="c1"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||
<span class="n">hist</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="mi">150</span><span class="p">,</span>
|
||
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span><span class="n">validation_split</span><span class="o">=</span><span class="mf">0.05</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||
<span class="c1"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||
<span class="c1"># being overtrained.</span>
|
||
<span class="k">for</span> <span class="n">label</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">"loss"</span><span class="p">,</span><span class="s2">"val_loss"</span><span class="p">]:</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="n">label</span><span class="p">],</span><span class="n">label</span><span class="o">=</span><span class="n">label</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s2">"loss"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s2">"epoch"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">"The final validation loss: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hist</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="s2">"val_loss"</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]))</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Use the trained neural network to predict the remaining data points</span>
|
||
<span class="n">X_pred</span> <span class="o">=</span> <span class="n">X_tot</span><span class="p">[</span><span class="n">dim</span><span class="p">:]</span>
|
||
<span class="n">X_pred</span> <span class="o">=</span> <span class="n">X_pred</span><span class="o">.</span><span class="n">reshape</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">X_pred</span><span class="p">),</span> <span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">y_model</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_pred</span><span class="p">)</span>
|
||
<span class="n">y_pred</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="n">y_tot</span><span class="p">[:</span><span class="n">dim</span><span class="p">],</span> <span class="n">y_model</span><span class="o">.</span><span class="n">flatten</span><span class="p">()))</span>
|
||
|
||
<span class="c1"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||
<span class="c1"># for the training data.</span>
|
||
<span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">()</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">X_tot</span><span class="p">,</span> <span class="n">y_tot</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"true"</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">X_tot</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="s1">'g-.'</span><span class="p">,</span><span class="n">label</span><span class="o">=</span><span class="s2">"predicted"</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="c1"># Created a red region to represent the points used in the training data.</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">axvspan</span><span class="p">(</span><span class="n">X_tot</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">X_tot</span><span class="p">[</span><span class="n">dim</span><span class="p">],</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.25</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="s1">'red'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Stop the timer and calculate the total time needed.</span>
|
||
<span class="n">end</span> <span class="o">=</span> <span class="n">timer</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Time: '</span><span class="p">,</span> <span class="n">end</span><span class="o">-</span><span class="n">start</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tex2jax_ignore mathjax_ignore section" id="generative-models">
|
||
<h1><span class="section-number">18. </span>Generative Models<a class="headerlink" href="#generative-models" title="Permalink to this headline">¶</a></h1>
|
||
<p><strong>Generative models</strong> describe a class of statistical models that are a contrast
|
||
to <strong>discriminative models</strong>. Informally we say that generative models can
|
||
generate new data instances while discriminative models discriminate between
|
||
different kinds of data instances. A generative model could generate new photos
|
||
of animals that look like ‘real’ animals while a discriminative model could tell
|
||
a dog from a cat. More formally, given a data set <span class="math notranslate nohighlight">\(x\)</span> and a set of labels /
|
||
targets <span class="math notranslate nohighlight">\(y\)</span>. Generative models capture the joint probability <span class="math notranslate nohighlight">\(p(x, y)\)</span>, or
|
||
just <span class="math notranslate nohighlight">\(p(x)\)</span> if there are no labels, while discriminative models capture the
|
||
conditional probability <span class="math notranslate nohighlight">\(p(y | x)\)</span>. Discriminative models generally try to draw
|
||
boundaries in the data space (often high dimensional), while generative models
|
||
try to model how data is placed throughout the space.</p>
|
||
<p><strong>Note</strong>: this material is thanks to Linus Ekstrøm.</p>
|
||
<div class="section" id="generative-adversarial-networks">
|
||
<h2><span class="section-number">18.1. </span>Generative Adversarial Networks<a class="headerlink" href="#generative-adversarial-networks" title="Permalink to this headline">¶</a></h2>
|
||
<p><strong>Generative Adversarial Networks</strong> are a type of unsupervised machine learning
|
||
algorithm proposed by <a class="reference external" href="https://arxiv.org/pdf/1406.2661.pdf">Goodfellow et. al</a>
|
||
in 2014 (short and good article).</p>
|
||
<p>The simplest formulation of
|
||
the model is based on a game theoretic approach, <em>zero sum game</em>, where we pit
|
||
two neural networks against one another. We define two rival networks, one
|
||
generator <span class="math notranslate nohighlight">\(g\)</span>, and one discriminator <span class="math notranslate nohighlight">\(d\)</span>. The generator directly produces
|
||
samples</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
x = g(z; \theta^{(g)})
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>The discriminator attempts to distinguish between samples drawn from the
|
||
training data and samples drawn from the generator. In other words, it tries to
|
||
tell the difference between the fake data produced by <span class="math notranslate nohighlight">\(g\)</span> and the actual data
|
||
samples we want to do prediction on. The discriminator outputs a probability
|
||
value given by</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto2"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
d(x; \theta^{(d)})
|
||
\label{_auto2} \tag{2}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>indicating the probability that <span class="math notranslate nohighlight">\(x\)</span> is a real training example rather than a
|
||
fake sample the generator has generated. The simplest way to formulate the
|
||
learning process in a generative adversarial network is a zero-sum game, in
|
||
which a function</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto3"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
v(\theta^{(g)}, \theta^{(d)})
|
||
\label{_auto3} \tag{3}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>determines the reward for the discriminator, while the generator gets the
|
||
conjugate reward</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto4"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
-v(\theta^{(g)}, \theta^{(d)})
|
||
\label{_auto4} \tag{4}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>During learning both of the networks maximize their own reward function, so that
|
||
the generator gets better and better at tricking the discriminator, while the
|
||
discriminator gets better and better at telling the difference between the fake
|
||
and real data. The generator and discriminator alternate on which one trains at
|
||
one time (i.e. for one epoch). In other words, we keep the generator constant
|
||
and train the discriminator, then we keep the discriminator constant to train
|
||
the generator and repeat. It is this back and forth dynamic which lets GANs
|
||
tackle otherwise intractable generative problems. As the generator improves with
|
||
training, the discriminator’s performance gets worse because it cannot easily
|
||
tell the difference between real and fake. If the generator ends up succeeding
|
||
perfectly, the the discriminator will do no better than random guessing i.e.
|
||
50%. This progression in the training poses a problem for the convergence
|
||
criteria for GANs. The discriminator feedback gets less meaningful over time,
|
||
if we continue training after this point then the generator is effectively
|
||
training on junk data which can undo the learning up to that point. Therefore,
|
||
we stop training when the discriminator starts outputting <span class="math notranslate nohighlight">\(1/2\)</span> everywhere.</p>
|
||
<p>At convergence we have</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto5"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt}
|
||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||
\label{_auto5} \tag{5}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>The default choice for <span class="math notranslate nohighlight">\(v\)</span> is</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto6"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x)
|
||
+ \mathbb{E}_{x\sim p_\mathrm{model}}
|
||
\log (1 - d(x))
|
||
\label{_auto6} \tag{6}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>The main motivation for the design of GANs is that the learning process requires
|
||
neither approximate inference (variational autoencoders for example) nor
|
||
approximation of a partition function. In the case where</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto7"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||
\label{_auto7} \tag{7}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is
|
||
asymptotically consistent
|
||
( <a class="reference external" href="https://arxiv.org/pdf/1804.09139.pdf">Seth Lloyd on QuGANs</a> ).</p>
|
||
<p>This is in
|
||
general not the case and it is possible to get situations where the training
|
||
process never converges because the generator and discriminator chase one
|
||
another around in the parameter space indefinitely. A much deeper discussion on
|
||
the currently open research problem of GAN convergence is available
|
||
<a class="reference external" href="https://www.deeplearningbook.org/contents/generative_models.html">here</a>. To
|
||
anyone interested in learning more about GANs it is a highly recommended read.
|
||
Direct quote: “In this best-performing formulation, the generator aims to
|
||
increase the log probability that the discriminator makes a mistake, rather than
|
||
aiming to decrease the log probability that the discriminator makes the correct
|
||
prediction.” <a class="reference external" href="https://arxiv.org/abs/1701.00160">Another interesting read</a></p>
|
||
</div>
|
||
<div class="section" id="writing-our-first-generative-adversarial-network">
|
||
<h2><span class="section-number">18.2. </span>Writing Our First Generative Adversarial Network<a class="headerlink" href="#writing-our-first-generative-adversarial-network" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let us now move on to actually implementing a GAN in tensorflow. We will study
|
||
the performance of our GAN on the MNIST dataset. This code is based on and
|
||
adapted from the
|
||
<a class="reference external" href="https://www.tensorflow.org/tutorials/generative/dcgan">google tutorial</a></p>
|
||
<p>First we import our libraries</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">os</span>
|
||
<span class="kn">import</span> <span class="nn">time</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">layers</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.utils</span> <span class="kn">import</span> <span class="n">plot_model</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Next we define our hyperparameters and import our data the usual way</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">BUFFER_SIZE</span> <span class="o">=</span> <span class="mi">60000</span>
|
||
<span class="n">BATCH_SIZE</span> <span class="o">=</span> <span class="mi">256</span>
|
||
<span class="n">EPOCHS</span> <span class="o">=</span> <span class="mi">30</span>
|
||
|
||
<span class="n">data</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">datasets</span><span class="o">.</span><span class="n">mnist</span><span class="o">.</span><span class="n">load_data</span><span class="p">()</span>
|
||
<span class="p">(</span><span class="n">train_images</span><span class="p">,</span> <span class="n">train_labels</span><span class="p">),</span> <span class="p">(</span><span class="n">test_images</span><span class="p">,</span> <span class="n">test_labels</span><span class="p">)</span> <span class="o">=</span> <span class="n">data</span>
|
||
<span class="n">train_images</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="n">train_images</span><span class="p">,</span> <span class="p">(</span><span class="n">train_images</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
|
||
<span class="mi">28</span><span class="p">,</span>
|
||
<span class="mi">28</span><span class="p">,</span>
|
||
<span class="mi">1</span><span class="p">))</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">'float32'</span><span class="p">)</span>
|
||
|
||
<span class="c1"># we normalize between -1 and 1</span>
|
||
<span class="n">train_images</span> <span class="o">=</span> <span class="p">(</span><span class="n">train_images</span> <span class="o">-</span> <span class="mf">127.5</span><span class="p">)</span> <span class="o">/</span> <span class="mf">127.5</span>
|
||
<span class="n">training_dataset</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">Dataset</span><span class="o">.</span><span class="n">from_tensor_slices</span><span class="p">(</span>
|
||
<span class="n">train_images</span><span class="p">)</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">BUFFER_SIZE</span><span class="p">)</span><span class="o">.</span><span class="n">batch</span><span class="p">(</span><span class="n">BATCH_SIZE</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="mnist-and-gans">
|
||
<h3><span class="section-number">18.2.1. </span>MNIST and GANs<a class="headerlink" href="#mnist-and-gans" title="Permalink to this headline">¶</a></h3>
|
||
<p>Let’s have a quick look</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">train_images</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">'Greys'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Now we define our two models. This is where the ‘magic’ happens. There are a
|
||
huge amount of possible formulations for both models. A lot of engineering and
|
||
trial and error can be done here to try to produce better performing models. For
|
||
more advanced GANs this is by far the step where you can ‘make or break’ a
|
||
model.</p>
|
||
<p>We start with the generator. As stated in the introductory text the generator
|
||
<span class="math notranslate nohighlight">\(g\)</span> upsamples from a random sample to the shape of what we want to predict. In
|
||
our case we are trying to predict MNIST images (<span class="math notranslate nohighlight">\(28\times 28\)</span> pixels).</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">generator_model</span><span class="p">():</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to</span>
|
||
<span class="sd"> produce an image from a random seed. We start with a Dense layer taking this</span>
|
||
<span class="sd"> random sample as an input and subsequently upsample through multiple</span>
|
||
<span class="sd"> convolutional layers.</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="c1"># we define our model</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
|
||
|
||
|
||
<span class="c1"># adding our input layer. Dense means that every neuron is connected and</span>
|
||
<span class="c1"># the input shape is the shape of our random noise. The units need to match</span>
|
||
<span class="c1"># in some sense the upsampling strides to reach our desired output shape.</span>
|
||
<span class="c1"># we are using 100 random numbers as our seed</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">units</span><span class="o">=</span><span class="mi">7</span><span class="o">*</span><span class="mi">7</span><span class="o">*</span><span class="n">BATCH_SIZE</span><span class="p">,</span>
|
||
<span class="n">use_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||
<span class="n">input_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="p">)))</span>
|
||
<span class="c1"># we normalize the output form the Dense layer</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">BatchNormalization</span><span class="p">())</span>
|
||
<span class="c1"># and add an activation function to our 'layer'. LeakyReLU avoids vanishing</span>
|
||
<span class="c1"># gradient problem</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">LeakyReLU</span><span class="p">())</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Reshape</span><span class="p">((</span><span class="mi">7</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="n">BATCH_SIZE</span><span class="p">)))</span>
|
||
<span class="k">assert</span> <span class="n">model</span><span class="o">.</span><span class="n">output_shape</span> <span class="o">==</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="n">BATCH_SIZE</span><span class="p">)</span>
|
||
<span class="c1"># even though we just added four keras layers we think of everything above</span>
|
||
<span class="c1"># as 'one' layer</span>
|
||
|
||
<span class="c1"># next we add our upscaling convolutional layers</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2DTranspose</span><span class="p">(</span><span class="n">filters</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||
<span class="n">kernel_size</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span>
|
||
<span class="n">strides</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
|
||
<span class="n">padding</span><span class="o">=</span><span class="s1">'same'</span><span class="p">,</span>
|
||
<span class="n">use_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">BatchNormalization</span><span class="p">())</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">LeakyReLU</span><span class="p">())</span>
|
||
<span class="k">assert</span> <span class="n">model</span><span class="o">.</span><span class="n">output_shape</span> <span class="o">==</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">128</span><span class="p">)</span>
|
||
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2DTranspose</span><span class="p">(</span><span class="n">filters</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span>
|
||
<span class="n">kernel_size</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span>
|
||
<span class="n">strides</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span>
|
||
<span class="n">padding</span><span class="o">=</span><span class="s1">'same'</span><span class="p">,</span>
|
||
<span class="n">use_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">BatchNormalization</span><span class="p">())</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">LeakyReLU</span><span class="p">())</span>
|
||
<span class="k">assert</span> <span class="n">model</span><span class="o">.</span><span class="n">output_shape</span> <span class="o">==</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">14</span><span class="p">,</span> <span class="mi">64</span><span class="p">)</span>
|
||
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2DTranspose</span><span class="p">(</span><span class="n">filters</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||
<span class="n">kernel_size</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span>
|
||
<span class="n">strides</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span>
|
||
<span class="n">padding</span><span class="o">=</span><span class="s1">'same'</span><span class="p">,</span>
|
||
<span class="n">use_bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||
<span class="n">activation</span><span class="o">=</span><span class="s1">'tanh'</span><span class="p">))</span>
|
||
<span class="k">assert</span> <span class="n">model</span><span class="o">.</span><span class="n">output_shape</span> <span class="o">==</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="mi">28</span><span class="p">,</span> <span class="mi">28</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">model</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>And there we have our ‘simple’ generator model. Now we move on to defining our
|
||
discriminator model <span class="math notranslate nohighlight">\(d\)</span>, which is a convolutional neural network based image
|
||
classifier.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">discriminator_model</span><span class="p">():</span>
|
||
<span class="sd">"""</span>
|
||
<span class="sd"> The discriminator is a convolutional neural network based image classifier</span>
|
||
<span class="sd"> """</span>
|
||
|
||
<span class="c1"># we define our model</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2D</span><span class="p">(</span><span class="n">filters</span><span class="o">=</span><span class="mi">64</span><span class="p">,</span>
|
||
<span class="n">kernel_size</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span>
|
||
<span class="n">strides</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span>
|
||
<span class="n">padding</span><span class="o">=</span><span class="s1">'same'</span><span class="p">,</span>
|
||
<span class="n">input_shape</span><span class="o">=</span><span class="p">[</span><span class="mi">28</span><span class="p">,</span> <span class="mi">28</span><span class="p">,</span> <span class="mi">1</span><span class="p">]))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">LeakyReLU</span><span class="p">())</span>
|
||
<span class="c1"># adding a dropout layer as you do in conv-nets</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="mf">0.3</span><span class="p">))</span>
|
||
|
||
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2D</span><span class="p">(</span><span class="n">filters</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||
<span class="n">kernel_size</span><span class="o">=</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">5</span><span class="p">),</span>
|
||
<span class="n">strides</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span>
|
||
<span class="n">padding</span><span class="o">=</span><span class="s1">'same'</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">LeakyReLU</span><span class="p">())</span>
|
||
<span class="c1"># adding a dropout layer as you do in conv-nets</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dropout</span><span class="p">(</span><span class="mf">0.3</span><span class="p">))</span>
|
||
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Flatten</span><span class="p">())</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="mi">1</span><span class="p">))</span>
|
||
|
||
<span class="k">return</span> <span class="n">model</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Let us take a look at our models.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">generator</span> <span class="o">=</span> <span class="n">generator_model</span><span class="p">()</span>
|
||
<span class="n">plot_model</span><span class="p">(</span><span class="n">generator</span><span class="p">,</span> <span class="n">show_shapes</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">rankdir</span><span class="o">=</span><span class="s1">'LR'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">discriminator</span> <span class="o">=</span> <span class="n">discriminator_model</span><span class="p">()</span>
|
||
<span class="n">plot_model</span><span class="p">(</span><span class="n">discriminator</span><span class="p">,</span> <span class="n">show_shapes</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">rankdir</span><span class="o">=</span><span class="s1">'LR'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Next we need a few helper objects we will use in training</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">cross_entropy</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">losses</span><span class="o">.</span><span class="n">BinaryCrossentropy</span><span class="p">(</span><span class="n">from_logits</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">generator_optimizer</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">optimizers</span><span class="o">.</span><span class="n">Adam</span><span class="p">(</span><span class="mf">1e-4</span><span class="p">)</span>
|
||
<span class="n">discriminator_optimizer</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">keras</span><span class="o">.</span><span class="n">optimizers</span><span class="o">.</span><span class="n">Adam</span><span class="p">(</span><span class="mf">1e-4</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The first object, <em>cross_entropy</em> is our loss function and the two others are
|
||
our optimizers. Notice we use the same learning rate for both <span class="math notranslate nohighlight">\(g\)</span> and <span class="math notranslate nohighlight">\(d\)</span>. This
|
||
is because they need to improve their accuracy at approximately equal speeds to
|
||
get convergence (not necessarily exactly equal). Now we define our loss
|
||
functions</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">generator_loss</span><span class="p">(</span><span class="n">fake_output</span><span class="p">):</span>
|
||
<span class="n">loss</span> <span class="o">=</span> <span class="n">cross_entropy</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">fake_output</span><span class="p">),</span> <span class="n">fake_output</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">loss</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">discriminator_loss</span><span class="p">(</span><span class="n">real_output</span><span class="p">,</span> <span class="n">fake_output</span><span class="p">):</span>
|
||
<span class="n">real_loss</span> <span class="o">=</span> <span class="n">cross_entropy</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">real_output</span><span class="p">),</span> <span class="n">real_output</span><span class="p">)</span>
|
||
<span class="n">fake_loss</span> <span class="o">=</span> <span class="n">cross_entropy</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">zeros_liks</span><span class="p">(</span><span class="n">fake_output</span><span class="p">),</span> <span class="n">fake_output</span><span class="p">)</span>
|
||
<span class="n">total_loss</span> <span class="o">=</span> <span class="n">real_loss</span> <span class="o">+</span> <span class="n">fake_loss</span>
|
||
|
||
<span class="k">return</span> <span class="n">total_loss</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Next we define a kind of seed to help us compare the learning process over
|
||
multiple training epochs.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">noise_dimension</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">n_examples_to_generate</span> <span class="o">=</span> <span class="mi">16</span>
|
||
<span class="n">seed_images</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">([</span><span class="n">n_examples_to_generate</span><span class="p">,</span> <span class="n">noise_dimension</span><span class="p">])</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Now we have everything we need to define our training step, which we will apply
|
||
for every step in our training loop. Notice the @tf.function flag signifying
|
||
that the function is tensorflow ‘compiled’. Removing this flag doubles the
|
||
computation time.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nd">@tf</span><span class="o">.</span><span class="n">function</span>
|
||
<span class="k">def</span> <span class="nf">train_step</span><span class="p">(</span><span class="n">images</span><span class="p">):</span>
|
||
<span class="n">noise</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">([</span><span class="n">BATCH_SIZE</span><span class="p">,</span> <span class="n">noise_dimension</span><span class="p">])</span>
|
||
|
||
<span class="k">with</span> <span class="n">tf</span><span class="o">.</span><span class="n">GradientTape</span><span class="p">()</span> <span class="k">as</span> <span class="n">gen_tape</span><span class="p">,</span> <span class="n">tf</span><span class="o">.</span><span class="n">GradientTape</span><span class="p">()</span> <span class="k">as</span> <span class="n">disc_tape</span><span class="p">:</span>
|
||
<span class="n">generated_images</span> <span class="o">=</span> <span class="n">generator</span><span class="p">(</span><span class="n">noise</span><span class="p">,</span> <span class="n">training</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="n">real_output</span> <span class="o">=</span> <span class="n">discriminator</span><span class="p">(</span><span class="n">images</span><span class="p">,</span> <span class="n">training</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="n">fake_output</span> <span class="o">=</span> <span class="n">discriminator</span><span class="p">(</span><span class="n">generated_images</span><span class="p">,</span> <span class="n">training</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
|
||
<span class="n">gen_loss</span> <span class="o">=</span> <span class="n">generator_loss</span><span class="p">(</span><span class="n">fake_output</span><span class="p">)</span>
|
||
<span class="n">disc_loss</span> <span class="o">=</span> <span class="n">discriminator_loss</span><span class="p">(</span><span class="n">real_output</span><span class="p">,</span> <span class="n">fake_output</span><span class="p">)</span>
|
||
|
||
<span class="n">gradients_of_generator</span> <span class="o">=</span> <span class="n">gen_tape</span><span class="o">.</span><span class="n">gradient</span><span class="p">(</span><span class="n">gen_loss</span><span class="p">,</span>
|
||
<span class="n">generator</span><span class="o">.</span><span class="n">trainable_variables</span><span class="p">)</span>
|
||
<span class="n">gradients_of_discriminator</span> <span class="o">=</span> <span class="n">disc_tape</span><span class="o">.</span><span class="n">gradient</span><span class="p">(</span><span class="n">disc_loss</span><span class="p">,</span>
|
||
<span class="n">discriminator</span><span class="o">.</span><span class="n">trainable_variables</span><span class="p">)</span>
|
||
<span class="n">generator_optimizer</span><span class="o">.</span><span class="n">apply_gradients</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">gradients_of_generator</span><span class="p">,</span>
|
||
<span class="n">generator</span><span class="o">.</span><span class="n">trainable_variables</span><span class="p">))</span>
|
||
<span class="n">discriminator_optimizer</span><span class="o">.</span><span class="n">apply_gradients</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">gradients_of_discriminator</span><span class="p">,</span>
|
||
<span class="n">discriminator</span><span class="o">.</span><span class="n">trainable_variables</span><span class="p">))</span>
|
||
|
||
<span class="k">return</span> <span class="n">gen_loss</span><span class="p">,</span> <span class="n">disc_loss</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Next we define a helper function to produce an output over our training epochs
|
||
to see the predictive progression of our generator model. <strong>Note</strong>: I am including
|
||
this code here, but comment it out in the training loop.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">generate_and_save_images</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">epoch</span><span class="p">,</span> <span class="n">test_input</span><span class="p">):</span>
|
||
<span class="c1"># we're making inferences here</span>
|
||
<span class="n">predictions</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">test_input</span><span class="p">,</span> <span class="n">training</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
|
||
<span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">predictions</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]):</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">predictions</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="p">:,</span> <span class="p">:,</span> <span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="mf">127.5</span> <span class="o">+</span> <span class="mf">127.5</span><span class="p">,</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">'gray'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s1">'off'</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">savefig</span><span class="p">(</span><span class="sa">f</span><span class="s1">'./images_from_seed_images/image_at_epoch_</span><span class="si">{</span><span class="nb">str</span><span class="p">(</span><span class="n">epoch</span><span class="p">)</span><span class="o">.</span><span class="n">zfill</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span><span class="si">}</span><span class="s1">.png'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||
<span class="c1">#plt.show()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Setting up checkpoints to periodically save our model during training so that
|
||
everything is not lost even if the program were to somehow terminate while
|
||
training.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Setting up checkpoints to save model during training</span>
|
||
<span class="n">checkpoint_dir</span> <span class="o">=</span> <span class="s1">'./training_checkpoints'</span>
|
||
<span class="n">checkpoint_prefix</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">checkpoint_dir</span><span class="p">,</span> <span class="s1">'ckpt'</span><span class="p">)</span>
|
||
<span class="n">checkpoint</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">train</span><span class="o">.</span><span class="n">Checkpoint</span><span class="p">(</span><span class="n">generator_optimizer</span><span class="o">=</span><span class="n">generator_optimizer</span><span class="p">,</span>
|
||
<span class="n">discriminator_optimizer</span><span class="o">=</span><span class="n">discriminator_optimizer</span><span class="p">,</span>
|
||
<span class="n">generator</span><span class="o">=</span><span class="n">generator</span><span class="p">,</span>
|
||
<span class="n">discriminator</span><span class="o">=</span><span class="n">discriminator</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Now we define our training loop</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">epochs</span><span class="p">):</span>
|
||
<span class="n">generator_loss_list</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="n">discriminator_loss_list</span> <span class="o">=</span> <span class="p">[]</span>
|
||
|
||
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">epochs</span><span class="p">):</span>
|
||
<span class="n">start</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
|
||
|
||
<span class="k">for</span> <span class="n">image_batch</span> <span class="ow">in</span> <span class="n">dataset</span><span class="p">:</span>
|
||
<span class="n">gen_loss</span><span class="p">,</span> <span class="n">disc_loss</span> <span class="o">=</span> <span class="n">train_step</span><span class="p">(</span><span class="n">image_batch</span><span class="p">)</span>
|
||
<span class="n">generator_loss_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">gen_loss</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||
<span class="n">discriminator_loss_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">disc_loss</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||
|
||
<span class="c1">#generate_and_save_images(generator, epoch + 1, seed_images)</span>
|
||
|
||
<span class="k">if</span> <span class="p">(</span><span class="n">epoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">%</span> <span class="mi">15</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||
<span class="n">checkpoint</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">file_prefix</span><span class="o">=</span><span class="n">checkpoint_prefix</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">'Time for epoch </span><span class="si">{</span><span class="n">epoch</span><span class="si">}</span><span class="s1"> is </span><span class="si">{</span><span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start</span><span class="si">}</span><span class="s1">'</span><span class="p">)</span>
|
||
|
||
<span class="c1">#generate_and_save_images(generator, epochs, seed_images)</span>
|
||
|
||
<span class="n">loss_file</span> <span class="o">=</span> <span class="s1">'./data/lossfile.txt'</span>
|
||
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">loss_file</span><span class="p">,</span> <span class="s1">'w'</span><span class="p">)</span> <span class="k">as</span> <span class="n">outfile</span><span class="p">:</span>
|
||
<span class="n">outfile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">generator_loss_list</span><span class="p">))</span>
|
||
<span class="n">outfile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
|
||
<span class="n">outfile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
|
||
<span class="n">outfile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">discriminator_loss_list</span><span class="p">))</span>
|
||
<span class="n">outfile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
|
||
<span class="n">outfile</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>To train simply call this function. <strong>Warning</strong>: this might take a long time so
|
||
there is a folder of a pretrained network already included in the repository.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">train</span><span class="p">(</span><span class="n">train_dataset</span><span class="p">,</span> <span class="n">EPOCHS</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Now to avoid having to train and everything, which will take a while depending
|
||
on your computer setup we now load in the model which produced the above gif.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">checkpoint</span><span class="o">.</span><span class="n">restore</span><span class="p">(</span><span class="n">tf</span><span class="o">.</span><span class="n">train</span><span class="o">.</span><span class="n">latest_checkpoint</span><span class="p">(</span><span class="n">checkpoint_dir</span><span class="p">))</span>
|
||
<span class="n">restored_generator</span> <span class="o">=</span> <span class="n">checkpoint</span><span class="o">.</span><span class="n">generator</span>
|
||
<span class="n">restored_discriminator</span> <span class="o">=</span> <span class="n">checkpoint</span><span class="o">.</span><span class="n">discriminator</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">restored_generator</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">restored_discriminator</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We have successfully loaded in our latest model. Let us now play around a bit
|
||
and see what kind of things we can learn about this model. Our generator takes
|
||
an array of 100 numbers. One idea can be to try to systematically change our
|
||
input. Let us try and see what we get</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">generate_latent_points</span><span class="p">(</span><span class="n">number</span><span class="o">=</span><span class="mi">100</span><span class="p">,</span> <span class="n">scale_means</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">scale_stds</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">latent_dim</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">means</span> <span class="o">=</span> <span class="n">scale_means</span> <span class="o">*</span> <span class="n">tf</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">num</span><span class="o">=</span><span class="n">latent_dim</span><span class="p">)</span>
|
||
<span class="n">stds</span> <span class="o">=</span> <span class="n">scale_stds</span> <span class="o">*</span> <span class="n">tf</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">num</span><span class="o">=</span><span class="n">latent_dim</span><span class="p">)</span>
|
||
<span class="n">latent_space_value_range</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">([</span><span class="n">number</span><span class="p">,</span> <span class="n">latent_dim</span><span class="p">],</span>
|
||
<span class="n">means</span><span class="p">,</span>
|
||
<span class="n">stds</span><span class="p">,</span>
|
||
<span class="n">dtype</span><span class="o">=</span><span class="n">tf</span><span class="o">.</span><span class="n">float64</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">latent_space_value_range</span>
|
||
|
||
<span class="k">def</span> <span class="nf">generate_images</span><span class="p">(</span><span class="n">latent_points</span><span class="p">):</span>
|
||
<span class="c1"># notice we set training to false because we are making inferences</span>
|
||
<span class="n">generated_images</span> <span class="o">=</span> <span class="n">restored_generator</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">latent_points</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">generated_images</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">plot_result</span><span class="p">(</span><span class="n">generated_images</span><span class="p">,</span> <span class="n">number</span><span class="o">=</span><span class="mi">100</span><span class="p">):</span>
|
||
<span class="c1"># obviously this assumes sqrt number is an int</span>
|
||
<span class="n">fig</span><span class="p">,</span> <span class="n">axs</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">number</span><span class="p">)),</span> <span class="nb">int</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">number</span><span class="p">)),</span>
|
||
<span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">number</span><span class="p">))):</span>
|
||
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">number</span><span class="p">))):</span>
|
||
<span class="n">axs</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">]</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">generated_images</span><span class="p">[</span><span class="n">i</span><span class="o">*</span><span class="n">j</span><span class="p">],</span> <span class="n">cmap</span><span class="o">=</span><span class="s1">'Greys'</span><span class="p">)</span>
|
||
<span class="n">axs</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="n">j</span><span class="p">]</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s1">'off'</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">generated_images</span> <span class="o">=</span> <span class="n">generate_images</span><span class="p">(</span><span class="n">generate_latent_points</span><span class="p">())</span>
|
||
<span class="n">plot_result</span><span class="p">(</span><span class="n">generated_images</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We see that the generator generates images that look like MNIST
|
||
numbers: <span class="math notranslate nohighlight">\(1, 4, 7, 9\)</span>. Let’s try to tweak it a bit more to see if we are able
|
||
to generate a similar plot where we generate every MNIST number. Let us now try
|
||
to ‘move’ a bit around in the latent space. <strong>Note</strong>: decrease the plot number if
|
||
these following cells take too long to run on your computer.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">plot_number</span> <span class="o">=</span> <span class="mi">225</span>
|
||
|
||
<span class="n">generated_images</span> <span class="o">=</span> <span class="n">generate_images</span><span class="p">(</span><span class="n">generate_latent_points</span><span class="p">(</span><span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">,</span>
|
||
<span class="n">scale_means</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
|
||
<span class="n">scale_stds</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">plot_result</span><span class="p">(</span><span class="n">generated_images</span><span class="p">,</span> <span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">)</span>
|
||
|
||
<span class="n">generated_images</span> <span class="o">=</span> <span class="n">generate_images</span><span class="p">(</span><span class="n">generate_latent_points</span><span class="p">(</span><span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">,</span>
|
||
<span class="n">scale_means</span><span class="o">=-</span><span class="mi">5</span><span class="p">,</span>
|
||
<span class="n">scale_stds</span><span class="o">=</span><span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">plot_result</span><span class="p">(</span><span class="n">generated_images</span><span class="p">,</span> <span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">)</span>
|
||
|
||
<span class="n">generated_images</span> <span class="o">=</span> <span class="n">generate_images</span><span class="p">(</span><span class="n">generate_latent_points</span><span class="p">(</span><span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">,</span>
|
||
<span class="n">scale_means</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||
<span class="n">scale_stds</span><span class="o">=</span><span class="mi">5</span><span class="p">))</span>
|
||
<span class="n">plot_result</span><span class="p">(</span><span class="n">generated_images</span><span class="p">,</span> <span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Again, we have found something interesting. <em>Moving</em> around using our means
|
||
takes us from digit to digit, while <em>moving</em> around using our standard
|
||
deviations seem to increase the number of different digits! In the last image
|
||
above, we can barely make out every MNIST digit. Let us make on last plot using
|
||
this information by upping the standard deviation of our Gaussian noises.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">plot_number</span> <span class="o">=</span> <span class="mi">400</span>
|
||
<span class="n">generated_images</span> <span class="o">=</span> <span class="n">generate_images</span><span class="p">(</span><span class="n">generate_latent_points</span><span class="p">(</span><span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">,</span>
|
||
<span class="n">scale_means</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||
<span class="n">scale_stds</span><span class="o">=</span><span class="mi">10</span><span class="p">))</span>
|
||
<span class="n">plot_result</span><span class="p">(</span><span class="n">generated_images</span><span class="p">,</span> <span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>A pretty cool result! We see that our generator indeed has learned a
|
||
distribution which qualitatively looks a whole lot like the MNIST dataset.</p>
|
||
<p>Another interesting way to explore the latent space of our generator model is by
|
||
interpolating between the MNIST digits. This section is largely based on
|
||
<a class="reference external" href="https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/">this excellent blogpost</a>
|
||
by Jason Brownlee.</p>
|
||
<p>So let us start by defining a function to interpolate between two points in the
|
||
latent space.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">interpolation</span><span class="p">(</span><span class="n">point_1</span><span class="p">,</span> <span class="n">point_2</span><span class="p">,</span> <span class="n">n_steps</span><span class="o">=</span><span class="mi">10</span><span class="p">):</span>
|
||
<span class="n">ratios</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">num</span><span class="o">=</span><span class="n">n_steps</span><span class="p">)</span>
|
||
<span class="n">vectors</span> <span class="o">=</span> <span class="p">[]</span>
|
||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">ratio</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">ratios</span><span class="p">):</span>
|
||
<span class="n">vectors</span><span class="o">.</span><span class="n">append</span><span class="p">(((</span><span class="mf">1.0</span> <span class="o">-</span> <span class="n">ratio</span><span class="p">)</span> <span class="o">*</span> <span class="n">point_1</span> <span class="o">+</span> <span class="n">ratio</span> <span class="o">*</span> <span class="n">point_2</span><span class="p">))</span>
|
||
|
||
<span class="k">return</span> <span class="n">tf</span><span class="o">.</span><span class="n">stack</span><span class="p">(</span><span class="n">vectors</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Now we have all we need to do our interpolation analysis.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">plot_number</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">latent_points</span> <span class="o">=</span> <span class="n">generate_latent_points</span><span class="p">(</span><span class="n">number</span><span class="o">=</span><span class="n">plot_number</span><span class="p">)</span>
|
||
<span class="n">results</span> <span class="o">=</span> <span class="kc">None</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">plot_number</span><span class="p">),</span> <span class="mi">2</span><span class="p">):</span>
|
||
<span class="n">interpolated</span> <span class="o">=</span> <span class="n">interpolation</span><span class="p">(</span><span class="n">latent_points</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">latent_points</span><span class="p">[</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="n">generated_images</span> <span class="o">=</span> <span class="n">generate_images</span><span class="p">(</span><span class="n">interpolated</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="n">results</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="n">results</span> <span class="o">=</span> <span class="n">generated_images</span>
|
||
<span class="k">else</span><span class="p">:</span>
|
||
<span class="n">results</span> <span class="o">=</span> <span class="n">tf</span><span class="o">.</span><span class="n">stack</span><span class="p">((</span><span class="n">results</span><span class="p">,</span> <span class="n">generated_images</span><span class="p">))</span>
|
||
|
||
<span class="n">plot_results</span><span class="p">(</span><span class="n">results</span><span class="p">,</span> <span class="n">plot_number</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
kernelName: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Previous / next buttons -->
|
||
<div class='prev-next-area'>
|
||
<a class='left-prev' id="prev-link" href="chapter12.html" title="previous page">
|
||
<i class="fas fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title"><span class="section-number">16. </span>Convolutional Neural Networks</p>
|
||
</div>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
<footer class="footer">
|
||
<p>
|
||
|
||
By Morten Hjorth-Jensen<br/>
|
||
|
||
© Copyright 2021.<br/>
|
||
</p>
|
||
</footer>
|
||
</main>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
|
||
|
||
</body>
|
||
</html> |