Merge branch 'master' of https://github.com/CompPhysics/MachineLearning
# Conflicts: # doc/pub/week42/ipynb/week42.ipynb
This commit is contained in:
Morten Hjorth-Jensen
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -186,7 +190,7 @@ MathJax.Hub.Config({
|
||||
|
||||
|
||||
<div class="jumbotron">
|
||||
<center><h1>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</h1></center> <!-- document title -->
|
||||
<center><h1>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</h1></center> <!-- document title -->
|
||||
|
||||
<p>
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -202,7 +206,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 15, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Oct 16, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -226,7 +230,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week42-bs008.html">9</a></li>
|
||||
<li><a href="._week42-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -186,12 +190,25 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0" class="anchor">Plan for week 42 </h2>
|
||||
|
||||
<ul>
|
||||
<li> Thursday: Convolutional Neural Networks and examples</li>
|
||||
<li> Friday: Recurrent Neural Networks and Autoencoders</li>
|
||||
<li> Thursday: Convolutional Neural Networks and examples. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober15.mp4?vrtx=view-as-webpage" target="_self">Video of Lecture</a></li>
|
||||
<li> Friday: Recurrent Neural Networks. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage" target="_self">Video of Lecture</a></li>
|
||||
</ul>
|
||||
|
||||
Reading suggestions for both days: <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_self">Aurelien Geron's chapters 13 and 14</a>. Autoencoders are discussed in chapter 15 of Geron's text.
|
||||
|
||||
<p>
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-body">
|
||||
<p> <!-- subsequent paragraphs come in larger fonts, so start with a paragraph -->
|
||||
|
||||
<ul>
|
||||
<li> <a href="https://www.youtube.com/watch?v=iaSUYvmCekI&ab_channel=AlexanderAmini" target="_self">Video on Convolutional Neural Networks from MIT</a></li>
|
||||
<li> <a href="https://www.youtube.com/watch?v=SEnXr6v2ifU&ab_channel=AlexanderAmini" target="_self">Video on Recurrent Neural Networks from MIT</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -209,7 +226,7 @@ Reading suggestions for both days: <a href="https://github.com/CompPhysics/Machi
|
||||
<li><a href="._week42-bs009.html">10</a></li>
|
||||
<li><a href="._week42-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -237,7 +241,7 @@ Another good read is the article here <a href="https://arxiv.org/pdf/1603.07285.
|
||||
<li><a href="._week42-bs010.html">11</a></li>
|
||||
<li><a href="._week42-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -213,7 +217,7 @@ before the transformation.
|
||||
<li><a href="._week42-bs011.html">12</a></li>
|
||||
<li><a href="._week42-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -228,7 +232,7 @@ in the input).
|
||||
<li><a href="._week42-bs012.html">13</a></li>
|
||||
<li><a href="._week42-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -229,7 +233,7 @@ would quickly lead to possible overfitting.
|
||||
<li><a href="._week42-bs013.html">14</a></li>
|
||||
<li><a href="._week42-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -242,7 +246,7 @@ dimension.
|
||||
<li><a href="._week42-bs014.html">15</a></li>
|
||||
<li><a href="._week42-bs015.html">16</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -226,7 +230,7 @@ A simple CNN for image classification could have the architecture:
|
||||
<li><a href="._week42-bs015.html">16</a></li>
|
||||
<li><a href="._week42-bs016.html">17</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -223,7 +227,7 @@ are consistent with the labels in the training set for each image.
|
||||
<li><a href="._week42-bs016.html">17</a></li>
|
||||
<li><a href="._week42-bs017.html">18</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -226,7 +230,7 @@ and the slides of <a href="http://cs231n.github.io/convolutional-networks/" targ
|
||||
<li><a href="._week42-bs017.html">18</a></li>
|
||||
<li><a href="._week42-bs018.html">19</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -222,7 +226,7 @@ matrices, typically 1 for each color dimension (Red, Green, Blue).
|
||||
<li><a href="._week42-bs018.html">19</a></li>
|
||||
<li><a href="._week42-bs019.html">20</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -218,7 +222,7 @@ $$
|
||||
<li><a href="._week42-bs019.html">20</a></li>
|
||||
<li><a href="._week42-bs020.html">21</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -224,7 +228,7 @@ single neuron in the first hidden layer.
|
||||
<li><a href="._week42-bs020.html">21</a></li>
|
||||
<li><a href="._week42-bs021.html">22</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -224,7 +228,7 @@ fixed, and known as a <a href="https://en.wikipedia.org/wiki/Receptive_field" ta
|
||||
<li><a href="._week42-bs021.html">22</a></li>
|
||||
<li><a href="._week42-bs022.html">23</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -228,7 +232,7 @@ layer.
|
||||
<li><a href="._week42-bs022.html">23</a></li>
|
||||
<li><a href="._week42-bs023.html">24</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -221,7 +225,7 @@ classification.
|
||||
<li><a href="._week42-bs023.html">24</a></li>
|
||||
<li><a href="._week42-bs024.html">25</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs016.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -254,7 +258,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week42-bs024.html">25</a></li>
|
||||
<li><a href="._week42-bs025.html">26</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs017.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -236,7 +240,7 @@ X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_tes
|
||||
<li><a href="._week42-bs025.html">26</a></li>
|
||||
<li><a href="._week42-bs026.html">27</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs018.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -241,7 +245,7 @@ lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.
|
||||
<li><a href="._week42-bs026.html">27</a></li>
|
||||
<li><a href="._week42-bs027.html">28</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs019.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,7 +235,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week42-bs027.html">28</a></li>
|
||||
<li><a href="._week42-bs028.html">29</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -245,7 +249,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week42-bs028.html">29</a></li>
|
||||
<li><a href="._week42-bs029.html">30</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -231,7 +235,7 @@ train_images, test_images <span style="color: #666666">=</span> train_images <sp
|
||||
<li><a href="._week42-bs029.html">30</a></li>
|
||||
<li><a href="._week42-bs030.html">31</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -232,7 +236,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week42-bs030.html">31</a></li>
|
||||
<li><a href="._week42-bs031.html">32</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs023.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -234,7 +238,7 @@ You can see that the output of every Conv2D and MaxPooling2D layer is a 3D tenso
|
||||
<li><a href="._week42-bs031.html">32</a></li>
|
||||
<li><a href="._week42-bs032.html">33</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs024.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -232,6 +236,8 @@ As you can see, our (4, 4, 64) outputs were flattened into vectors of shape (102
|
||||
<li><a href="._week42-bs031.html">32</a></li>
|
||||
<li><a href="._week42-bs032.html">33</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs025.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -88,16 +88,19 @@ Automatically generated HTML file from DocOnce source
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -127,7 +130,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</a>
|
||||
<a class="navbar-brand" href="week42-bs.html">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -162,12 +165,13 @@ MathJax.Hub.Config({
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs025.html#___sec24" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs026.html#___sec25" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs027.html#___sec26" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">Solving differential equations and eigenvalue problems with RNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Long-Short Time Memory</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Autoencoders: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Simple examples of Autoencoders</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs028.html#___sec27" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs029.html#___sec28" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs030.html#___sec29" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs031.html#___sec30" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs032.html#___sec31" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs033.html#___sec32" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week42-bs034.html#___sec33" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -186,7 +190,7 @@ MathJax.Hub.Config({
|
||||
|
||||
|
||||
<div class="jumbotron">
|
||||
<center><h1>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</h1></center> <!-- document title -->
|
||||
<center><h1>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</h1></center> <!-- document title -->
|
||||
|
||||
<p>
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -202,7 +206,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 15, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Oct 16, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -226,7 +230,7 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week42-bs008.html">9</a></li>
|
||||
<li><a href="._week42-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week42-bs033.html">34</a></li>
|
||||
<li><a href="._week42-bs034.html">35</a></li>
|
||||
<li><a href="._week42-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
|
||||
|
||||
@@ -132,7 +132,7 @@ MathJax.Hub.Config({
|
||||
|
||||
|
||||
|
||||
<center><h1 style="text-align: center;">Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</h1></center> <!-- document title -->
|
||||
<center><h1 style="text-align: center;">Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</h1></center> <!-- document title -->
|
||||
|
||||
<p>
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p> <br>
|
||||
<center><h4>Oct 15, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Oct 16, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
|
||||
@@ -162,12 +162,21 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Plan for week 42 </h2>
|
||||
|
||||
<ul>
|
||||
<p><li> Thursday: Convolutional Neural Networks and examples</li>
|
||||
<p><li> Friday: Recurrent Neural Networks and Autoencoders</li>
|
||||
<p><li> Thursday: Convolutional Neural Networks and examples. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober15.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a></li>
|
||||
<p><li> Friday: Recurrent Neural Networks. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a></li>
|
||||
</ul>
|
||||
<p>
|
||||
|
||||
Reading suggestions for both days: <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_blank">Aurelien Geron's chapters 13 and 14</a>. Autoencoders are discussed in chapter 15 of Geron's text.
|
||||
|
||||
<p>
|
||||
<div class="alert alert-block alert-block alert-text-normal">
|
||||
<b>Excellent lectures on CNNs and RNNs.</b>
|
||||
<ul>
|
||||
<p><li> <a href="https://www.youtube.com/watch?v=iaSUYvmCekI&ab_channel=AlexanderAmini" target="_blank">Video on Convolutional Neural Networks from MIT</a></li>
|
||||
<p><li> <a href="https://www.youtube.com/watch?v=SEnXr6v2ifU&ab_channel=AlexanderAmini" target="_blank">Video on Recurrent Neural Networks from MIT</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -842,7 +851,18 @@ systems such as automatic translation and speech-to-text.
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec27">A simple example </h2>
|
||||
<h2 id="___sec27">Set up of an RNN </h2>
|
||||
|
||||
<p>
|
||||
See the <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesOctober16.pdf" target="_blank">handwritten notes</a> and the <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage" target="_blank">video from the lecture of October 16</a>.
|
||||
|
||||
<p>
|
||||
More text will be added later.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec28">A simple example </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -920,84 +940,589 @@ plt.show()
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec28">Set up of an RNN </h2>
|
||||
<h2 id="___sec29">An extrapolation example </h2>
|
||||
|
||||
<p>
|
||||
The figure here displays a simple example of an RNN, with inputs \( x_t \)
|
||||
at a given time \( t \) and outputs \( y_t \). Introducing time as a variable
|
||||
offers an intutitive way of understanding these networks. In addition
|
||||
to the inputs \( x_t \), the layer at a time \( t \) receives also as input
|
||||
the output from the previous layer \( t-1 \), that is \( y_{t1} \).
|
||||
The following code provides an example of how recurrent neural
|
||||
networks can be used to extrapolate to unknown values of physics data
|
||||
sets. Specifically, the data sets used in this program come from
|
||||
a quantum mechanical many-body calculation of energies as functions of the number of particles.
|
||||
|
||||
<p>
|
||||
This means also that we need to have weights that link both the inputs
|
||||
\( x_t \) to the outputs \( y_t \) as well as weights that link the output
|
||||
from the previous time \( y_{t-1} \) and \( y_t \). The figure here shows an
|
||||
example of a simple RNN.
|
||||
|
||||
<p>
|
||||
More material will be added here.
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># For matrices and calculations</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #228B22"># For machine learning (backend for keras)</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
|
||||
<span style="color: #228B22"># User-friendly machine learning library</span>
|
||||
<span style="color: #228B22"># Front end for TensorFlow</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span>
|
||||
<span style="color: #228B22"># Different methods from Keras needed to create an RNN</span>
|
||||
<span style="color: #228B22"># This is not necessary but it shortened function calls </span>
|
||||
<span style="color: #228B22"># that need to be used in the code.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets, layers, models
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Input
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> regularizers
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.models</span> <span style="color: #8B008B; font-weight: bold">import</span> Model, Sequential
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Dense, SimpleRNN, LSTM, GRU
|
||||
<span style="color: #228B22"># For timing the code</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">timeit</span> <span style="color: #8B008B; font-weight: bold">import</span> default_timer <span style="color: #8B008B; font-weight: bold">as</span> timer
|
||||
<span style="color: #228B22"># For plotting</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
|
||||
|
||||
<span style="color: #228B22"># The data set</span>
|
||||
datatype=<span style="color: #CD5555">'VaryDimension'</span>
|
||||
X_tot = np.arange(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">42</span>, <span style="color: #B452CD">2</span>)
|
||||
y_tot = np.array([-<span style="color: #B452CD">0.03077640549</span>, -<span style="color: #B452CD">0.08336233266</span>, -<span style="color: #B452CD">0.1446729567</span>, -<span style="color: #B452CD">0.2116753732</span>, -<span style="color: #B452CD">0.2830637392</span>, -<span style="color: #B452CD">0.3581341341</span>, -<span style="color: #B452CD">0.436462435</span>, -<span style="color: #B452CD">0.5177783846</span>,
|
||||
-<span style="color: #B452CD">0.6019067271</span>, -<span style="color: #B452CD">0.6887363571</span>, -<span style="color: #B452CD">0.7782028952</span>, -<span style="color: #B452CD">0.8702784034</span>, -<span style="color: #B452CD">0.9649652536</span>, -<span style="color: #B452CD">1.062292565</span>, -<span style="color: #B452CD">1.16231451</span>,
|
||||
-<span style="color: #B452CD">1.265109911</span>, -<span style="color: #B452CD">1.370782966</span>, -<span style="color: #B452CD">1.479465113</span>, -<span style="color: #B452CD">1.591317992</span>, -<span style="color: #B452CD">1.70653767</span>])
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec29">Solving differential equations and eigenvalue problems with RNNs </h2>
|
||||
<h2 id="___sec30">Formatting the Data </h2>
|
||||
|
||||
<p>
|
||||
In our discussions of ordinary differential equations and partial
|
||||
differential equations using neural networks. Here we will discuss how
|
||||
we can solve say ordinary differential equations and eigenvalue
|
||||
problems using RNNs. Eigenvalue problems can be solved using RNNs by
|
||||
rewriting such a problems as a non-linear differential equation.
|
||||
The way the recurrent neural networks are trained in this program
|
||||
differs from how machine learning algorithms are usually trained.
|
||||
Typically a machine learning algorithm is trained by learning the
|
||||
relationship between the x data and the y data. In this program, the
|
||||
recurrent neural network will be trained to recognize the relationship
|
||||
in a sequence of y values. This is type of data formatting is
|
||||
typically used time series forcasting, but it can also be used in any
|
||||
extrapolation (time series forecasting is just a specific type of
|
||||
extrapolation along the time axis). This method of data formatting
|
||||
does not use the x data and assumes that the y data are evenly spaced.
|
||||
|
||||
<p>
|
||||
Instead of starting with a well-known ordinary differential equation,
|
||||
we start directly with an eigenvaule problem.
|
||||
For a standard machine learning algorithm, the training data has the
|
||||
form of (x,y) so the machine learning algorithm learns to assiciate a
|
||||
y value with a given x value. This is useful when the test data has x
|
||||
values within the same range as the training data. However, for this
|
||||
application, the x values of the test data are outside of the x values
|
||||
of the training data and the traditional method of training a machine
|
||||
learning algorithm does not work as well. For this reason, the
|
||||
recurrent neural network is trained on sequences of y values of the
|
||||
form ((y1, y2), y3), so that the network is concerned with learning
|
||||
the pattern of the y data and not the relation between the x and y
|
||||
data. As long as the pattern of y data outside of the training region
|
||||
stays relatively stable compared to what was inside the training
|
||||
region, this method of training can produce accurate extrapolations to
|
||||
y values far removed from the training data set.
|
||||
|
||||
<p>
|
||||
<!-- -->
|
||||
<!-- The idea behind formatting the data in this way comes from [this resource](https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/) and [this one](https://fairyonice.github.io/Understand-Keras%27s-RNN-behind-the-scenes-with-a-sin-wave-example.html). -->
|
||||
<!-- -->
|
||||
<!-- The following method takes in a y data set and formats it so the "x data" are of the form (y1, y2) and the "y data" are of the form y3, with extra brackets added in to make the resulting arrays compatable with both Keras and Tensorflow. -->
|
||||
<!-- -->
|
||||
<!-- Note: Using a sequence length of two is not required for time series forecasting so any lenght of sequence could be used (for example instead of ((y1, y2) y3) you could change the length of sequence to be 4 and the resulting data points would have the form ((y1, y2, y3, y4), y5)). While the following method can be used to create a data set of any sequence length, the remainder of the code expects the length of sequence to be 2. This is because the data sets are very small and the higher the lenght of the sequence the less resulting data points. -->
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #228B22"># FORMAT_DATA</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">format_data</span>(data, length_of_sequence = <span style="color: #B452CD">2</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> data(a numpy array): the data that will be the inputs to the recurrent neural</span>
|
||||
<span style="color: #CD5555"> network</span>
|
||||
<span style="color: #CD5555"> length_of_sequence (an int): the number of elements in one iteration of the</span>
|
||||
<span style="color: #CD5555"> sequence patter. For a function approximator use length_of_sequence = 2.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> rnn_input (a 3D numpy array): the input data for the recurrent neural network. Its</span>
|
||||
<span style="color: #CD5555"> dimensions are length of data - length of sequence, length of sequence, </span>
|
||||
<span style="color: #CD5555"> dimnsion of data</span>
|
||||
<span style="color: #CD5555"> rnn_output (a numpy array): the training data for the neural network</span>
|
||||
<span style="color: #CD5555"> Formats data to be used in a recurrent neural network.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
X, Y = [], []
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(data)-length_of_sequence):
|
||||
<span style="color: #228B22"># Get the next length_of_sequence elements</span>
|
||||
a = data[i:i+length_of_sequence]
|
||||
<span style="color: #228B22"># Get the element that immediately follows that</span>
|
||||
b = data[i+length_of_sequence]
|
||||
<span style="color: #228B22"># Reshape so that each data point is contained in its own array</span>
|
||||
a = np.reshape (a, (<span style="color: #658b00">len</span>(a), <span style="color: #B452CD">1</span>))
|
||||
X.append(a)
|
||||
Y.append(b)
|
||||
rnn_input = np.array(X)
|
||||
rnn_output = np.array(Y)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> rnn_input, rnn_output
|
||||
|
||||
|
||||
<span style="color: #228B22"># ## Defining the Recurrent Neural Network Using Keras</span>
|
||||
<span style="color: #228B22"># </span>
|
||||
<span style="color: #228B22"># The following method defines a simple recurrent neural network in keras consisting of one input layer, one hidden layer, and one output layer.</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">rnn</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with one hidden layer and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons in the input and output layers</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
<span style="color: #228B22"># Number of neurons in the hidden layer</span>
|
||||
hidden_neurons = <span style="color: #B452CD">200</span>
|
||||
<span style="color: #228B22"># Define the input layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Define the hidden layer as a simple RNN layer with a set number of neurons and add it to </span>
|
||||
<span style="color: #228B22"># the network immediately after the input layer</span>
|
||||
rnn = SimpleRNN(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN"</span>)(inp)
|
||||
<span style="color: #228B22"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||||
<span style="color: #228B22">#and add it to the network immediately after the hidden layer.</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn)
|
||||
<span style="color: #228B22"># Create the machine learning model starting with the input layer and ending with the </span>
|
||||
<span style="color: #228B22"># output layer</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||||
<span style="color: #228B22"># function and an Adams optimizer.</span>
|
||||
model.compile(loss=<span style="color: #CD5555">"mean_squared_error"</span>, optimizer=<span style="color: #CD5555">"adam"</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec30">Long-Short Time Memory </h2>
|
||||
<h2 id="___sec31">Predicting New Points With A Trained Recurrent Neural Network </h2>
|
||||
|
||||
<p>
|
||||
Discussions about dynamic unrolling through time. discuss memory cells, input and output
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">test_rnn</span> (x1, y_test, plot_min, plot_max):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> x1 (a list or numpy array): The complete x component of the data set</span>
|
||||
<span style="color: #CD5555"> y_test (a list or numpy array): The complete y component of the data set</span>
|
||||
<span style="color: #CD5555"> plot_min (an int or float): the smallest x value used in the training data</span>
|
||||
<span style="color: #CD5555"> plot_max (an int or float): the largest x valye used in the training data</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> None.</span>
|
||||
<span style="color: #CD5555"> Uses a trained recurrent neural network model to predict future points in the </span>
|
||||
<span style="color: #CD5555"> series. Computes the MSE of the predicted data set from the true data set, saves</span>
|
||||
<span style="color: #CD5555"> the predicted data set to a csv file, and plots the predicted and true data sets w</span>
|
||||
<span style="color: #CD5555"> while also displaying the data range used for training.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Add the training data as the first dim points in the predicted data array as these</span>
|
||||
<span style="color: #228B22"># are known values.</span>
|
||||
y_pred = y_test[:dim].tolist()
|
||||
<span style="color: #228B22"># Generate the first input to the trained recurrent neural network using the last two </span>
|
||||
<span style="color: #228B22"># points of the training data. Based on how the network was trained this means that it</span>
|
||||
<span style="color: #228B22"># will predict the first point in the data set after the training data. All of the </span>
|
||||
<span style="color: #228B22"># brackets are necessary for Tensorflow.</span>
|
||||
next_input = np.array([[[y_test[dim-<span style="color: #B452CD">2</span>]], [y_test[dim-<span style="color: #B452CD">1</span>]]]])
|
||||
<span style="color: #228B22"># Save the very last point in the training data set. This will be used later.</span>
|
||||
last = [y_test[dim-<span style="color: #B452CD">1</span>]]
|
||||
|
||||
<span style="color: #228B22"># Iterate until the complete data set is created.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span> (dim, <span style="color: #658b00">len</span>(y_test)):
|
||||
<span style="color: #228B22"># Predict the next point in the data set using the previous two points.</span>
|
||||
<span style="color: #658b00">next</span> = model.predict(next_input)
|
||||
<span style="color: #228B22"># Append just the number of the predicted data set</span>
|
||||
y_pred.append(<span style="color: #658b00">next</span>[<span style="color: #B452CD">0</span>][<span style="color: #B452CD">0</span>])
|
||||
<span style="color: #228B22"># Create the input that will be used to predict the next data point in the data set.</span>
|
||||
next_input = np.array([[last, <span style="color: #658b00">next</span>[<span style="color: #B452CD">0</span>]]], dtype=np.float64)
|
||||
last = <span style="color: #658b00">next</span>
|
||||
|
||||
<span style="color: #228B22"># Print the mean squared error between the known data set and the predicted data set.</span>
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'MSE: '</span>, np.square(np.subtract(y_test, y_pred)).mean())
|
||||
<span style="color: #228B22"># Save the predicted data set as a csv file for later use</span>
|
||||
name = datatype + <span style="color: #CD5555">'Predicted'</span>+<span style="color: #658b00">str</span>(dim)+<span style="color: #CD5555">'.csv'</span>
|
||||
np.savetxt(name, y_pred, delimiter=<span style="color: #CD5555">','</span>)
|
||||
<span style="color: #228B22"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||||
<span style="color: #228B22"># for the training data.</span>
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(x1, y_test, label=<span style="color: #CD5555">"true"</span>, linewidth=<span style="color: #B452CD">3</span>)
|
||||
ax.plot(x1, y_pred, <span style="color: #CD5555">'g-.'</span>,label=<span style="color: #CD5555">"predicted"</span>, linewidth=<span style="color: #B452CD">4</span>)
|
||||
ax.legend()
|
||||
<span style="color: #228B22"># Created a red region to represent the points used in the training data.</span>
|
||||
ax.axvspan(plot_min, plot_max, alpha=<span style="color: #B452CD">0.25</span>, color=<span style="color: #CD5555">'red'</span>)
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #228B22"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training = format_data(y_train, <span style="color: #B452CD">2</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
model = rnn(length_of_sequences = rnn_input.shape[<span style="color: #B452CD">1</span>])
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #B452CD">0</span>], X_tot[dim-<span style="color: #B452CD">1</span>])
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec31">Autoencoders: Overarching view </h2>
|
||||
<h2 id="___sec32">Other Things to Try </h2>
|
||||
|
||||
<p>
|
||||
Autoencoders are artificial neural networks capable of learning
|
||||
efficient representations of the input data (these representations are called codings) without
|
||||
any supervision (i.e., the training set is unlabeled). These codings
|
||||
typically have a much lower dimensionality than the input data, making
|
||||
autoencoders useful for dimensionality reduction.
|
||||
Changing the size of the recurrent neural network and its parameters
|
||||
can drastically change the results you get from the model. The below
|
||||
code takes the simple recurrent neural network from above and adds a
|
||||
second hidden layer, changes the number of neurons in the hidden
|
||||
layer, and explicitly declares the activation function of the hidden
|
||||
layers to be a sigmoid function. The loss function and optimizer can
|
||||
also be changed but are kept the same as the above network. These
|
||||
parameters can be tuned to provide the optimal result from the
|
||||
network. For some ideas on how to improve the performance of a
|
||||
<a href="https://danijar.com/tips-for-training-recurrent-neural-networks" target="_blank">recurrent neural network</a>.
|
||||
|
||||
<p>
|
||||
More importantly, autoencoders act as powerful feature detectors, and
|
||||
they can be used for unsupervised pretraining of deep neural networks.
|
||||
|
||||
<p>
|
||||
Lastly, they are capable of randomly generating new data that looks
|
||||
very similar to the training data; this is called a generative
|
||||
model. For example, you could train an autoencoder on pictures of
|
||||
faces, and it would then be able to generate new faces. Surprisingly,
|
||||
autoencoders work by simply learning to copy their inputs to their
|
||||
outputs. This may sound like a trivial task, but we will see that
|
||||
constraining the network in various ways can make it rather
|
||||
difficult. For example, you can limit the size of the internal
|
||||
representation, or you can add noise to the inputs and train the
|
||||
network to recover the original inputs. These constraints prevent the
|
||||
autoencoder from trivially copying the inputs directly to the outputs,
|
||||
which forces it to learn efficient ways of representing the data. In
|
||||
short, the codings are byproducts of the autoencoder’s attempt to
|
||||
learn the identity function under some constraints.
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">rnn_2layers</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with two hidden layers and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons in the input and output layers</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
<span style="color: #228B22"># Number of neurons in the hidden layer, increased from the first network</span>
|
||||
hidden_neurons = <span style="color: #B452CD">500</span>
|
||||
<span style="color: #228B22"># Define the input layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Create two hidden layers instead of one hidden layer. Explicitly set the activation</span>
|
||||
<span style="color: #228B22"># function to be the sigmoid function (the default value is hyperbolic tangent)</span>
|
||||
rnn1 = SimpleRNN(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">True</span>, <span style="color: #228B22"># This needs to be True if another hidden layer is to follow</span>
|
||||
stateful = stateful, activation = <span style="color: #CD5555">'sigmoid'</span>,
|
||||
name=<span style="color: #CD5555">"RNN1"</span>)(inp)
|
||||
rnn2 = SimpleRNN(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>, activation = <span style="color: #CD5555">'sigmoid'</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN2"</span>)(rnn1)
|
||||
<span style="color: #228B22"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||||
<span style="color: #228B22">#and add it to the network immediately after the hidden layer.</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn2)
|
||||
<span style="color: #228B22"># Create the machine learning model starting with the input layer and ending with the </span>
|
||||
<span style="color: #228B22"># output layer</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||||
<span style="color: #228B22"># function and an Adams optimizer.</span>
|
||||
model.compile(loss=<span style="color: #CD5555">"mean_squared_error"</span>, optimizer=<span style="color: #CD5555">"adam"</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #228B22"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training = format_data(y_train, <span style="color: #B452CD">2</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
model = rnn_2layers(length_of_sequences = <span style="color: #B452CD">2</span>)
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #228B22"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #228B22"># being overtrained.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #B452CD">0</span>], X_tot[dim-<span style="color: #B452CD">1</span>])
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="___sec32">Simple examples of Autoencoders </h2>
|
||||
<h2 id="___sec33">Other Types of Recurrent Neural Networks </h2>
|
||||
|
||||
<p>
|
||||
Besides a simple recurrent neural network layer, there are two other
|
||||
commonly used types of recurrent neural network layers: Long Short
|
||||
Term Memory (LSTM) and Gated Recurrent Unit (GRU). For a short
|
||||
introduction to these layers see <a href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b" target="_blank"><tt>https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</tt></a>
|
||||
and <a href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b" target="_blank"><tt>https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</tt></a>.
|
||||
|
||||
<p>
|
||||
The first network created below is similar to the previous network,
|
||||
but it replaces the SimpleRNN layers with LSTM layers. The second
|
||||
network below has two hidden layers made up of GRUs, which are
|
||||
preceeded by two dense (feeddorward) neural network layers. These
|
||||
dense layers "preprocess" the data before it reaches the recurrent
|
||||
layers. This architecture has been shown to improve the performance
|
||||
of recurrent neural networks (see the link above and also
|
||||
<a href="https://arxiv.org/pdf/1807.02857.pdf" target="_blank"><tt>https://arxiv.org/pdf/1807.02857.pdf</tt></a>.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">lstm_2layers</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with two LSTM hidden layers and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons on the input/output layer and the number of neurons in the hidden layer</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
hidden_neurons = <span style="color: #B452CD">250</span>
|
||||
<span style="color: #228B22"># Input Layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Hidden layers (in this case they are LSTM layers instead if SimpleRNN layers)</span>
|
||||
rnn= LSTM(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">True</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>, activation=<span style="color: #CD5555">'tanh'</span>)(inp)
|
||||
rnn1 = LSTM(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN1"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>, activation=<span style="color: #CD5555">'tanh'</span>)(rnn)
|
||||
<span style="color: #228B22"># Output layer</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn1)
|
||||
<span style="color: #228B22"># Define the midel</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the model</span>
|
||||
model.compile(loss=<span style="color: #CD5555">'mean_squared_error'</span>, optimizer=<span style="color: #CD5555">'adam'</span>)
|
||||
<span style="color: #228B22"># Return the model</span>
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">dnn2_gru2</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with four hidden layers (two dense followed by</span>
|
||||
<span style="color: #CD5555"> two GRU layers) and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons on the input/output layers and hidden layers</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
hidden_neurons = <span style="color: #B452CD">250</span>
|
||||
<span style="color: #228B22"># Input layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Hidden Dense (feedforward) layers</span>
|
||||
dnn = Dense(hidden_neurons/<span style="color: #B452CD">2</span>, activation=<span style="color: #CD5555">'relu'</span>, name=<span style="color: #CD5555">'dnn'</span>)(inp)
|
||||
dnn1 = Dense(hidden_neurons/<span style="color: #B452CD">2</span>, activation=<span style="color: #CD5555">'relu'</span>, name=<span style="color: #CD5555">'dnn1'</span>)(dnn)
|
||||
<span style="color: #228B22"># Hidden GRU layers</span>
|
||||
rnn1 = GRU(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">True</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN1"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>)(dnn1)
|
||||
rnn = GRU(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>)(rnn1)
|
||||
<span style="color: #228B22"># Output layer</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn)
|
||||
<span style="color: #228B22"># Define the model</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the mdoel</span>
|
||||
model.compile(loss=<span style="color: #CD5555">'mean_squared_error'</span>, optimizer=<span style="color: #CD5555">'adam'</span>)
|
||||
<span style="color: #228B22"># Return the model</span>
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #228B22"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training = format_data(y_train, <span style="color: #B452CD">2</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
<span style="color: #228B22"># Change the method name to reflect which network you want to use</span>
|
||||
model = dnn2_gru2(length_of_sequences = <span style="color: #B452CD">2</span>)
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #228B22"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #228B22"># being overtrained.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #B452CD">0</span>], X_tot[dim-<span style="color: #B452CD">1</span>])
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
|
||||
|
||||
<span style="color: #228B22"># ### Training Recurrent Neural Networks in the Standard Way (i.e. learning the relationship between the X and Y data)</span>
|
||||
<span style="color: #228B22"># </span>
|
||||
<span style="color: #228B22"># Finally, comparing the performace of a recurrent neural network using the standard data formatting to the performance of the network with time sequence data formatting shows the benefit of this type of data formatting with extrapolation.</span>
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
<span style="color: #228B22"># Reshape the data for Keras specifications</span>
|
||||
X_train = X_train.reshape((dim, <span style="color: #B452CD">1</span>))
|
||||
y_train = y_train.reshape((dim, <span style="color: #B452CD">1</span>))
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
<span style="color: #228B22"># Set the sequence length to 1 for regular data formatting </span>
|
||||
model = rnn(length_of_sequences = <span style="color: #B452CD">1</span>)
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(X_train, y_train, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #228B22"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #228B22"># being overtrained.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict the remaining data points</span>
|
||||
X_pred = X_tot[dim:]
|
||||
X_pred = X_pred.reshape((<span style="color: #658b00">len</span>(X_pred), <span style="color: #B452CD">1</span>))
|
||||
y_model = model.predict(X_pred)
|
||||
y_pred = np.concatenate((y_tot[:dim], y_model.flatten()))
|
||||
|
||||
<span style="color: #228B22"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||||
<span style="color: #228B22"># for the training data.</span>
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(X_tot, y_tot, label=<span style="color: #CD5555">"true"</span>, linewidth=<span style="color: #B452CD">3</span>)
|
||||
ax.plot(X_tot, y_pred, <span style="color: #CD5555">'g-.'</span>,label=<span style="color: #CD5555">"predicted"</span>, linewidth=<span style="color: #B452CD">4</span>)
|
||||
ax.legend()
|
||||
<span style="color: #228B22"># Created a red region to represent the points used in the training data.</span>
|
||||
ax.axvspan(X_tot[<span style="color: #B452CD">0</span>], X_tot[dim], alpha=<span style="color: #B452CD">0.25</span>, color=<span style="color: #CD5555">'red'</span>)
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
|
||||
<link href="https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_styles/style_solarized_box/css/solarized_light_code.css" rel="stylesheet" type="text/css" title="light"/>
|
||||
@@ -26,6 +26,32 @@ pre {
|
||||
border: 0pt solid #93a1a1;
|
||||
box-shadow: none;
|
||||
}
|
||||
.alert-text-small { font-size: 80%; }
|
||||
.alert-text-large { font-size: 130%; }
|
||||
.alert-text-normal { font-size: 90%; }
|
||||
.alert {
|
||||
padding:8px 35px 8px 14px; margin-bottom:18px;
|
||||
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
||||
border:1px solid #93a1a1;
|
||||
border-radius: 4px;
|
||||
-webkit-border-radius: 4px;
|
||||
-moz-border-radius: 4px;
|
||||
color: #555;
|
||||
background-color: #eee8d5;
|
||||
background-position: 10px 5px;
|
||||
background-repeat: no-repeat;
|
||||
background-size: 38px;
|
||||
padding-left: 55px;
|
||||
width: 75%;
|
||||
}
|
||||
.alert-block {padding-top:14px; padding-bottom:14px}
|
||||
.alert-block > p, .alert-block > ul {margin-bottom:1em}
|
||||
.alert li {margin-top: 1em}
|
||||
.alert-block p+p {margin-top:5px}
|
||||
.alert-notice { background-image: url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_yellow_notice.png); }
|
||||
.alert-summary { background-image:url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_yellow_summary.png); }
|
||||
.alert-warning { background-image: url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_yellow_warning.png); }
|
||||
.alert-question {background-image:url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_yellow_question.png); }
|
||||
|
||||
div { text-align: justify; text-justify: inter-word; }
|
||||
</style>
|
||||
@@ -82,16 +108,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -117,7 +146,7 @@ MathJax.Hub.Config({
|
||||
|
||||
|
||||
|
||||
<center><h1>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</h1></center> <!-- document title -->
|
||||
<center><h1>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</h1></center> <!-- document title -->
|
||||
|
||||
<p>
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -133,7 +162,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 15, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Oct 16, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -141,12 +170,24 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Plan for week 42 </h2>
|
||||
|
||||
<ul>
|
||||
<li> Thursday: Convolutional Neural Networks and examples</li>
|
||||
<li> Friday: Recurrent Neural Networks and Autoencoders</li>
|
||||
<li> Thursday: Convolutional Neural Networks and examples. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober15.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a></li>
|
||||
<li> Friday: Recurrent Neural Networks. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a></li>
|
||||
</ul>
|
||||
|
||||
Reading suggestions for both days: <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_blank">Aurelien Geron's chapters 13 and 14</a>. Autoencoders are discussed in chapter 15 of Geron's text.
|
||||
|
||||
<p>
|
||||
<div class="alert alert-block alert-block alert-text-normal">
|
||||
<b>Excellent lectures on CNNs and RNNs.</b>
|
||||
<p>
|
||||
|
||||
<ul>
|
||||
<li> <a href="https://www.youtube.com/watch?v=iaSUYvmCekI&ab_channel=AlexanderAmini" target="_blank">Video on Convolutional Neural Networks from MIT</a></li>
|
||||
<li> <a href="https://www.youtube.com/watch?v=SEnXr6v2ifU&ab_channel=AlexanderAmini" target="_blank">Video on Recurrent Neural Networks from MIT</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
@@ -805,7 +846,18 @@ systems such as automatic translation and speech-to-text.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">A simple example </h2>
|
||||
<h2 id="___sec27">Set up of an RNN </h2>
|
||||
|
||||
<p>
|
||||
See the <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesOctober16.pdf" target="_blank">handwritten notes</a> and the <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage" target="_blank">video from the lecture of October 16</a>.
|
||||
|
||||
<p>
|
||||
More text will be added later.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">A simple example </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -882,85 +934,585 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">Set up of an RNN </h2>
|
||||
<h2 id="___sec29">An extrapolation example </h2>
|
||||
|
||||
<p>
|
||||
The figure here displays a simple example of an RNN, with inputs \( x_t \)
|
||||
at a given time \( t \) and outputs \( y_t \). Introducing time as a variable
|
||||
offers an intutitive way of understanding these networks. In addition
|
||||
to the inputs \( x_t \), the layer at a time \( t \) receives also as input
|
||||
the output from the previous layer \( t-1 \), that is \( y_{t1} \).
|
||||
The following code provides an example of how recurrent neural
|
||||
networks can be used to extrapolate to unknown values of physics data
|
||||
sets. Specifically, the data sets used in this program come from
|
||||
a quantum mechanical many-body calculation of energies as functions of the number of particles.
|
||||
|
||||
<p>
|
||||
This means also that we need to have weights that link both the inputs
|
||||
\( x_t \) to the outputs \( y_t \) as well as weights that link the output
|
||||
from the previous time \( y_{t-1} \) and \( y_t \). The figure here shows an
|
||||
example of a simple RNN.
|
||||
|
||||
<p>
|
||||
More material will be added here.
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># For matrices and calculations</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #228B22"># For machine learning (backend for keras)</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
|
||||
<span style="color: #228B22"># User-friendly machine learning library</span>
|
||||
<span style="color: #228B22"># Front end for TensorFlow</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span>
|
||||
<span style="color: #228B22"># Different methods from Keras needed to create an RNN</span>
|
||||
<span style="color: #228B22"># This is not necessary but it shortened function calls </span>
|
||||
<span style="color: #228B22"># that need to be used in the code.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> datasets, layers, models
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Input
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> regularizers
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.models</span> <span style="color: #8B008B; font-weight: bold">import</span> Model, Sequential
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.layers</span> <span style="color: #8B008B; font-weight: bold">import</span> Dense, SimpleRNN, LSTM, GRU
|
||||
<span style="color: #228B22"># For timing the code</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">timeit</span> <span style="color: #8B008B; font-weight: bold">import</span> default_timer <span style="color: #8B008B; font-weight: bold">as</span> timer
|
||||
<span style="color: #228B22"># For plotting</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
|
||||
|
||||
<span style="color: #228B22"># The data set</span>
|
||||
datatype=<span style="color: #CD5555">'VaryDimension'</span>
|
||||
X_tot = np.arange(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">42</span>, <span style="color: #B452CD">2</span>)
|
||||
y_tot = np.array([-<span style="color: #B452CD">0.03077640549</span>, -<span style="color: #B452CD">0.08336233266</span>, -<span style="color: #B452CD">0.1446729567</span>, -<span style="color: #B452CD">0.2116753732</span>, -<span style="color: #B452CD">0.2830637392</span>, -<span style="color: #B452CD">0.3581341341</span>, -<span style="color: #B452CD">0.436462435</span>, -<span style="color: #B452CD">0.5177783846</span>,
|
||||
-<span style="color: #B452CD">0.6019067271</span>, -<span style="color: #B452CD">0.6887363571</span>, -<span style="color: #B452CD">0.7782028952</span>, -<span style="color: #B452CD">0.8702784034</span>, -<span style="color: #B452CD">0.9649652536</span>, -<span style="color: #B452CD">1.062292565</span>, -<span style="color: #B452CD">1.16231451</span>,
|
||||
-<span style="color: #B452CD">1.265109911</span>, -<span style="color: #B452CD">1.370782966</span>, -<span style="color: #B452CD">1.479465113</span>, -<span style="color: #B452CD">1.591317992</span>, -<span style="color: #B452CD">1.70653767</span>])
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec29">Solving differential equations and eigenvalue problems with RNNs </h2>
|
||||
<h2 id="___sec30">Formatting the Data </h2>
|
||||
|
||||
<p>
|
||||
In our discussions of ordinary differential equations and partial
|
||||
differential equations using neural networks. Here we will discuss how
|
||||
we can solve say ordinary differential equations and eigenvalue
|
||||
problems using RNNs. Eigenvalue problems can be solved using RNNs by
|
||||
rewriting such a problems as a non-linear differential equation.
|
||||
The way the recurrent neural networks are trained in this program
|
||||
differs from how machine learning algorithms are usually trained.
|
||||
Typically a machine learning algorithm is trained by learning the
|
||||
relationship between the x data and the y data. In this program, the
|
||||
recurrent neural network will be trained to recognize the relationship
|
||||
in a sequence of y values. This is type of data formatting is
|
||||
typically used time series forcasting, but it can also be used in any
|
||||
extrapolation (time series forecasting is just a specific type of
|
||||
extrapolation along the time axis). This method of data formatting
|
||||
does not use the x data and assumes that the y data are evenly spaced.
|
||||
|
||||
<p>
|
||||
Instead of starting with a well-known ordinary differential equation,
|
||||
we start directly with an eigenvaule problem.
|
||||
For a standard machine learning algorithm, the training data has the
|
||||
form of (x,y) so the machine learning algorithm learns to assiciate a
|
||||
y value with a given x value. This is useful when the test data has x
|
||||
values within the same range as the training data. However, for this
|
||||
application, the x values of the test data are outside of the x values
|
||||
of the training data and the traditional method of training a machine
|
||||
learning algorithm does not work as well. For this reason, the
|
||||
recurrent neural network is trained on sequences of y values of the
|
||||
form ((y1, y2), y3), so that the network is concerned with learning
|
||||
the pattern of the y data and not the relation between the x and y
|
||||
data. As long as the pattern of y data outside of the training region
|
||||
stays relatively stable compared to what was inside the training
|
||||
region, this method of training can produce accurate extrapolations to
|
||||
y values far removed from the training data set.
|
||||
|
||||
<p>
|
||||
<!-- -->
|
||||
<!-- The idea behind formatting the data in this way comes from [this resource](https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/) and [this one](https://fairyonice.github.io/Understand-Keras%27s-RNN-behind-the-scenes-with-a-sin-wave-example.html). -->
|
||||
<!-- -->
|
||||
<!-- The following method takes in a y data set and formats it so the "x data" are of the form (y1, y2) and the "y data" are of the form y3, with extra brackets added in to make the resulting arrays compatable with both Keras and Tensorflow. -->
|
||||
<!-- -->
|
||||
<!-- Note: Using a sequence length of two is not required for time series forecasting so any lenght of sequence could be used (for example instead of ((y1, y2) y3) you could change the length of sequence to be 4 and the resulting data points would have the form ((y1, y2, y3, y4), y5)). While the following method can be used to create a data set of any sequence length, the remainder of the code expects the length of sequence to be 2. This is because the data sets are very small and the higher the lenght of the sequence the less resulting data points. -->
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #228B22"># FORMAT_DATA</span>
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">format_data</span>(data, length_of_sequence = <span style="color: #B452CD">2</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> data(a numpy array): the data that will be the inputs to the recurrent neural</span>
|
||||
<span style="color: #CD5555"> network</span>
|
||||
<span style="color: #CD5555"> length_of_sequence (an int): the number of elements in one iteration of the</span>
|
||||
<span style="color: #CD5555"> sequence patter. For a function approximator use length_of_sequence = 2.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> rnn_input (a 3D numpy array): the input data for the recurrent neural network. Its</span>
|
||||
<span style="color: #CD5555"> dimensions are length of data - length of sequence, length of sequence, </span>
|
||||
<span style="color: #CD5555"> dimnsion of data</span>
|
||||
<span style="color: #CD5555"> rnn_output (a numpy array): the training data for the neural network</span>
|
||||
<span style="color: #CD5555"> Formats data to be used in a recurrent neural network.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
X, Y = [], []
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">len</span>(data)-length_of_sequence):
|
||||
<span style="color: #228B22"># Get the next length_of_sequence elements</span>
|
||||
a = data[i:i+length_of_sequence]
|
||||
<span style="color: #228B22"># Get the element that immediately follows that</span>
|
||||
b = data[i+length_of_sequence]
|
||||
<span style="color: #228B22"># Reshape so that each data point is contained in its own array</span>
|
||||
a = np.reshape (a, (<span style="color: #658b00">len</span>(a), <span style="color: #B452CD">1</span>))
|
||||
X.append(a)
|
||||
Y.append(b)
|
||||
rnn_input = np.array(X)
|
||||
rnn_output = np.array(Y)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> rnn_input, rnn_output
|
||||
|
||||
|
||||
<span style="color: #228B22"># ## Defining the Recurrent Neural Network Using Keras</span>
|
||||
<span style="color: #228B22"># </span>
|
||||
<span style="color: #228B22"># The following method defines a simple recurrent neural network in keras consisting of one input layer, one hidden layer, and one output layer.</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">rnn</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with one hidden layer and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons in the input and output layers</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
<span style="color: #228B22"># Number of neurons in the hidden layer</span>
|
||||
hidden_neurons = <span style="color: #B452CD">200</span>
|
||||
<span style="color: #228B22"># Define the input layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Define the hidden layer as a simple RNN layer with a set number of neurons and add it to </span>
|
||||
<span style="color: #228B22"># the network immediately after the input layer</span>
|
||||
rnn = SimpleRNN(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN"</span>)(inp)
|
||||
<span style="color: #228B22"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||||
<span style="color: #228B22">#and add it to the network immediately after the hidden layer.</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn)
|
||||
<span style="color: #228B22"># Create the machine learning model starting with the input layer and ending with the </span>
|
||||
<span style="color: #228B22"># output layer</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||||
<span style="color: #228B22"># function and an Adams optimizer.</span>
|
||||
model.compile(loss=<span style="color: #CD5555">"mean_squared_error"</span>, optimizer=<span style="color: #CD5555">"adam"</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">Long-Short Time Memory </h2>
|
||||
<h2 id="___sec31">Predicting New Points With A Trained Recurrent Neural Network </h2>
|
||||
|
||||
<p>
|
||||
Discussions about dynamic unrolling through time. discuss memory cells, input and output
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">test_rnn</span> (x1, y_test, plot_min, plot_max):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> x1 (a list or numpy array): The complete x component of the data set</span>
|
||||
<span style="color: #CD5555"> y_test (a list or numpy array): The complete y component of the data set</span>
|
||||
<span style="color: #CD5555"> plot_min (an int or float): the smallest x value used in the training data</span>
|
||||
<span style="color: #CD5555"> plot_max (an int or float): the largest x valye used in the training data</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> None.</span>
|
||||
<span style="color: #CD5555"> Uses a trained recurrent neural network model to predict future points in the </span>
|
||||
<span style="color: #CD5555"> series. Computes the MSE of the predicted data set from the true data set, saves</span>
|
||||
<span style="color: #CD5555"> the predicted data set to a csv file, and plots the predicted and true data sets w</span>
|
||||
<span style="color: #CD5555"> while also displaying the data range used for training.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Add the training data as the first dim points in the predicted data array as these</span>
|
||||
<span style="color: #228B22"># are known values.</span>
|
||||
y_pred = y_test[:dim].tolist()
|
||||
<span style="color: #228B22"># Generate the first input to the trained recurrent neural network using the last two </span>
|
||||
<span style="color: #228B22"># points of the training data. Based on how the network was trained this means that it</span>
|
||||
<span style="color: #228B22"># will predict the first point in the data set after the training data. All of the </span>
|
||||
<span style="color: #228B22"># brackets are necessary for Tensorflow.</span>
|
||||
next_input = np.array([[[y_test[dim-<span style="color: #B452CD">2</span>]], [y_test[dim-<span style="color: #B452CD">1</span>]]]])
|
||||
<span style="color: #228B22"># Save the very last point in the training data set. This will be used later.</span>
|
||||
last = [y_test[dim-<span style="color: #B452CD">1</span>]]
|
||||
|
||||
<span style="color: #228B22"># Iterate until the complete data set is created.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span> (dim, <span style="color: #658b00">len</span>(y_test)):
|
||||
<span style="color: #228B22"># Predict the next point in the data set using the previous two points.</span>
|
||||
<span style="color: #658b00">next</span> = model.predict(next_input)
|
||||
<span style="color: #228B22"># Append just the number of the predicted data set</span>
|
||||
y_pred.append(<span style="color: #658b00">next</span>[<span style="color: #B452CD">0</span>][<span style="color: #B452CD">0</span>])
|
||||
<span style="color: #228B22"># Create the input that will be used to predict the next data point in the data set.</span>
|
||||
next_input = np.array([[last, <span style="color: #658b00">next</span>[<span style="color: #B452CD">0</span>]]], dtype=np.float64)
|
||||
last = <span style="color: #658b00">next</span>
|
||||
|
||||
<span style="color: #228B22"># Print the mean squared error between the known data set and the predicted data set.</span>
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'MSE: '</span>, np.square(np.subtract(y_test, y_pred)).mean())
|
||||
<span style="color: #228B22"># Save the predicted data set as a csv file for later use</span>
|
||||
name = datatype + <span style="color: #CD5555">'Predicted'</span>+<span style="color: #658b00">str</span>(dim)+<span style="color: #CD5555">'.csv'</span>
|
||||
np.savetxt(name, y_pred, delimiter=<span style="color: #CD5555">','</span>)
|
||||
<span style="color: #228B22"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||||
<span style="color: #228B22"># for the training data.</span>
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(x1, y_test, label=<span style="color: #CD5555">"true"</span>, linewidth=<span style="color: #B452CD">3</span>)
|
||||
ax.plot(x1, y_pred, <span style="color: #CD5555">'g-.'</span>,label=<span style="color: #CD5555">"predicted"</span>, linewidth=<span style="color: #B452CD">4</span>)
|
||||
ax.legend()
|
||||
<span style="color: #228B22"># Created a red region to represent the points used in the training data.</span>
|
||||
ax.axvspan(plot_min, plot_max, alpha=<span style="color: #B452CD">0.25</span>, color=<span style="color: #CD5555">'red'</span>)
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #228B22"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training = format_data(y_train, <span style="color: #B452CD">2</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
model = rnn(length_of_sequences = rnn_input.shape[<span style="color: #B452CD">1</span>])
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #B452CD">0</span>], X_tot[dim-<span style="color: #B452CD">1</span>])
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec31">Autoencoders: Overarching view </h2>
|
||||
<h2 id="___sec32">Other Things to Try </h2>
|
||||
|
||||
<p>
|
||||
Autoencoders are artificial neural networks capable of learning
|
||||
efficient representations of the input data (these representations are called codings) without
|
||||
any supervision (i.e., the training set is unlabeled). These codings
|
||||
typically have a much lower dimensionality than the input data, making
|
||||
autoencoders useful for dimensionality reduction.
|
||||
Changing the size of the recurrent neural network and its parameters
|
||||
can drastically change the results you get from the model. The below
|
||||
code takes the simple recurrent neural network from above and adds a
|
||||
second hidden layer, changes the number of neurons in the hidden
|
||||
layer, and explicitly declares the activation function of the hidden
|
||||
layers to be a sigmoid function. The loss function and optimizer can
|
||||
also be changed but are kept the same as the above network. These
|
||||
parameters can be tuned to provide the optimal result from the
|
||||
network. For some ideas on how to improve the performance of a
|
||||
<a href="https://danijar.com/tips-for-training-recurrent-neural-networks" target="_blank">recurrent neural network</a>.
|
||||
|
||||
<p>
|
||||
More importantly, autoencoders act as powerful feature detectors, and
|
||||
they can be used for unsupervised pretraining of deep neural networks.
|
||||
|
||||
<p>
|
||||
Lastly, they are capable of randomly generating new data that looks
|
||||
very similar to the training data; this is called a generative
|
||||
model. For example, you could train an autoencoder on pictures of
|
||||
faces, and it would then be able to generate new faces. Surprisingly,
|
||||
autoencoders work by simply learning to copy their inputs to their
|
||||
outputs. This may sound like a trivial task, but we will see that
|
||||
constraining the network in various ways can make it rather
|
||||
difficult. For example, you can limit the size of the internal
|
||||
representation, or you can add noise to the inputs and train the
|
||||
network to recover the original inputs. These constraints prevent the
|
||||
autoencoder from trivially copying the inputs directly to the outputs,
|
||||
which forces it to learn efficient ways of representing the data. In
|
||||
short, the codings are byproducts of the autoencoder’s attempt to
|
||||
learn the identity function under some constraints.
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">rnn_2layers</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with two hidden layers and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons in the input and output layers</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
<span style="color: #228B22"># Number of neurons in the hidden layer, increased from the first network</span>
|
||||
hidden_neurons = <span style="color: #B452CD">500</span>
|
||||
<span style="color: #228B22"># Define the input layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Create two hidden layers instead of one hidden layer. Explicitly set the activation</span>
|
||||
<span style="color: #228B22"># function to be the sigmoid function (the default value is hyperbolic tangent)</span>
|
||||
rnn1 = SimpleRNN(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">True</span>, <span style="color: #228B22"># This needs to be True if another hidden layer is to follow</span>
|
||||
stateful = stateful, activation = <span style="color: #CD5555">'sigmoid'</span>,
|
||||
name=<span style="color: #CD5555">"RNN1"</span>)(inp)
|
||||
rnn2 = SimpleRNN(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>, activation = <span style="color: #CD5555">'sigmoid'</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN2"</span>)(rnn1)
|
||||
<span style="color: #228B22"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||||
<span style="color: #228B22">#and add it to the network immediately after the hidden layer.</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn2)
|
||||
<span style="color: #228B22"># Create the machine learning model starting with the input layer and ending with the </span>
|
||||
<span style="color: #228B22"># output layer</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||||
<span style="color: #228B22"># function and an Adams optimizer.</span>
|
||||
model.compile(loss=<span style="color: #CD5555">"mean_squared_error"</span>, optimizer=<span style="color: #CD5555">"adam"</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #228B22"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training = format_data(y_train, <span style="color: #B452CD">2</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
model = rnn_2layers(length_of_sequences = <span style="color: #B452CD">2</span>)
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #228B22"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #228B22"># being overtrained.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #B452CD">0</span>], X_tot[dim-<span style="color: #B452CD">1</span>])
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec32">Simple examples of Autoencoders </h2>
|
||||
<h2 id="___sec33">Other Types of Recurrent Neural Networks </h2>
|
||||
|
||||
<p>
|
||||
Besides a simple recurrent neural network layer, there are two other
|
||||
commonly used types of recurrent neural network layers: Long Short
|
||||
Term Memory (LSTM) and Gated Recurrent Unit (GRU). For a short
|
||||
introduction to these layers see <a href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b" target="_blank"><tt>https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</tt></a>
|
||||
and <a href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b" target="_blank"><tt>https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</tt></a>.
|
||||
|
||||
<p>
|
||||
The first network created below is similar to the previous network,
|
||||
but it replaces the SimpleRNN layers with LSTM layers. The second
|
||||
network below has two hidden layers made up of GRUs, which are
|
||||
preceeded by two dense (feeddorward) neural network layers. These
|
||||
dense layers "preprocess" the data before it reaches the recurrent
|
||||
layers. This architecture has been shown to improve the performance
|
||||
of recurrent neural networks (see the link above and also
|
||||
<a href="https://arxiv.org/pdf/1807.02857.pdf" target="_blank"><tt>https://arxiv.org/pdf/1807.02857.pdf</tt></a>.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">lstm_2layers</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with two LSTM hidden layers and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons on the input/output layer and the number of neurons in the hidden layer</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
hidden_neurons = <span style="color: #B452CD">250</span>
|
||||
<span style="color: #228B22"># Input Layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Hidden layers (in this case they are LSTM layers instead if SimpleRNN layers)</span>
|
||||
rnn= LSTM(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">True</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>, activation=<span style="color: #CD5555">'tanh'</span>)(inp)
|
||||
rnn1 = LSTM(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN1"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>, activation=<span style="color: #CD5555">'tanh'</span>)(rnn)
|
||||
<span style="color: #228B22"># Output layer</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn1)
|
||||
<span style="color: #228B22"># Define the midel</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the model</span>
|
||||
model.compile(loss=<span style="color: #CD5555">'mean_squared_error'</span>, optimizer=<span style="color: #CD5555">'adam'</span>)
|
||||
<span style="color: #228B22"># Return the model</span>
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">dnn2_gru2</span>(length_of_sequences, batch_size = <span style="color: #8B008B; font-weight: bold">None</span>, stateful = <span style="color: #8B008B; font-weight: bold">False</span>):
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> Inputs:</span>
|
||||
<span style="color: #CD5555"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #CD5555"> when the data is formatted</span>
|
||||
<span style="color: #CD5555"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #CD5555"> Returns:</span>
|
||||
<span style="color: #CD5555"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #CD5555"> method</span>
|
||||
<span style="color: #CD5555"> Builds and compiles a recurrent neural network with four hidden layers (two dense followed by</span>
|
||||
<span style="color: #CD5555"> two GRU layers) and returns the model.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
<span style="color: #228B22"># Number of neurons on the input/output layers and hidden layers</span>
|
||||
in_out_neurons = <span style="color: #B452CD">1</span>
|
||||
hidden_neurons = <span style="color: #B452CD">250</span>
|
||||
<span style="color: #228B22"># Input layer</span>
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #228B22"># Hidden Dense (feedforward) layers</span>
|
||||
dnn = Dense(hidden_neurons/<span style="color: #B452CD">2</span>, activation=<span style="color: #CD5555">'relu'</span>, name=<span style="color: #CD5555">'dnn'</span>)(inp)
|
||||
dnn1 = Dense(hidden_neurons/<span style="color: #B452CD">2</span>, activation=<span style="color: #CD5555">'relu'</span>, name=<span style="color: #CD5555">'dnn1'</span>)(dnn)
|
||||
<span style="color: #228B22"># Hidden GRU layers</span>
|
||||
rnn1 = GRU(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">True</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN1"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>)(dnn1)
|
||||
rnn = GRU(hidden_neurons,
|
||||
return_sequences=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
stateful = stateful,
|
||||
name=<span style="color: #CD5555">"RNN"</span>, use_bias=<span style="color: #8B008B; font-weight: bold">True</span>)(rnn1)
|
||||
<span style="color: #228B22"># Output layer</span>
|
||||
dens = Dense(in_out_neurons,name=<span style="color: #CD5555">"dense"</span>)(rnn)
|
||||
<span style="color: #228B22"># Define the model</span>
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
<span style="color: #228B22"># Compile the mdoel</span>
|
||||
model.compile(loss=<span style="color: #CD5555">'mean_squared_error'</span>, optimizer=<span style="color: #CD5555">'adam'</span>)
|
||||
<span style="color: #228B22"># Return the model</span>
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #228B22"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training = format_data(y_train, <span style="color: #B452CD">2</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
<span style="color: #228B22"># Change the method name to reflect which network you want to use</span>
|
||||
model = dnn2_gru2(length_of_sequences = <span style="color: #B452CD">2</span>)
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #228B22"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #228B22"># being overtrained.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #B452CD">0</span>], X_tot[dim-<span style="color: #B452CD">1</span>])
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
|
||||
|
||||
<span style="color: #228B22"># ### Training Recurrent Neural Networks in the Standard Way (i.e. learning the relationship between the X and Y data)</span>
|
||||
<span style="color: #228B22"># </span>
|
||||
<span style="color: #228B22"># Finally, comparing the performace of a recurrent neural network using the standard data formatting to the performance of the network with time sequence data formatting shows the benefit of this type of data formatting with extrapolation.</span>
|
||||
|
||||
<span style="color: #228B22"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> <span style="color: #658b00">len</span>(X_tot) == <span style="color: #658b00">len</span>(y_tot)
|
||||
|
||||
<span style="color: #228B22"># This is the number of points that will be used in as the training data</span>
|
||||
dim=<span style="color: #B452CD">12</span>
|
||||
|
||||
<span style="color: #228B22"># Separate the training data from the whole data set</span>
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
<span style="color: #228B22"># Reshape the data for Keras specifications</span>
|
||||
X_train = X_train.reshape((dim, <span style="color: #B452CD">1</span>))
|
||||
y_train = y_train.reshape((dim, <span style="color: #B452CD">1</span>))
|
||||
|
||||
|
||||
<span style="color: #228B22"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #228B22"># machine learning model</span>
|
||||
<span style="color: #228B22"># Set the sequence length to 1 for regular data formatting </span>
|
||||
model = rnn(length_of_sequences = <span style="color: #B452CD">1</span>)
|
||||
model.summary()
|
||||
|
||||
<span style="color: #228B22"># Start the timer. Want to time training+testing</span>
|
||||
start = timer()
|
||||
<span style="color: #228B22"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #228B22"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist = model.fit(X_train, y_train, batch_size=<span style="color: #8B008B; font-weight: bold">None</span>, epochs=<span style="color: #B452CD">150</span>,
|
||||
verbose=<span style="color: #8B008B; font-weight: bold">True</span>,validation_split=<span style="color: #B452CD">0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #228B22"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #228B22"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #228B22"># being overtrained.</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> label <span style="color: #8B008B">in</span> [<span style="color: #CD5555">"loss"</span>,<span style="color: #CD5555">"val_loss"</span>]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel(<span style="color: #CD5555">"loss"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"epoch"</span>)
|
||||
plt.title(<span style="color: #CD5555">"The final validation loss: {}"</span>.format(hist.history[<span style="color: #CD5555">"val_loss"</span>][-<span style="color: #B452CD">1</span>]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Use the trained neural network to predict the remaining data points</span>
|
||||
X_pred = X_tot[dim:]
|
||||
X_pred = X_pred.reshape((<span style="color: #658b00">len</span>(X_pred), <span style="color: #B452CD">1</span>))
|
||||
y_model = model.predict(X_pred)
|
||||
y_pred = np.concatenate((y_tot[:dim], y_model.flatten()))
|
||||
|
||||
<span style="color: #228B22"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||||
<span style="color: #228B22"># for the training data.</span>
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(X_tot, y_tot, label=<span style="color: #CD5555">"true"</span>, linewidth=<span style="color: #B452CD">3</span>)
|
||||
ax.plot(X_tot, y_pred, <span style="color: #CD5555">'g-.'</span>,label=<span style="color: #CD5555">"predicted"</span>, linewidth=<span style="color: #B452CD">4</span>)
|
||||
ax.legend()
|
||||
<span style="color: #228B22"># Created a red region to represent the points used in the training data.</span>
|
||||
ax.axvspan(X_tot[<span style="color: #B452CD">0</span>], X_tot[dim], alpha=<span style="color: #B452CD">0.25</span>, color=<span style="color: #CD5555">'red'</span>)
|
||||
plt.show()
|
||||
|
||||
<span style="color: #228B22"># Stop the timer and calculate the total time needed.</span>
|
||||
end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
+613
-61
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 42 Convolutional and Recurrent Neural Networks and Autoencoders">
|
||||
<meta name="description" content="Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks">
|
||||
|
||||
<title>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</title>
|
||||
<title>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</title>
|
||||
|
||||
|
||||
<style type="text/css">
|
||||
@@ -31,6 +31,32 @@ p { text-indent: 0px; }
|
||||
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
||||
p.caption { width: 80%; font-style: normal; text-align: left; }
|
||||
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
||||
.alert-text-small { font-size: 80%; }
|
||||
.alert-text-large { font-size: 130%; }
|
||||
.alert-text-normal { font-size: 90%; }
|
||||
.alert {
|
||||
padding:8px 35px 8px 14px; margin-bottom:18px;
|
||||
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
||||
border:1px solid #bababa;
|
||||
border-radius: 4px;
|
||||
-webkit-border-radius: 4px;
|
||||
-moz-border-radius: 4px;
|
||||
color: #555;
|
||||
background-color: #f8f8f8;
|
||||
background-position: 10px 5px;
|
||||
background-repeat: no-repeat;
|
||||
background-size: 38px;
|
||||
padding-left: 55px;
|
||||
width: 75%;
|
||||
}
|
||||
.alert-block {padding-top:14px; padding-bottom:14px}
|
||||
.alert-block > p, .alert-block > ul {margin-bottom:1em}
|
||||
.alert li {margin-top: 1em}
|
||||
.alert-block p+p {margin-top:5px}
|
||||
.alert-notice { background-image: url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_gray_notice.png); }
|
||||
.alert-summary { background-image:url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_gray_summary.png); }
|
||||
.alert-warning { background-image: url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_gray_warning.png); }
|
||||
.alert-question {background-image:url(https://cdn.rawgit.com/hplgit/doconce/master/bundled/html_images/small_gray_question.png); }
|
||||
|
||||
div { text-align: justify; text-justify: inter-word; }
|
||||
</style>
|
||||
@@ -87,16 +113,19 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
2,
|
||||
None,
|
||||
'___sec26'),
|
||||
('A simple example', 2, None, '___sec27'),
|
||||
('Set up of an RNN', 2, None, '___sec28'),
|
||||
('Solving differential equations and eigenvalue problems with '
|
||||
'RNNs',
|
||||
('Set up of an RNN', 2, None, '___sec27'),
|
||||
('A simple example', 2, None, '___sec28'),
|
||||
('An extrapolation example', 2, None, '___sec29'),
|
||||
('Formatting the Data', 2, None, '___sec30'),
|
||||
('Predicting New Points With A Trained Recurrent Neural Network',
|
||||
2,
|
||||
None,
|
||||
'___sec29'),
|
||||
('Long-Short Time Memory', 2, None, '___sec30'),
|
||||
('Autoencoders: Overarching view', 2, None, '___sec31'),
|
||||
('Simple examples of Autoencoders', 2, None, '___sec32')]}
|
||||
'___sec31'),
|
||||
('Other Things to Try', 2, None, '___sec32'),
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'___sec33')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -122,7 +151,7 @@ MathJax.Hub.Config({
|
||||
|
||||
|
||||
|
||||
<center><h1>Week 42 Convolutional and Recurrent Neural Networks and Autoencoders</h1></center> <!-- document title -->
|
||||
<center><h1>Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks</h1></center> <!-- document title -->
|
||||
|
||||
<p>
|
||||
<!-- author(s): Morten Hjorth-Jensen -->
|
||||
@@ -138,7 +167,7 @@ MathJax.Hub.Config({
|
||||
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
|
||||
<br>
|
||||
<p>
|
||||
<center><h4>Oct 15, 2020</h4></center> <!-- date -->
|
||||
<center><h4>Oct 16, 2020</h4></center> <!-- date -->
|
||||
<br>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
@@ -146,12 +175,24 @@ MathJax.Hub.Config({
|
||||
<h2 id="___sec0">Plan for week 42 </h2>
|
||||
|
||||
<ul>
|
||||
<li> Thursday: Convolutional Neural Networks and examples</li>
|
||||
<li> Friday: Recurrent Neural Networks and Autoencoders</li>
|
||||
<li> Thursday: Convolutional Neural Networks and examples. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober15.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a></li>
|
||||
<li> Friday: Recurrent Neural Networks. <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage" target="_blank">Video of Lecture</a></li>
|
||||
</ul>
|
||||
|
||||
Reading suggestions for both days: <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf" target="_blank">Aurelien Geron's chapters 13 and 14</a>. Autoencoders are discussed in chapter 15 of Geron's text.
|
||||
|
||||
<p>
|
||||
<div class="alert alert-block alert-block alert-text-normal">
|
||||
<b>Excellent lectures on CNNs and RNNs.</b>
|
||||
<p>
|
||||
|
||||
<ul>
|
||||
<li> <a href="https://www.youtube.com/watch?v=iaSUYvmCekI&ab_channel=AlexanderAmini" target="_blank">Video on Convolutional Neural Networks from MIT</a></li>
|
||||
<li> <a href="https://www.youtube.com/watch?v=SEnXr6v2ifU&ab_channel=AlexanderAmini" target="_blank">Video on Recurrent Neural Networks from MIT</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
@@ -810,7 +851,18 @@ systems such as automatic translation and speech-to-text.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec27">A simple example </h2>
|
||||
<h2 id="___sec27">Set up of an RNN </h2>
|
||||
|
||||
<p>
|
||||
See the <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesOctober16.pdf" target="_blank">handwritten notes</a> and the <a href="https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage" target="_blank">video from the lecture of October 16</a>.
|
||||
|
||||
<p>
|
||||
More text will be added later.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">A simple example </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -887,85 +939,585 @@ plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec28">Set up of an RNN </h2>
|
||||
<h2 id="___sec29">An extrapolation example </h2>
|
||||
|
||||
<p>
|
||||
The figure here displays a simple example of an RNN, with inputs \( x_t \)
|
||||
at a given time \( t \) and outputs \( y_t \). Introducing time as a variable
|
||||
offers an intutitive way of understanding these networks. In addition
|
||||
to the inputs \( x_t \), the layer at a time \( t \) receives also as input
|
||||
the output from the previous layer \( t-1 \), that is \( y_{t1} \).
|
||||
The following code provides an example of how recurrent neural
|
||||
networks can be used to extrapolate to unknown values of physics data
|
||||
sets. Specifically, the data sets used in this program come from
|
||||
a quantum mechanical many-body calculation of energies as functions of the number of particles.
|
||||
|
||||
<p>
|
||||
This means also that we need to have weights that link both the inputs
|
||||
\( x_t \) to the outputs \( y_t \) as well as weights that link the output
|
||||
from the previous time \( y_{t-1} \) and \( y_t \). The figure here shows an
|
||||
example of a simple RNN.
|
||||
|
||||
<p>
|
||||
More material will be added here.
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># For matrices and calculations</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #408080; font-style: italic"># For machine learning (backend for keras)</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
|
||||
<span style="color: #408080; font-style: italic"># User-friendly machine learning library</span>
|
||||
<span style="color: #408080; font-style: italic"># Front end for TensorFlow</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span>
|
||||
<span style="color: #408080; font-style: italic"># Different methods from Keras needed to create an RNN</span>
|
||||
<span style="color: #408080; font-style: italic"># This is not necessary but it shortened function calls </span>
|
||||
<span style="color: #408080; font-style: italic"># that need to be used in the code.</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> datasets, layers, models
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Input
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> regularizers
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.models</span> <span style="color: #008000; font-weight: bold">import</span> Model, Sequential
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Dense, SimpleRNN, LSTM, GRU
|
||||
<span style="color: #408080; font-style: italic"># For timing the code</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">timeit</span> <span style="color: #008000; font-weight: bold">import</span> default_timer <span style="color: #008000; font-weight: bold">as</span> timer
|
||||
<span style="color: #408080; font-style: italic"># For plotting</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># The data set</span>
|
||||
datatype<span style="color: #666666">=</span><span style="color: #BA2121">'VaryDimension'</span>
|
||||
X_tot <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(<span style="color: #666666">2</span>, <span style="color: #666666">42</span>, <span style="color: #666666">2</span>)
|
||||
y_tot <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([<span style="color: #666666">-0.03077640549</span>, <span style="color: #666666">-0.08336233266</span>, <span style="color: #666666">-0.1446729567</span>, <span style="color: #666666">-0.2116753732</span>, <span style="color: #666666">-0.2830637392</span>, <span style="color: #666666">-0.3581341341</span>, <span style="color: #666666">-0.436462435</span>, <span style="color: #666666">-0.5177783846</span>,
|
||||
<span style="color: #666666">-0.6019067271</span>, <span style="color: #666666">-0.6887363571</span>, <span style="color: #666666">-0.7782028952</span>, <span style="color: #666666">-0.8702784034</span>, <span style="color: #666666">-0.9649652536</span>, <span style="color: #666666">-1.062292565</span>, <span style="color: #666666">-1.16231451</span>,
|
||||
<span style="color: #666666">-1.265109911</span>, <span style="color: #666666">-1.370782966</span>, <span style="color: #666666">-1.479465113</span>, <span style="color: #666666">-1.591317992</span>, <span style="color: #666666">-1.70653767</span>])
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec29">Solving differential equations and eigenvalue problems with RNNs </h2>
|
||||
<h2 id="___sec30">Formatting the Data </h2>
|
||||
|
||||
<p>
|
||||
In our discussions of ordinary differential equations and partial
|
||||
differential equations using neural networks. Here we will discuss how
|
||||
we can solve say ordinary differential equations and eigenvalue
|
||||
problems using RNNs. Eigenvalue problems can be solved using RNNs by
|
||||
rewriting such a problems as a non-linear differential equation.
|
||||
The way the recurrent neural networks are trained in this program
|
||||
differs from how machine learning algorithms are usually trained.
|
||||
Typically a machine learning algorithm is trained by learning the
|
||||
relationship between the x data and the y data. In this program, the
|
||||
recurrent neural network will be trained to recognize the relationship
|
||||
in a sequence of y values. This is type of data formatting is
|
||||
typically used time series forcasting, but it can also be used in any
|
||||
extrapolation (time series forecasting is just a specific type of
|
||||
extrapolation along the time axis). This method of data formatting
|
||||
does not use the x data and assumes that the y data are evenly spaced.
|
||||
|
||||
<p>
|
||||
Instead of starting with a well-known ordinary differential equation,
|
||||
we start directly with an eigenvaule problem.
|
||||
For a standard machine learning algorithm, the training data has the
|
||||
form of (x,y) so the machine learning algorithm learns to assiciate a
|
||||
y value with a given x value. This is useful when the test data has x
|
||||
values within the same range as the training data. However, for this
|
||||
application, the x values of the test data are outside of the x values
|
||||
of the training data and the traditional method of training a machine
|
||||
learning algorithm does not work as well. For this reason, the
|
||||
recurrent neural network is trained on sequences of y values of the
|
||||
form ((y1, y2), y3), so that the network is concerned with learning
|
||||
the pattern of the y data and not the relation between the x and y
|
||||
data. As long as the pattern of y data outside of the training region
|
||||
stays relatively stable compared to what was inside the training
|
||||
region, this method of training can produce accurate extrapolations to
|
||||
y values far removed from the training data set.
|
||||
|
||||
<p>
|
||||
<!-- -->
|
||||
<!-- The idea behind formatting the data in this way comes from [this resource](https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/) and [this one](https://fairyonice.github.io/Understand-Keras%27s-RNN-behind-the-scenes-with-a-sin-wave-example.html). -->
|
||||
<!-- -->
|
||||
<!-- The following method takes in a y data set and formats it so the "x data" are of the form (y1, y2) and the "y data" are of the form y3, with extra brackets added in to make the resulting arrays compatable with both Keras and Tensorflow. -->
|
||||
<!-- -->
|
||||
<!-- Note: Using a sequence length of two is not required for time series forecasting so any lenght of sequence could be used (for example instead of ((y1, y2) y3) you could change the length of sequence to be 4 and the resulting data points would have the form ((y1, y2, y3, y4), y5)). While the following method can be used to create a data set of any sequence length, the remainder of the code expects the length of sequence to be 2. This is because the data sets are very small and the higher the lenght of the sequence the less resulting data points. -->
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #408080; font-style: italic"># FORMAT_DATA</span>
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">format_data</span>(data, length_of_sequence <span style="color: #666666">=</span> <span style="color: #666666">2</span>):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Inputs:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> data(a numpy array): the data that will be the inputs to the recurrent neural</span>
|
||||
<span style="color: #BA2121; font-style: italic"> network</span>
|
||||
<span style="color: #BA2121; font-style: italic"> length_of_sequence (an int): the number of elements in one iteration of the</span>
|
||||
<span style="color: #BA2121; font-style: italic"> sequence patter. For a function approximator use length_of_sequence = 2.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> rnn_input (a 3D numpy array): the input data for the recurrent neural network. Its</span>
|
||||
<span style="color: #BA2121; font-style: italic"> dimensions are length of data - length of sequence, length of sequence, </span>
|
||||
<span style="color: #BA2121; font-style: italic"> dimnsion of data</span>
|
||||
<span style="color: #BA2121; font-style: italic"> rnn_output (a numpy array): the training data for the neural network</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Formats data to be used in a recurrent neural network.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
|
||||
X, Y <span style="color: #666666">=</span> [], []
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(data)<span style="color: #666666">-</span>length_of_sequence):
|
||||
<span style="color: #408080; font-style: italic"># Get the next length_of_sequence elements</span>
|
||||
a <span style="color: #666666">=</span> data[i:i<span style="color: #666666">+</span>length_of_sequence]
|
||||
<span style="color: #408080; font-style: italic"># Get the element that immediately follows that</span>
|
||||
b <span style="color: #666666">=</span> data[i<span style="color: #666666">+</span>length_of_sequence]
|
||||
<span style="color: #408080; font-style: italic"># Reshape so that each data point is contained in its own array</span>
|
||||
a <span style="color: #666666">=</span> np<span style="color: #666666">.</span>reshape (a, (<span style="color: #008000">len</span>(a), <span style="color: #666666">1</span>))
|
||||
X<span style="color: #666666">.</span>append(a)
|
||||
Y<span style="color: #666666">.</span>append(b)
|
||||
rnn_input <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(X)
|
||||
rnn_output <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(Y)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> rnn_input, rnn_output
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># ## Defining the Recurrent Neural Network Using Keras</span>
|
||||
<span style="color: #408080; font-style: italic"># </span>
|
||||
<span style="color: #408080; font-style: italic"># The following method defines a simple recurrent neural network in keras consisting of one input layer, one hidden layer, and one output layer.</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">rnn</span>(length_of_sequences, batch_size <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>, stateful <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">False</span>):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Inputs:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #BA2121; font-style: italic"> when the data is formatted</span>
|
||||
<span style="color: #BA2121; font-style: italic"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #BA2121; font-style: italic"> method</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Builds and compiles a recurrent neural network with one hidden layer and returns the model.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
<span style="color: #408080; font-style: italic"># Number of neurons in the input and output layers</span>
|
||||
in_out_neurons <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||||
<span style="color: #408080; font-style: italic"># Number of neurons in the hidden layer</span>
|
||||
hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">200</span>
|
||||
<span style="color: #408080; font-style: italic"># Define the input layer</span>
|
||||
inp <span style="color: #666666">=</span> Input(batch_shape<span style="color: #666666">=</span>(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #408080; font-style: italic"># Define the hidden layer as a simple RNN layer with a set number of neurons and add it to </span>
|
||||
<span style="color: #408080; font-style: italic"># the network immediately after the input layer</span>
|
||||
rnn <span style="color: #666666">=</span> SimpleRNN(hidden_neurons,
|
||||
return_sequences<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||||
stateful <span style="color: #666666">=</span> stateful,
|
||||
name<span style="color: #666666">=</span><span style="color: #BA2121">"RNN"</span>)(inp)
|
||||
<span style="color: #408080; font-style: italic"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||||
<span style="color: #408080; font-style: italic">#and add it to the network immediately after the hidden layer.</span>
|
||||
dens <span style="color: #666666">=</span> Dense(in_out_neurons,name<span style="color: #666666">=</span><span style="color: #BA2121">"dense"</span>)(rnn)
|
||||
<span style="color: #408080; font-style: italic"># Create the machine learning model starting with the input layer and ending with the </span>
|
||||
<span style="color: #408080; font-style: italic"># output layer</span>
|
||||
model <span style="color: #666666">=</span> Model(inputs<span style="color: #666666">=</span>[inp],outputs<span style="color: #666666">=</span>[dens])
|
||||
<span style="color: #408080; font-style: italic"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||||
<span style="color: #408080; font-style: italic"># function and an Adams optimizer.</span>
|
||||
model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">"mean_squared_error"</span>, optimizer<span style="color: #666666">=</span><span style="color: #BA2121">"adam"</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec30">Long-Short Time Memory </h2>
|
||||
<h2 id="___sec31">Predicting New Points With A Trained Recurrent Neural Network </h2>
|
||||
|
||||
<p>
|
||||
Discussions about dynamic unrolling through time. discuss memory cells, input and output
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">test_rnn</span> (x1, y_test, plot_min, plot_max):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Inputs:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> x1 (a list or numpy array): The complete x component of the data set</span>
|
||||
<span style="color: #BA2121; font-style: italic"> y_test (a list or numpy array): The complete y component of the data set</span>
|
||||
<span style="color: #BA2121; font-style: italic"> plot_min (an int or float): the smallest x value used in the training data</span>
|
||||
<span style="color: #BA2121; font-style: italic"> plot_max (an int or float): the largest x valye used in the training data</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> None.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Uses a trained recurrent neural network model to predict future points in the </span>
|
||||
<span style="color: #BA2121; font-style: italic"> series. Computes the MSE of the predicted data set from the true data set, saves</span>
|
||||
<span style="color: #BA2121; font-style: italic"> the predicted data set to a csv file, and plots the predicted and true data sets w</span>
|
||||
<span style="color: #BA2121; font-style: italic"> while also displaying the data range used for training.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
<span style="color: #408080; font-style: italic"># Add the training data as the first dim points in the predicted data array as these</span>
|
||||
<span style="color: #408080; font-style: italic"># are known values.</span>
|
||||
y_pred <span style="color: #666666">=</span> y_test[:dim]<span style="color: #666666">.</span>tolist()
|
||||
<span style="color: #408080; font-style: italic"># Generate the first input to the trained recurrent neural network using the last two </span>
|
||||
<span style="color: #408080; font-style: italic"># points of the training data. Based on how the network was trained this means that it</span>
|
||||
<span style="color: #408080; font-style: italic"># will predict the first point in the data set after the training data. All of the </span>
|
||||
<span style="color: #408080; font-style: italic"># brackets are necessary for Tensorflow.</span>
|
||||
next_input <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[[y_test[dim<span style="color: #666666">-2</span>]], [y_test[dim<span style="color: #666666">-1</span>]]]])
|
||||
<span style="color: #408080; font-style: italic"># Save the very last point in the training data set. This will be used later.</span>
|
||||
last <span style="color: #666666">=</span> [y_test[dim<span style="color: #666666">-1</span>]]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Iterate until the complete data set is created.</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span> (dim, <span style="color: #008000">len</span>(y_test)):
|
||||
<span style="color: #408080; font-style: italic"># Predict the next point in the data set using the previous two points.</span>
|
||||
<span style="color: #008000">next</span> <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(next_input)
|
||||
<span style="color: #408080; font-style: italic"># Append just the number of the predicted data set</span>
|
||||
y_pred<span style="color: #666666">.</span>append(<span style="color: #008000">next</span>[<span style="color: #666666">0</span>][<span style="color: #666666">0</span>])
|
||||
<span style="color: #408080; font-style: italic"># Create the input that will be used to predict the next data point in the data set.</span>
|
||||
next_input <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[last, <span style="color: #008000">next</span>[<span style="color: #666666">0</span>]]], dtype<span style="color: #666666">=</span>np<span style="color: #666666">.</span>float64)
|
||||
last <span style="color: #666666">=</span> <span style="color: #008000">next</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Print the mean squared error between the known data set and the predicted data set.</span>
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'MSE: '</span>, np<span style="color: #666666">.</span>square(np<span style="color: #666666">.</span>subtract(y_test, y_pred))<span style="color: #666666">.</span>mean())
|
||||
<span style="color: #408080; font-style: italic"># Save the predicted data set as a csv file for later use</span>
|
||||
name <span style="color: #666666">=</span> datatype <span style="color: #666666">+</span> <span style="color: #BA2121">'Predicted'</span><span style="color: #666666">+</span><span style="color: #008000">str</span>(dim)<span style="color: #666666">+</span><span style="color: #BA2121">'.csv'</span>
|
||||
np<span style="color: #666666">.</span>savetxt(name, y_pred, delimiter<span style="color: #666666">=</span><span style="color: #BA2121">','</span>)
|
||||
<span style="color: #408080; font-style: italic"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||||
<span style="color: #408080; font-style: italic"># for the training data.</span>
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots()
|
||||
ax<span style="color: #666666">.</span>plot(x1, y_test, label<span style="color: #666666">=</span><span style="color: #BA2121">"true"</span>, linewidth<span style="color: #666666">=3</span>)
|
||||
ax<span style="color: #666666">.</span>plot(x1, y_pred, <span style="color: #BA2121">'g-.'</span>,label<span style="color: #666666">=</span><span style="color: #BA2121">"predicted"</span>, linewidth<span style="color: #666666">=4</span>)
|
||||
ax<span style="color: #666666">.</span>legend()
|
||||
<span style="color: #408080; font-style: italic"># Created a red region to represent the points used in the training data.</span>
|
||||
ax<span style="color: #666666">.</span>axvspan(plot_min, plot_max, alpha<span style="color: #666666">=0.25</span>, color<span style="color: #666666">=</span><span style="color: #BA2121">'red'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #008000; font-weight: bold">assert</span> <span style="color: #008000">len</span>(X_tot) <span style="color: #666666">==</span> <span style="color: #008000">len</span>(y_tot)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># This is the number of points that will be used in as the training data</span>
|
||||
dim<span style="color: #666666">=12</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Separate the training data from the whole data set</span>
|
||||
X_train <span style="color: #666666">=</span> X_tot[:dim]
|
||||
y_train <span style="color: #666666">=</span> y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training <span style="color: #666666">=</span> format_data(y_train, <span style="color: #666666">2</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #408080; font-style: italic"># machine learning model</span>
|
||||
model <span style="color: #666666">=</span> rnn(length_of_sequences <span style="color: #666666">=</span> rnn_input<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>])
|
||||
model<span style="color: #666666">.</span>summary()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Start the timer. Want to time training+testing</span>
|
||||
start <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #408080; font-style: italic"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #408080; font-style: italic"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist <span style="color: #666666">=</span> model<span style="color: #666666">.</span>fit(rnn_input, rnn_training, batch_size<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, epochs<span style="color: #666666">=150</span>,
|
||||
verbose<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,validation_split<span style="color: #666666">=0.05</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> label <span style="color: #AA22FF; font-weight: bold">in</span> [<span style="color: #BA2121">"loss"</span>,<span style="color: #BA2121">"val_loss"</span>]:
|
||||
plt<span style="color: #666666">.</span>plot(hist<span style="color: #666666">.</span>history[label],label<span style="color: #666666">=</span>label)
|
||||
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"loss"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"epoch"</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"The final validation loss: </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(hist<span style="color: #666666">.</span>history[<span style="color: #BA2121">"val_loss"</span>][<span style="color: #666666">-1</span>]))
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #666666">0</span>], X_tot[dim<span style="color: #666666">-1</span>])
|
||||
<span style="color: #408080; font-style: italic"># Stop the timer and calculate the total time needed.</span>
|
||||
end <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Time: '</span>, end<span style="color: #666666">-</span>start)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec31">Autoencoders: Overarching view </h2>
|
||||
<h2 id="___sec32">Other Things to Try </h2>
|
||||
|
||||
<p>
|
||||
Autoencoders are artificial neural networks capable of learning
|
||||
efficient representations of the input data (these representations are called codings) without
|
||||
any supervision (i.e., the training set is unlabeled). These codings
|
||||
typically have a much lower dimensionality than the input data, making
|
||||
autoencoders useful for dimensionality reduction.
|
||||
Changing the size of the recurrent neural network and its parameters
|
||||
can drastically change the results you get from the model. The below
|
||||
code takes the simple recurrent neural network from above and adds a
|
||||
second hidden layer, changes the number of neurons in the hidden
|
||||
layer, and explicitly declares the activation function of the hidden
|
||||
layers to be a sigmoid function. The loss function and optimizer can
|
||||
also be changed but are kept the same as the above network. These
|
||||
parameters can be tuned to provide the optimal result from the
|
||||
network. For some ideas on how to improve the performance of a
|
||||
<a href="https://danijar.com/tips-for-training-recurrent-neural-networks" target="_blank">recurrent neural network</a>.
|
||||
|
||||
<p>
|
||||
More importantly, autoencoders act as powerful feature detectors, and
|
||||
they can be used for unsupervised pretraining of deep neural networks.
|
||||
|
||||
<p>
|
||||
Lastly, they are capable of randomly generating new data that looks
|
||||
very similar to the training data; this is called a generative
|
||||
model. For example, you could train an autoencoder on pictures of
|
||||
faces, and it would then be able to generate new faces. Surprisingly,
|
||||
autoencoders work by simply learning to copy their inputs to their
|
||||
outputs. This may sound like a trivial task, but we will see that
|
||||
constraining the network in various ways can make it rather
|
||||
difficult. For example, you can limit the size of the internal
|
||||
representation, or you can add noise to the inputs and train the
|
||||
network to recover the original inputs. These constraints prevent the
|
||||
autoencoder from trivially copying the inputs directly to the outputs,
|
||||
which forces it to learn efficient ways of representing the data. In
|
||||
short, the codings are byproducts of the autoencoder’s attempt to
|
||||
learn the identity function under some constraints.
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">rnn_2layers</span>(length_of_sequences, batch_size <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>, stateful <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">False</span>):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Inputs:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #BA2121; font-style: italic"> when the data is formatted</span>
|
||||
<span style="color: #BA2121; font-style: italic"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #BA2121; font-style: italic"> method</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Builds and compiles a recurrent neural network with two hidden layers and returns the model.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
<span style="color: #408080; font-style: italic"># Number of neurons in the input and output layers</span>
|
||||
in_out_neurons <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||||
<span style="color: #408080; font-style: italic"># Number of neurons in the hidden layer, increased from the first network</span>
|
||||
hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">500</span>
|
||||
<span style="color: #408080; font-style: italic"># Define the input layer</span>
|
||||
inp <span style="color: #666666">=</span> Input(batch_shape<span style="color: #666666">=</span>(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #408080; font-style: italic"># Create two hidden layers instead of one hidden layer. Explicitly set the activation</span>
|
||||
<span style="color: #408080; font-style: italic"># function to be the sigmoid function (the default value is hyperbolic tangent)</span>
|
||||
rnn1 <span style="color: #666666">=</span> SimpleRNN(hidden_neurons,
|
||||
return_sequences<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, <span style="color: #408080; font-style: italic"># This needs to be True if another hidden layer is to follow</span>
|
||||
stateful <span style="color: #666666">=</span> stateful, activation <span style="color: #666666">=</span> <span style="color: #BA2121">'sigmoid'</span>,
|
||||
name<span style="color: #666666">=</span><span style="color: #BA2121">"RNN1"</span>)(inp)
|
||||
rnn2 <span style="color: #666666">=</span> SimpleRNN(hidden_neurons,
|
||||
return_sequences<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>, activation <span style="color: #666666">=</span> <span style="color: #BA2121">'sigmoid'</span>,
|
||||
stateful <span style="color: #666666">=</span> stateful,
|
||||
name<span style="color: #666666">=</span><span style="color: #BA2121">"RNN2"</span>)(rnn1)
|
||||
<span style="color: #408080; font-style: italic"># Define the output layer as a dense neural network layer (standard neural network layer)</span>
|
||||
<span style="color: #408080; font-style: italic">#and add it to the network immediately after the hidden layer.</span>
|
||||
dens <span style="color: #666666">=</span> Dense(in_out_neurons,name<span style="color: #666666">=</span><span style="color: #BA2121">"dense"</span>)(rnn2)
|
||||
<span style="color: #408080; font-style: italic"># Create the machine learning model starting with the input layer and ending with the </span>
|
||||
<span style="color: #408080; font-style: italic"># output layer</span>
|
||||
model <span style="color: #666666">=</span> Model(inputs<span style="color: #666666">=</span>[inp],outputs<span style="color: #666666">=</span>[dens])
|
||||
<span style="color: #408080; font-style: italic"># Compile the machine learning model using the mean squared error function as the loss </span>
|
||||
<span style="color: #408080; font-style: italic"># function and an Adams optimizer.</span>
|
||||
model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">"mean_squared_error"</span>, optimizer<span style="color: #666666">=</span><span style="color: #BA2121">"adam"</span>)
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #008000; font-weight: bold">assert</span> <span style="color: #008000">len</span>(X_tot) <span style="color: #666666">==</span> <span style="color: #008000">len</span>(y_tot)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># This is the number of points that will be used in as the training data</span>
|
||||
dim<span style="color: #666666">=12</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Separate the training data from the whole data set</span>
|
||||
X_train <span style="color: #666666">=</span> X_tot[:dim]
|
||||
y_train <span style="color: #666666">=</span> y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training <span style="color: #666666">=</span> format_data(y_train, <span style="color: #666666">2</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #408080; font-style: italic"># machine learning model</span>
|
||||
model <span style="color: #666666">=</span> rnn_2layers(length_of_sequences <span style="color: #666666">=</span> <span style="color: #666666">2</span>)
|
||||
model<span style="color: #666666">.</span>summary()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Start the timer. Want to time training+testing</span>
|
||||
start <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #408080; font-style: italic"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #408080; font-style: italic"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist <span style="color: #666666">=</span> model<span style="color: #666666">.</span>fit(rnn_input, rnn_training, batch_size<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, epochs<span style="color: #666666">=150</span>,
|
||||
verbose<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,validation_split<span style="color: #666666">=0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #408080; font-style: italic"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #408080; font-style: italic"># being overtrained.</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> label <span style="color: #AA22FF; font-weight: bold">in</span> [<span style="color: #BA2121">"loss"</span>,<span style="color: #BA2121">"val_loss"</span>]:
|
||||
plt<span style="color: #666666">.</span>plot(hist<span style="color: #666666">.</span>history[label],label<span style="color: #666666">=</span>label)
|
||||
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"loss"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"epoch"</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"The final validation loss: </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(hist<span style="color: #666666">.</span>history[<span style="color: #BA2121">"val_loss"</span>][<span style="color: #666666">-1</span>]))
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #666666">0</span>], X_tot[dim<span style="color: #666666">-1</span>])
|
||||
<span style="color: #408080; font-style: italic"># Stop the timer and calculate the total time needed.</span>
|
||||
end <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Time: '</span>, end<span style="color: #666666">-</span>start)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec32">Simple examples of Autoencoders </h2>
|
||||
<h2 id="___sec33">Other Types of Recurrent Neural Networks </h2>
|
||||
|
||||
<p>
|
||||
Besides a simple recurrent neural network layer, there are two other
|
||||
commonly used types of recurrent neural network layers: Long Short
|
||||
Term Memory (LSTM) and Gated Recurrent Unit (GRU). For a short
|
||||
introduction to these layers see <a href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b" target="_blank"><tt>https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</tt></a>
|
||||
and <a href="https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b" target="_blank"><tt>https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b</tt></a>.
|
||||
|
||||
<p>
|
||||
The first network created below is similar to the previous network,
|
||||
but it replaces the SimpleRNN layers with LSTM layers. The second
|
||||
network below has two hidden layers made up of GRUs, which are
|
||||
preceeded by two dense (feeddorward) neural network layers. These
|
||||
dense layers "preprocess" the data before it reaches the recurrent
|
||||
layers. This architecture has been shown to improve the performance
|
||||
of recurrent neural networks (see the link above and also
|
||||
<a href="https://arxiv.org/pdf/1807.02857.pdf" target="_blank"><tt>https://arxiv.org/pdf/1807.02857.pdf</tt></a>.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">lstm_2layers</span>(length_of_sequences, batch_size <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>, stateful <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">False</span>):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Inputs:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #BA2121; font-style: italic"> when the data is formatted</span>
|
||||
<span style="color: #BA2121; font-style: italic"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #BA2121; font-style: italic"> method</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Builds and compiles a recurrent neural network with two LSTM hidden layers and returns the model.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
<span style="color: #408080; font-style: italic"># Number of neurons on the input/output layer and the number of neurons in the hidden layer</span>
|
||||
in_out_neurons <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||||
hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">250</span>
|
||||
<span style="color: #408080; font-style: italic"># Input Layer</span>
|
||||
inp <span style="color: #666666">=</span> Input(batch_shape<span style="color: #666666">=</span>(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #408080; font-style: italic"># Hidden layers (in this case they are LSTM layers instead if SimpleRNN layers)</span>
|
||||
rnn<span style="color: #666666">=</span> LSTM(hidden_neurons,
|
||||
return_sequences<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,
|
||||
stateful <span style="color: #666666">=</span> stateful,
|
||||
name<span style="color: #666666">=</span><span style="color: #BA2121">"RNN"</span>, use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, activation<span style="color: #666666">=</span><span style="color: #BA2121">'tanh'</span>)(inp)
|
||||
rnn1 <span style="color: #666666">=</span> LSTM(hidden_neurons,
|
||||
return_sequences<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||||
stateful <span style="color: #666666">=</span> stateful,
|
||||
name<span style="color: #666666">=</span><span style="color: #BA2121">"RNN1"</span>, use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, activation<span style="color: #666666">=</span><span style="color: #BA2121">'tanh'</span>)(rnn)
|
||||
<span style="color: #408080; font-style: italic"># Output layer</span>
|
||||
dens <span style="color: #666666">=</span> Dense(in_out_neurons,name<span style="color: #666666">=</span><span style="color: #BA2121">"dense"</span>)(rnn1)
|
||||
<span style="color: #408080; font-style: italic"># Define the midel</span>
|
||||
model <span style="color: #666666">=</span> Model(inputs<span style="color: #666666">=</span>[inp],outputs<span style="color: #666666">=</span>[dens])
|
||||
<span style="color: #408080; font-style: italic"># Compile the model</span>
|
||||
model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">'mean_squared_error'</span>, optimizer<span style="color: #666666">=</span><span style="color: #BA2121">'adam'</span>)
|
||||
<span style="color: #408080; font-style: italic"># Return the model</span>
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">dnn2_gru2</span>(length_of_sequences, batch_size <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>, stateful <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">False</span>):
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Inputs:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> length_of_sequences (an int): the number of y values in "x data". This is determined</span>
|
||||
<span style="color: #BA2121; font-style: italic"> when the data is formatted</span>
|
||||
<span style="color: #BA2121; font-style: italic"> batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||||
<span style="color: #BA2121; font-style: italic"> model (a Keras model): The recurrent neural network that is built and compiled by this</span>
|
||||
<span style="color: #BA2121; font-style: italic"> method</span>
|
||||
<span style="color: #BA2121; font-style: italic"> Builds and compiles a recurrent neural network with four hidden layers (two dense followed by</span>
|
||||
<span style="color: #BA2121; font-style: italic"> two GRU layers) and returns the model.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
<span style="color: #408080; font-style: italic"># Number of neurons on the input/output layers and hidden layers</span>
|
||||
in_out_neurons <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||||
hidden_neurons <span style="color: #666666">=</span> <span style="color: #666666">250</span>
|
||||
<span style="color: #408080; font-style: italic"># Input layer</span>
|
||||
inp <span style="color: #666666">=</span> Input(batch_shape<span style="color: #666666">=</span>(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
<span style="color: #408080; font-style: italic"># Hidden Dense (feedforward) layers</span>
|
||||
dnn <span style="color: #666666">=</span> Dense(hidden_neurons<span style="color: #666666">/2</span>, activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>, name<span style="color: #666666">=</span><span style="color: #BA2121">'dnn'</span>)(inp)
|
||||
dnn1 <span style="color: #666666">=</span> Dense(hidden_neurons<span style="color: #666666">/2</span>, activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>, name<span style="color: #666666">=</span><span style="color: #BA2121">'dnn1'</span>)(dnn)
|
||||
<span style="color: #408080; font-style: italic"># Hidden GRU layers</span>
|
||||
rnn1 <span style="color: #666666">=</span> GRU(hidden_neurons,
|
||||
return_sequences<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,
|
||||
stateful <span style="color: #666666">=</span> stateful,
|
||||
name<span style="color: #666666">=</span><span style="color: #BA2121">"RNN1"</span>, use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)(dnn1)
|
||||
rnn <span style="color: #666666">=</span> GRU(hidden_neurons,
|
||||
return_sequences<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||||
stateful <span style="color: #666666">=</span> stateful,
|
||||
name<span style="color: #666666">=</span><span style="color: #BA2121">"RNN"</span>, use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)(rnn1)
|
||||
<span style="color: #408080; font-style: italic"># Output layer</span>
|
||||
dens <span style="color: #666666">=</span> Dense(in_out_neurons,name<span style="color: #666666">=</span><span style="color: #BA2121">"dense"</span>)(rnn)
|
||||
<span style="color: #408080; font-style: italic"># Define the model</span>
|
||||
model <span style="color: #666666">=</span> Model(inputs<span style="color: #666666">=</span>[inp],outputs<span style="color: #666666">=</span>[dens])
|
||||
<span style="color: #408080; font-style: italic"># Compile the mdoel</span>
|
||||
model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">'mean_squared_error'</span>, optimizer<span style="color: #666666">=</span><span style="color: #BA2121">'adam'</span>)
|
||||
<span style="color: #408080; font-style: italic"># Return the model</span>
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #008000; font-weight: bold">assert</span> <span style="color: #008000">len</span>(X_tot) <span style="color: #666666">==</span> <span style="color: #008000">len</span>(y_tot)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># This is the number of points that will be used in as the training data</span>
|
||||
dim<span style="color: #666666">=12</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Separate the training data from the whole data set</span>
|
||||
X_train <span style="color: #666666">=</span> X_tot[:dim]
|
||||
y_train <span style="color: #666666">=</span> y_tot[:dim]
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Generate the training data for the RNN, using a sequence of 2</span>
|
||||
rnn_input, rnn_training <span style="color: #666666">=</span> format_data(y_train, <span style="color: #666666">2</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #408080; font-style: italic"># machine learning model</span>
|
||||
<span style="color: #408080; font-style: italic"># Change the method name to reflect which network you want to use</span>
|
||||
model <span style="color: #666666">=</span> dnn2_gru2(length_of_sequences <span style="color: #666666">=</span> <span style="color: #666666">2</span>)
|
||||
model<span style="color: #666666">.</span>summary()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Start the timer. Want to time training+testing</span>
|
||||
start <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #408080; font-style: italic"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #408080; font-style: italic"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist <span style="color: #666666">=</span> model<span style="color: #666666">.</span>fit(rnn_input, rnn_training, batch_size<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, epochs<span style="color: #666666">=150</span>,
|
||||
verbose<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,validation_split<span style="color: #666666">=0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #408080; font-style: italic"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #408080; font-style: italic"># being overtrained.</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> label <span style="color: #AA22FF; font-weight: bold">in</span> [<span style="color: #BA2121">"loss"</span>,<span style="color: #BA2121">"val_loss"</span>]:
|
||||
plt<span style="color: #666666">.</span>plot(hist<span style="color: #666666">.</span>history[label],label<span style="color: #666666">=</span>label)
|
||||
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"loss"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"epoch"</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"The final validation loss: </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(hist<span style="color: #666666">.</span>history[<span style="color: #BA2121">"val_loss"</span>][<span style="color: #666666">-1</span>]))
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Use the trained neural network to predict more points of the data set</span>
|
||||
test_rnn(X_tot, y_tot, X_tot[<span style="color: #666666">0</span>], X_tot[dim<span style="color: #666666">-1</span>])
|
||||
<span style="color: #408080; font-style: italic"># Stop the timer and calculate the total time needed.</span>
|
||||
end <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Time: '</span>, end<span style="color: #666666">-</span>start)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># ### Training Recurrent Neural Networks in the Standard Way (i.e. learning the relationship between the X and Y data)</span>
|
||||
<span style="color: #408080; font-style: italic"># </span>
|
||||
<span style="color: #408080; font-style: italic"># Finally, comparing the performace of a recurrent neural network using the standard data formatting to the performance of the network with time sequence data formatting shows the benefit of this type of data formatting with extrapolation.</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Check to make sure the data set is complete</span>
|
||||
<span style="color: #008000; font-weight: bold">assert</span> <span style="color: #008000">len</span>(X_tot) <span style="color: #666666">==</span> <span style="color: #008000">len</span>(y_tot)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># This is the number of points that will be used in as the training data</span>
|
||||
dim<span style="color: #666666">=12</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Separate the training data from the whole data set</span>
|
||||
X_train <span style="color: #666666">=</span> X_tot[:dim]
|
||||
y_train <span style="color: #666666">=</span> y_tot[:dim]
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Reshape the data for Keras specifications</span>
|
||||
X_train <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>reshape((dim, <span style="color: #666666">1</span>))
|
||||
y_train <span style="color: #666666">=</span> y_train<span style="color: #666666">.</span>reshape((dim, <span style="color: #666666">1</span>))
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Create a recurrent neural network in Keras and produce a summary of the </span>
|
||||
<span style="color: #408080; font-style: italic"># machine learning model</span>
|
||||
<span style="color: #408080; font-style: italic"># Set the sequence length to 1 for regular data formatting </span>
|
||||
model <span style="color: #666666">=</span> rnn(length_of_sequences <span style="color: #666666">=</span> <span style="color: #666666">1</span>)
|
||||
model<span style="color: #666666">.</span>summary()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Start the timer. Want to time training+testing</span>
|
||||
start <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #408080; font-style: italic"># Fit the model using the training data genenerated above using 150 training iterations and a 5%</span>
|
||||
<span style="color: #408080; font-style: italic"># validation split. Setting verbose to True prints information about each training iteration.</span>
|
||||
hist <span style="color: #666666">=</span> model<span style="color: #666666">.</span>fit(X_train, y_train, batch_size<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>, epochs<span style="color: #666666">=150</span>,
|
||||
verbose<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,validation_split<span style="color: #666666">=0.05</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># This section plots the training loss and the validation loss as a function of training iteration.</span>
|
||||
<span style="color: #408080; font-style: italic"># This is not required for analyzing the couple cluster data but can help determine if the network is</span>
|
||||
<span style="color: #408080; font-style: italic"># being overtrained.</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> label <span style="color: #AA22FF; font-weight: bold">in</span> [<span style="color: #BA2121">"loss"</span>,<span style="color: #BA2121">"val_loss"</span>]:
|
||||
plt<span style="color: #666666">.</span>plot(hist<span style="color: #666666">.</span>history[label],label<span style="color: #666666">=</span>label)
|
||||
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"loss"</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"epoch"</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"The final validation loss: </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(hist<span style="color: #666666">.</span>history[<span style="color: #BA2121">"val_loss"</span>][<span style="color: #666666">-1</span>]))
|
||||
plt<span style="color: #666666">.</span>legend()
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Use the trained neural network to predict the remaining data points</span>
|
||||
X_pred <span style="color: #666666">=</span> X_tot[dim:]
|
||||
X_pred <span style="color: #666666">=</span> X_pred<span style="color: #666666">.</span>reshape((<span style="color: #008000">len</span>(X_pred), <span style="color: #666666">1</span>))
|
||||
y_model <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_pred)
|
||||
y_pred <span style="color: #666666">=</span> np<span style="color: #666666">.</span>concatenate((y_tot[:dim], y_model<span style="color: #666666">.</span>flatten()))
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Plot the known data set and the predicted data set. The red box represents the region that was used</span>
|
||||
<span style="color: #408080; font-style: italic"># for the training data.</span>
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots()
|
||||
ax<span style="color: #666666">.</span>plot(X_tot, y_tot, label<span style="color: #666666">=</span><span style="color: #BA2121">"true"</span>, linewidth<span style="color: #666666">=3</span>)
|
||||
ax<span style="color: #666666">.</span>plot(X_tot, y_pred, <span style="color: #BA2121">'g-.'</span>,label<span style="color: #666666">=</span><span style="color: #BA2121">"predicted"</span>, linewidth<span style="color: #666666">=4</span>)
|
||||
ax<span style="color: #666666">.</span>legend()
|
||||
<span style="color: #408080; font-style: italic"># Created a red region to represent the points used in the training data.</span>
|
||||
ax<span style="color: #666666">.</span>axvspan(X_tot[<span style="color: #666666">0</span>], X_tot[dim], alpha<span style="color: #666666">=0.25</span>, color<span style="color: #666666">=</span><span style="color: #BA2121">'red'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Stop the timer and calculate the total time needed.</span>
|
||||
end <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Time: '</span>, end<span style="color: #666666">-</span>start)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
Binary file not shown.
+679
-604
File diff suppressed because one or more lines are too long
+556
-47
@@ -1,4 +1,4 @@
|
||||
TITLE: Week 42 Convolutional and Recurrent Neural Networks and Autoencoders
|
||||
TITLE: Week 42 Convolutional (CNN) and Recurrent (RNN) Neural Networks
|
||||
AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo & Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
|
||||
DATE: today
|
||||
|
||||
@@ -6,11 +6,16 @@ DATE: today
|
||||
!split
|
||||
===== Plan for week 42 =====
|
||||
|
||||
* Thursday: Convolutional Neural Networks and examples
|
||||
* Friday: Recurrent Neural Networks and Autoencoders
|
||||
* Thursday: Convolutional Neural Networks and examples. "Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober15.mp4?vrtx=view-as-webpage"
|
||||
* Friday: Recurrent Neural Networks. "Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage"
|
||||
|
||||
Reading suggestions for both days: "Aurelien Geron's chapters 13 and 14":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/TensorflowML.pdf". Autoencoders are discussed in chapter 15 of Geron's text.
|
||||
|
||||
!bblock Excellent lectures on CNNs and RNNs
|
||||
* "Video on Convolutional Neural Networks from MIT":"https://www.youtube.com/watch?v=iaSUYvmCekI&ab_channel=AlexanderAmini"
|
||||
* "Video on Recurrent Neural Networks from MIT":"https://www.youtube.com/watch?v=SEnXr6v2ifU&ab_channel=AlexanderAmini"
|
||||
!eblock
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -589,6 +594,16 @@ input, making them extremely useful for natural language processing
|
||||
systems such as automatic translation and speech-to-text.
|
||||
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Set up of an RNN =====
|
||||
|
||||
See the "handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesOctober16.pdf" and the "video from the lecture of October 16":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureOctober16.mp4?vrtx=view-as-webpage".
|
||||
|
||||
More text will be added later.
|
||||
|
||||
|
||||
!split
|
||||
===== A simple example =====
|
||||
|
||||
@@ -665,76 +680,570 @@ plt.show()
|
||||
|
||||
|
||||
!split
|
||||
===== Set up of an RNN =====
|
||||
===== An extrapolation example =====
|
||||
|
||||
The figure here displays a simple example of an RNN, with inputs $x_t$
|
||||
at a given time $t$ and outputs $y_t$. Introducing time as a variable
|
||||
offers an intutitive way of understanding these networks. In addition
|
||||
to the inputs $x_t$, the layer at a time $t$ receives also as input
|
||||
the output from the previous layer $t-1$, that is $y_{t1}$.
|
||||
The following code provides an example of how recurrent neural
|
||||
networks can be used to extrapolate to unknown values of physics data
|
||||
sets. Specifically, the data sets used in this program come from
|
||||
a quantum mechanical many-body calculation of energies as functions of the number of particles.
|
||||
|
||||
This means also that we need to have weights that link both the inputs
|
||||
$x_t$ to the outputs $y_t$ as well as weights that link the output
|
||||
from the previous time $y_{t-1}$ and $y_t$. The figure here shows an
|
||||
example of a simple RNN.
|
||||
|
||||
More material will be added here.
|
||||
!bc pycod
|
||||
|
||||
# For matrices and calculations
|
||||
import numpy as np
|
||||
# For machine learning (backend for keras)
|
||||
import tensorflow as tf
|
||||
# User-friendly machine learning library
|
||||
# Front end for TensorFlow
|
||||
import tensorflow.keras
|
||||
# Different methods from Keras needed to create an RNN
|
||||
# This is not necessary but it shortened function calls
|
||||
# that need to be used in the code.
|
||||
from tensorflow.keras import datasets, layers, models
|
||||
from tensorflow.keras.layers import Input
|
||||
from tensorflow.keras import regularizers
|
||||
from tensorflow.keras.models import Model, Sequential
|
||||
from tensorflow.keras.layers import Dense, SimpleRNN, LSTM, GRU
|
||||
# For timing the code
|
||||
from timeit import default_timer as timer
|
||||
# For plotting
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
# The data set
|
||||
datatype='VaryDimension'
|
||||
X_tot = np.arange(2, 42, 2)
|
||||
y_tot = np.array([-0.03077640549, -0.08336233266, -0.1446729567, -0.2116753732, -0.2830637392, -0.3581341341, -0.436462435, -0.5177783846,
|
||||
-0.6019067271, -0.6887363571, -0.7782028952, -0.8702784034, -0.9649652536, -1.062292565, -1.16231451,
|
||||
-1.265109911, -1.370782966, -1.479465113, -1.591317992, -1.70653767])
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Solving differential equations and eigenvalue problems with RNNs =====
|
||||
===== Formatting the Data =====
|
||||
|
||||
The way the recurrent neural networks are trained in this program
|
||||
differs from how machine learning algorithms are usually trained.
|
||||
Typically a machine learning algorithm is trained by learning the
|
||||
relationship between the x data and the y data. In this program, the
|
||||
recurrent neural network will be trained to recognize the relationship
|
||||
in a sequence of y values. This is type of data formatting is
|
||||
typically used time series forcasting, but it can also be used in any
|
||||
extrapolation (time series forecasting is just a specific type of
|
||||
extrapolation along the time axis). This method of data formatting
|
||||
does not use the x data and assumes that the y data are evenly spaced.
|
||||
|
||||
For a standard machine learning algorithm, the training data has the
|
||||
form of (x,y) so the machine learning algorithm learns to assiciate a
|
||||
y value with a given x value. This is useful when the test data has x
|
||||
values within the same range as the training data. However, for this
|
||||
application, the x values of the test data are outside of the x values
|
||||
of the training data and the traditional method of training a machine
|
||||
learning algorithm does not work as well. For this reason, the
|
||||
recurrent neural network is trained on sequences of y values of the
|
||||
form ((y1, y2), y3), so that the network is concerned with learning
|
||||
the pattern of the y data and not the relation between the x and y
|
||||
data. As long as the pattern of y data outside of the training region
|
||||
stays relatively stable compared to what was inside the training
|
||||
region, this method of training can produce accurate extrapolations to
|
||||
y values far removed from the training data set.
|
||||
|
||||
|
||||
#
|
||||
# The idea behind formatting the data in this way comes from [this resource](https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/) and [this one](https://fairyonice.github.io/Understand-Keras%27s-RNN-behind-the-scenes-with-a-sin-wave-example.html).
|
||||
#
|
||||
# The following method takes in a y data set and formats it so the "x data" are of the form (y1, y2) and the "y data" are of the form y3, with extra brackets added in to make the resulting arrays compatable with both Keras and Tensorflow.
|
||||
#
|
||||
# Note: Using a sequence length of two is not required for time series forecasting so any lenght of sequence could be used (for example instead of ((y1, y2) y3) you could change the length of sequence to be 4 and the resulting data points would have the form ((y1, y2, y3, y4), y5)). While the following method can be used to create a data set of any sequence length, the remainder of the code expects the length of sequence to be 2. This is because the data sets are very small and the higher the lenght of the sequence the less resulting data points.
|
||||
|
||||
In our discussions of ordinary differential equations and partial
|
||||
differential equations using neural networks. Here we will discuss how
|
||||
we can solve say ordinary differential equations and eigenvalue
|
||||
problems using RNNs. Eigenvalue problems can be solved using RNNs by
|
||||
rewriting such a problems as a non-linear differential equation.
|
||||
!bc pycod
|
||||
# FORMAT_DATA
|
||||
def format_data(data, length_of_sequence = 2):
|
||||
"""
|
||||
Inputs:
|
||||
data(a numpy array): the data that will be the inputs to the recurrent neural
|
||||
network
|
||||
length_of_sequence (an int): the number of elements in one iteration of the
|
||||
sequence patter. For a function approximator use length_of_sequence = 2.
|
||||
Returns:
|
||||
rnn_input (a 3D numpy array): the input data for the recurrent neural network. Its
|
||||
dimensions are length of data - length of sequence, length of sequence,
|
||||
dimnsion of data
|
||||
rnn_output (a numpy array): the training data for the neural network
|
||||
Formats data to be used in a recurrent neural network.
|
||||
"""
|
||||
|
||||
Instead of starting with a well-known ordinary differential equation,
|
||||
we start directly with an eigenvaule problem.
|
||||
X, Y = [], []
|
||||
for i in range(len(data)-length_of_sequence):
|
||||
# Get the next length_of_sequence elements
|
||||
a = data[i:i+length_of_sequence]
|
||||
# Get the element that immediately follows that
|
||||
b = data[i+length_of_sequence]
|
||||
# Reshape so that each data point is contained in its own array
|
||||
a = np.reshape (a, (len(a), 1))
|
||||
X.append(a)
|
||||
Y.append(b)
|
||||
rnn_input = np.array(X)
|
||||
rnn_output = np.array(Y)
|
||||
|
||||
return rnn_input, rnn_output
|
||||
|
||||
|
||||
# ## Defining the Recurrent Neural Network Using Keras
|
||||
#
|
||||
# The following method defines a simple recurrent neural network in keras consisting of one input layer, one hidden layer, and one output layer.
|
||||
|
||||
def rnn(length_of_sequences, batch_size = None, stateful = False):
|
||||
"""
|
||||
Inputs:
|
||||
length_of_sequences (an int): the number of y values in "x data". This is determined
|
||||
when the data is formatted
|
||||
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
|
||||
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
|
||||
Returns:
|
||||
model (a Keras model): The recurrent neural network that is built and compiled by this
|
||||
method
|
||||
Builds and compiles a recurrent neural network with one hidden layer and returns the model.
|
||||
"""
|
||||
# Number of neurons in the input and output layers
|
||||
in_out_neurons = 1
|
||||
# Number of neurons in the hidden layer
|
||||
hidden_neurons = 200
|
||||
# Define the input layer
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
# Define the hidden layer as a simple RNN layer with a set number of neurons and add it to
|
||||
# the network immediately after the input layer
|
||||
rnn = SimpleRNN(hidden_neurons,
|
||||
return_sequences=False,
|
||||
stateful = stateful,
|
||||
name="RNN")(inp)
|
||||
# Define the output layer as a dense neural network layer (standard neural network layer)
|
||||
#and add it to the network immediately after the hidden layer.
|
||||
dens = Dense(in_out_neurons,name="dense")(rnn)
|
||||
# Create the machine learning model starting with the input layer and ending with the
|
||||
# output layer
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
# Compile the machine learning model using the mean squared error function as the loss
|
||||
# function and an Adams optimizer.
|
||||
model.compile(loss="mean_squared_error", optimizer="adam")
|
||||
return model
|
||||
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Long-Short Time Memory =====
|
||||
===== Predicting New Points With A Trained Recurrent Neural Network =====
|
||||
|
||||
Discussions about dynamic unrolling through time. discuss memory cells, input and output
|
||||
!bc pycod
|
||||
def test_rnn (x1, y_test, plot_min, plot_max):
|
||||
"""
|
||||
Inputs:
|
||||
x1 (a list or numpy array): The complete x component of the data set
|
||||
y_test (a list or numpy array): The complete y component of the data set
|
||||
plot_min (an int or float): the smallest x value used in the training data
|
||||
plot_max (an int or float): the largest x valye used in the training data
|
||||
Returns:
|
||||
None.
|
||||
Uses a trained recurrent neural network model to predict future points in the
|
||||
series. Computes the MSE of the predicted data set from the true data set, saves
|
||||
the predicted data set to a csv file, and plots the predicted and true data sets w
|
||||
while also displaying the data range used for training.
|
||||
"""
|
||||
# Add the training data as the first dim points in the predicted data array as these
|
||||
# are known values.
|
||||
y_pred = y_test[:dim].tolist()
|
||||
# Generate the first input to the trained recurrent neural network using the last two
|
||||
# points of the training data. Based on how the network was trained this means that it
|
||||
# will predict the first point in the data set after the training data. All of the
|
||||
# brackets are necessary for Tensorflow.
|
||||
next_input = np.array([[[y_test[dim-2]], [y_test[dim-1]]]])
|
||||
# Save the very last point in the training data set. This will be used later.
|
||||
last = [y_test[dim-1]]
|
||||
|
||||
# Iterate until the complete data set is created.
|
||||
for i in range (dim, len(y_test)):
|
||||
# Predict the next point in the data set using the previous two points.
|
||||
next = model.predict(next_input)
|
||||
# Append just the number of the predicted data set
|
||||
y_pred.append(next[0][0])
|
||||
# Create the input that will be used to predict the next data point in the data set.
|
||||
next_input = np.array([[last, next[0]]], dtype=np.float64)
|
||||
last = next
|
||||
|
||||
# Print the mean squared error between the known data set and the predicted data set.
|
||||
print('MSE: ', np.square(np.subtract(y_test, y_pred)).mean())
|
||||
# Save the predicted data set as a csv file for later use
|
||||
name = datatype + 'Predicted'+str(dim)+'.csv'
|
||||
np.savetxt(name, y_pred, delimiter=',')
|
||||
# Plot the known data set and the predicted data set. The red box represents the region that was used
|
||||
# for the training data.
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(x1, y_test, label="true", linewidth=3)
|
||||
ax.plot(x1, y_pred, 'g-.',label="predicted", linewidth=4)
|
||||
ax.legend()
|
||||
# Created a red region to represent the points used in the training data.
|
||||
ax.axvspan(plot_min, plot_max, alpha=0.25, color='red')
|
||||
plt.show()
|
||||
|
||||
# Check to make sure the data set is complete
|
||||
assert len(X_tot) == len(y_tot)
|
||||
|
||||
# This is the number of points that will be used in as the training data
|
||||
dim=12
|
||||
|
||||
# Separate the training data from the whole data set
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
# Generate the training data for the RNN, using a sequence of 2
|
||||
rnn_input, rnn_training = format_data(y_train, 2)
|
||||
|
||||
|
||||
# Create a recurrent neural network in Keras and produce a summary of the
|
||||
# machine learning model
|
||||
model = rnn(length_of_sequences = rnn_input.shape[1])
|
||||
model.summary()
|
||||
|
||||
# Start the timer. Want to time training+testing
|
||||
start = timer()
|
||||
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
|
||||
# validation split. Setting verbose to True prints information about each training iteration.
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=None, epochs=150,
|
||||
verbose=True,validation_split=0.05)
|
||||
|
||||
for label in ["loss","val_loss"]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel("loss")
|
||||
plt.xlabel("epoch")
|
||||
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
# Use the trained neural network to predict more points of the data set
|
||||
test_rnn(X_tot, y_tot, X_tot[0], X_tot[dim-1])
|
||||
# Stop the timer and calculate the total time needed.
|
||||
end = timer()
|
||||
print('Time: ', end-start)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Autoencoders: Overarching view =====
|
||||
===== Other Things to Try =====
|
||||
|
||||
Autoencoders are artificial neural networks capable of learning
|
||||
efficient representations of the input data (these representations are called codings) without
|
||||
any supervision (i.e., the training set is unlabeled). These codings
|
||||
typically have a much lower dimensionality than the input data, making
|
||||
autoencoders useful for dimensionality reduction.
|
||||
|
||||
More importantly, autoencoders act as powerful feature detectors, and
|
||||
they can be used for unsupervised pretraining of deep neural networks.
|
||||
Changing the size of the recurrent neural network and its parameters
|
||||
can drastically change the results you get from the model. The below
|
||||
code takes the simple recurrent neural network from above and adds a
|
||||
second hidden layer, changes the number of neurons in the hidden
|
||||
layer, and explicitly declares the activation function of the hidden
|
||||
layers to be a sigmoid function. The loss function and optimizer can
|
||||
also be changed but are kept the same as the above network. These
|
||||
parameters can be tuned to provide the optimal result from the
|
||||
network. For some ideas on how to improve the performance of a
|
||||
"recurrent neural network":"https://danijar.com/tips-for-training-recurrent-neural-networks".
|
||||
|
||||
Lastly, they are capable of randomly generating new data that looks
|
||||
very similar to the training data; this is called a generative
|
||||
model. For example, you could train an autoencoder on pictures of
|
||||
faces, and it would then be able to generate new faces. Surprisingly,
|
||||
autoencoders work by simply learning to copy their inputs to their
|
||||
outputs. This may sound like a trivial task, but we will see that
|
||||
constraining the network in various ways can make it rather
|
||||
difficult. For example, you can limit the size of the internal
|
||||
representation, or you can add noise to the inputs and train the
|
||||
network to recover the original inputs. These constraints prevent the
|
||||
autoencoder from trivially copying the inputs directly to the outputs,
|
||||
which forces it to learn efficient ways of representing the data. In
|
||||
short, the codings are byproducts of the autoencoder’s attempt to
|
||||
learn the identity function under some constraints.
|
||||
!bc pycod
|
||||
def rnn_2layers(length_of_sequences, batch_size = None, stateful = False):
|
||||
"""
|
||||
Inputs:
|
||||
length_of_sequences (an int): the number of y values in "x data". This is determined
|
||||
when the data is formatted
|
||||
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
|
||||
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
|
||||
Returns:
|
||||
model (a Keras model): The recurrent neural network that is built and compiled by this
|
||||
method
|
||||
Builds and compiles a recurrent neural network with two hidden layers and returns the model.
|
||||
"""
|
||||
# Number of neurons in the input and output layers
|
||||
in_out_neurons = 1
|
||||
# Number of neurons in the hidden layer, increased from the first network
|
||||
hidden_neurons = 500
|
||||
# Define the input layer
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
# Create two hidden layers instead of one hidden layer. Explicitly set the activation
|
||||
# function to be the sigmoid function (the default value is hyperbolic tangent)
|
||||
rnn1 = SimpleRNN(hidden_neurons,
|
||||
return_sequences=True, # This needs to be True if another hidden layer is to follow
|
||||
stateful = stateful, activation = 'sigmoid',
|
||||
name="RNN1")(inp)
|
||||
rnn2 = SimpleRNN(hidden_neurons,
|
||||
return_sequences=False, activation = 'sigmoid',
|
||||
stateful = stateful,
|
||||
name="RNN2")(rnn1)
|
||||
# Define the output layer as a dense neural network layer (standard neural network layer)
|
||||
#and add it to the network immediately after the hidden layer.
|
||||
dens = Dense(in_out_neurons,name="dense")(rnn2)
|
||||
# Create the machine learning model starting with the input layer and ending with the
|
||||
# output layer
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
# Compile the machine learning model using the mean squared error function as the loss
|
||||
# function and an Adams optimizer.
|
||||
model.compile(loss="mean_squared_error", optimizer="adam")
|
||||
return model
|
||||
|
||||
# Check to make sure the data set is complete
|
||||
assert len(X_tot) == len(y_tot)
|
||||
|
||||
# This is the number of points that will be used in as the training data
|
||||
dim=12
|
||||
|
||||
# Separate the training data from the whole data set
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
# Generate the training data for the RNN, using a sequence of 2
|
||||
rnn_input, rnn_training = format_data(y_train, 2)
|
||||
|
||||
|
||||
# Create a recurrent neural network in Keras and produce a summary of the
|
||||
# machine learning model
|
||||
model = rnn_2layers(length_of_sequences = 2)
|
||||
model.summary()
|
||||
|
||||
# Start the timer. Want to time training+testing
|
||||
start = timer()
|
||||
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
|
||||
# validation split. Setting verbose to True prints information about each training iteration.
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=None, epochs=150,
|
||||
verbose=True,validation_split=0.05)
|
||||
|
||||
|
||||
# This section plots the training loss and the validation loss as a function of training iteration.
|
||||
# This is not required for analyzing the couple cluster data but can help determine if the network is
|
||||
# being overtrained.
|
||||
for label in ["loss","val_loss"]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel("loss")
|
||||
plt.xlabel("epoch")
|
||||
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
# Use the trained neural network to predict more points of the data set
|
||||
test_rnn(X_tot, y_tot, X_tot[0], X_tot[dim-1])
|
||||
# Stop the timer and calculate the total time needed.
|
||||
end = timer()
|
||||
print('Time: ', end-start)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Simple examples of Autoencoders =====
|
||||
===== Other Types of Recurrent Neural Networks =====
|
||||
|
||||
Besides a simple recurrent neural network layer, there are two other
|
||||
commonly used types of recurrent neural network layers: Long Short
|
||||
Term Memory (LSTM) and Gated Recurrent Unit (GRU). For a short
|
||||
introduction to these layers see URL:"https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b"
|
||||
and URL:"https://medium.com/mindboard/lstm-vs-gru-experimental-comparison-955820c21e8b".
|
||||
|
||||
The first network created below is similar to the previous network,
|
||||
but it replaces the SimpleRNN layers with LSTM layers. The second
|
||||
network below has two hidden layers made up of GRUs, which are
|
||||
preceeded by two dense (feeddorward) neural network layers. These
|
||||
dense layers "preprocess" the data before it reaches the recurrent
|
||||
layers. This architecture has been shown to improve the performance
|
||||
of recurrent neural networks (see the link above and also
|
||||
URL:"https://arxiv.org/pdf/1807.02857.pdf".
|
||||
|
||||
!bc pycod
|
||||
def lstm_2layers(length_of_sequences, batch_size = None, stateful = False):
|
||||
"""
|
||||
Inputs:
|
||||
length_of_sequences (an int): the number of y values in "x data". This is determined
|
||||
when the data is formatted
|
||||
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
|
||||
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
|
||||
Returns:
|
||||
model (a Keras model): The recurrent neural network that is built and compiled by this
|
||||
method
|
||||
Builds and compiles a recurrent neural network with two LSTM hidden layers and returns the model.
|
||||
"""
|
||||
# Number of neurons on the input/output layer and the number of neurons in the hidden layer
|
||||
in_out_neurons = 1
|
||||
hidden_neurons = 250
|
||||
# Input Layer
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
# Hidden layers (in this case they are LSTM layers instead if SimpleRNN layers)
|
||||
rnn= LSTM(hidden_neurons,
|
||||
return_sequences=True,
|
||||
stateful = stateful,
|
||||
name="RNN", use_bias=True, activation='tanh')(inp)
|
||||
rnn1 = LSTM(hidden_neurons,
|
||||
return_sequences=False,
|
||||
stateful = stateful,
|
||||
name="RNN1", use_bias=True, activation='tanh')(rnn)
|
||||
# Output layer
|
||||
dens = Dense(in_out_neurons,name="dense")(rnn1)
|
||||
# Define the midel
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
# Compile the model
|
||||
model.compile(loss='mean_squared_error', optimizer='adam')
|
||||
# Return the model
|
||||
return model
|
||||
|
||||
def dnn2_gru2(length_of_sequences, batch_size = None, stateful = False):
|
||||
"""
|
||||
Inputs:
|
||||
length_of_sequences (an int): the number of y values in "x data". This is determined
|
||||
when the data is formatted
|
||||
batch_size (an int): Default value is None. See Keras documentation of SimpleRNN.
|
||||
stateful (a boolean): Default value is False. See Keras documentation of SimpleRNN.
|
||||
Returns:
|
||||
model (a Keras model): The recurrent neural network that is built and compiled by this
|
||||
method
|
||||
Builds and compiles a recurrent neural network with four hidden layers (two dense followed by
|
||||
two GRU layers) and returns the model.
|
||||
"""
|
||||
# Number of neurons on the input/output layers and hidden layers
|
||||
in_out_neurons = 1
|
||||
hidden_neurons = 250
|
||||
# Input layer
|
||||
inp = Input(batch_shape=(batch_size,
|
||||
length_of_sequences,
|
||||
in_out_neurons))
|
||||
# Hidden Dense (feedforward) layers
|
||||
dnn = Dense(hidden_neurons/2, activation='relu', name='dnn')(inp)
|
||||
dnn1 = Dense(hidden_neurons/2, activation='relu', name='dnn1')(dnn)
|
||||
# Hidden GRU layers
|
||||
rnn1 = GRU(hidden_neurons,
|
||||
return_sequences=True,
|
||||
stateful = stateful,
|
||||
name="RNN1", use_bias=True)(dnn1)
|
||||
rnn = GRU(hidden_neurons,
|
||||
return_sequences=False,
|
||||
stateful = stateful,
|
||||
name="RNN", use_bias=True)(rnn1)
|
||||
# Output layer
|
||||
dens = Dense(in_out_neurons,name="dense")(rnn)
|
||||
# Define the model
|
||||
model = Model(inputs=[inp],outputs=[dens])
|
||||
# Compile the mdoel
|
||||
model.compile(loss='mean_squared_error', optimizer='adam')
|
||||
# Return the model
|
||||
return model
|
||||
|
||||
# Check to make sure the data set is complete
|
||||
assert len(X_tot) == len(y_tot)
|
||||
|
||||
# This is the number of points that will be used in as the training data
|
||||
dim=12
|
||||
|
||||
# Separate the training data from the whole data set
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
|
||||
# Generate the training data for the RNN, using a sequence of 2
|
||||
rnn_input, rnn_training = format_data(y_train, 2)
|
||||
|
||||
|
||||
# Create a recurrent neural network in Keras and produce a summary of the
|
||||
# machine learning model
|
||||
# Change the method name to reflect which network you want to use
|
||||
model = dnn2_gru2(length_of_sequences = 2)
|
||||
model.summary()
|
||||
|
||||
# Start the timer. Want to time training+testing
|
||||
start = timer()
|
||||
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
|
||||
# validation split. Setting verbose to True prints information about each training iteration.
|
||||
hist = model.fit(rnn_input, rnn_training, batch_size=None, epochs=150,
|
||||
verbose=True,validation_split=0.05)
|
||||
|
||||
|
||||
# This section plots the training loss and the validation loss as a function of training iteration.
|
||||
# This is not required for analyzing the couple cluster data but can help determine if the network is
|
||||
# being overtrained.
|
||||
for label in ["loss","val_loss"]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel("loss")
|
||||
plt.xlabel("epoch")
|
||||
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
# Use the trained neural network to predict more points of the data set
|
||||
test_rnn(X_tot, y_tot, X_tot[0], X_tot[dim-1])
|
||||
# Stop the timer and calculate the total time needed.
|
||||
end = timer()
|
||||
print('Time: ', end-start)
|
||||
|
||||
|
||||
# ### Training Recurrent Neural Networks in the Standard Way (i.e. learning the relationship between the X and Y data)
|
||||
#
|
||||
# Finally, comparing the performace of a recurrent neural network using the standard data formatting to the performance of the network with time sequence data formatting shows the benefit of this type of data formatting with extrapolation.
|
||||
|
||||
# Check to make sure the data set is complete
|
||||
assert len(X_tot) == len(y_tot)
|
||||
|
||||
# This is the number of points that will be used in as the training data
|
||||
dim=12
|
||||
|
||||
# Separate the training data from the whole data set
|
||||
X_train = X_tot[:dim]
|
||||
y_train = y_tot[:dim]
|
||||
|
||||
# Reshape the data for Keras specifications
|
||||
X_train = X_train.reshape((dim, 1))
|
||||
y_train = y_train.reshape((dim, 1))
|
||||
|
||||
|
||||
# Create a recurrent neural network in Keras and produce a summary of the
|
||||
# machine learning model
|
||||
# Set the sequence length to 1 for regular data formatting
|
||||
model = rnn(length_of_sequences = 1)
|
||||
model.summary()
|
||||
|
||||
# Start the timer. Want to time training+testing
|
||||
start = timer()
|
||||
# Fit the model using the training data genenerated above using 150 training iterations and a 5%
|
||||
# validation split. Setting verbose to True prints information about each training iteration.
|
||||
hist = model.fit(X_train, y_train, batch_size=None, epochs=150,
|
||||
verbose=True,validation_split=0.05)
|
||||
|
||||
|
||||
# This section plots the training loss and the validation loss as a function of training iteration.
|
||||
# This is not required for analyzing the couple cluster data but can help determine if the network is
|
||||
# being overtrained.
|
||||
for label in ["loss","val_loss"]:
|
||||
plt.plot(hist.history[label],label=label)
|
||||
|
||||
plt.ylabel("loss")
|
||||
plt.xlabel("epoch")
|
||||
plt.title("The final validation loss: {}".format(hist.history["val_loss"][-1]))
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
# Use the trained neural network to predict the remaining data points
|
||||
X_pred = X_tot[dim:]
|
||||
X_pred = X_pred.reshape((len(X_pred), 1))
|
||||
y_model = model.predict(X_pred)
|
||||
y_pred = np.concatenate((y_tot[:dim], y_model.flatten()))
|
||||
|
||||
# Plot the known data set and the predicted data set. The red box represents the region that was used
|
||||
# for the training data.
|
||||
fig, ax = plt.subplots()
|
||||
ax.plot(X_tot, y_tot, label="true", linewidth=3)
|
||||
ax.plot(X_tot, y_pred, 'g-.',label="predicted", linewidth=4)
|
||||
ax.legend()
|
||||
# Created a red region to represent the points used in the training data.
|
||||
ax.axvspan(X_tot[0], X_tot[dim], alpha=0.25, color='red')
|
||||
plt.show()
|
||||
|
||||
# Stop the timer and calculate the total time needed.
|
||||
end = timer()
|
||||
print('Time: ', end-start)
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user