added material
This commit is contained in:
Morten Hjorth-Jensen
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -156,7 +215,7 @@ end of tocinfo -->
|
||||
<li><a href="._week43-bs008.html">9</a></li>
|
||||
<li><a href="._week43-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -163,7 +222,7 @@ end of tocinfo -->
|
||||
<li><a href="._week43-bs009.html">10</a></li>
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs002.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -137,7 +196,7 @@ end of tocinfo -->
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs003.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -139,6 +198,8 @@ The first type of deep learning networks work very well on homogeneous and struc
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs004.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -155,6 +214,9 @@ This is where recurrent nueral networks (RNNs) come to our rescue.
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs005.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -135,9 +194,6 @@ example, they can take sentences, documents, or audio samples as
|
||||
input, making them extremely useful for natural language processing
|
||||
systems such as automatic translation and speech-to-text.
|
||||
|
||||
<p>
|
||||
RNNs are also used in "text classification":"Text Classification Algorithms: A Survey"
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -156,6 +212,10 @@ RNNs are also used in "text classification":"Text Classification Algorithms: A S
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs006.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -115,6 +174,9 @@ end of tocinfo -->
|
||||
|
||||
<h2 id="set-up-of-an-rnn" class="anchor">Set up of an RNN </h2>
|
||||
|
||||
<p>
|
||||
More to text to be added by Wednesday October 27.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -133,6 +195,11 @@ end of tocinfo -->
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="._week43-bs015.html">16</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs007.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -205,6 +264,12 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="._week43-bs015.html">16</a></li>
|
||||
<li><a href="._week43-bs016.html">17</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs008.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -170,6 +229,13 @@ y_tot <span style="color: #666666">=</span> np<span style="color: #666666">.</sp
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="._week43-bs015.html">16</a></li>
|
||||
<li><a href="._week43-bs016.html">17</a></li>
|
||||
<li><a href="._week43-bs017.html">18</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs009.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -245,6 +304,14 @@ y values far removed from the training data set.
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="._week43-bs015.html">16</a></li>
|
||||
<li><a href="._week43-bs016.html">17</a></li>
|
||||
<li><a href="._week43-bs017.html">18</a></li>
|
||||
<li><a href="._week43-bs018.html">19</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs010.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -228,6 +287,15 @@ end <span style="color: #666666">=</span> timer()
|
||||
<li class="active"><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="._week43-bs015.html">16</a></li>
|
||||
<li><a href="._week43-bs016.html">17</a></li>
|
||||
<li><a href="._week43-bs017.html">18</a></li>
|
||||
<li><a href="._week43-bs018.html">19</a></li>
|
||||
<li><a href="._week43-bs019.html">20</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs011.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -234,6 +293,16 @@ end <span style="color: #666666">=</span> timer()
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li class="active"><a href="._week43-bs011.html">12</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="._week43-bs015.html">16</a></li>
|
||||
<li><a href="._week43-bs016.html">17</a></li>
|
||||
<li><a href="._week43-bs017.html">18</a></li>
|
||||
<li><a href="._week43-bs018.html">19</a></li>
|
||||
<li><a href="._week43-bs019.html">20</a></li>
|
||||
<li><a href="._week43-bs020.html">21</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs012.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -327,7 +386,6 @@ end <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Time: '</span>, end<span style="color: #666666">-</span>start)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -343,6 +401,18 @@ end <span style="color: #666666">=</span> timer()
|
||||
<li><a href="._week43-bs010.html">11</a></li>
|
||||
<li><a href="._week43-bs011.html">12</a></li>
|
||||
<li class="active"><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs013.html">14</a></li>
|
||||
<li><a href="._week43-bs014.html">15</a></li>
|
||||
<li><a href="._week43-bs015.html">16</a></li>
|
||||
<li><a href="._week43-bs016.html">17</a></li>
|
||||
<li><a href="._week43-bs017.html">18</a></li>
|
||||
<li><a href="._week43-bs018.html">19</a></li>
|
||||
<li><a href="._week43-bs019.html">20</a></li>
|
||||
<li><a href="._week43-bs020.html">21</a></li>
|
||||
<li><a href="._week43-bs021.html">22</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs013.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,20 +172,23 @@ MathJax.Hub.Config({
|
||||
<a name="part0013"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="strong-correlations" class="anchor">Strong correlations </h2>
|
||||
<h2 id="generative-models" class="anchor">Generative Models </h2>
|
||||
|
||||
<p>
|
||||
Images typically have strong local correlations, meaning that a small
|
||||
part of the image varies little from its neighboring regions. If for
|
||||
example we have an image of a blue car, we can roughly assume that a
|
||||
small blue part of the image is surrounded by other blue regions.
|
||||
<b>Generative models</b> describe a class of statistical models that are a contrast
|
||||
to <b>discriminative models</b>. Informally we say that generative models can
|
||||
generate new data instances while discriminative models discriminate between
|
||||
different kinds of data instances. A generative model could generate new photos
|
||||
of animals that look like 'real' animals while a discriminative model could tell
|
||||
a dog from a cat. More formally, given a data set \( x \) and a set of labels /
|
||||
targets \( y \). Generative models capture the joint probability \( p(x, y) \), or
|
||||
just \( p(x) \) if there are no labels, while discriminative models capture the
|
||||
conditional probability \( p(y | x) \). Discriminative models generally try to draw
|
||||
boundaries in the data space (often high dimensional), while generative models
|
||||
try to model how data is placed throughout the space.
|
||||
|
||||
<p>
|
||||
Therefore, instead of connecting every single pixel to a neuron in the
|
||||
first hidden layer, as we have previously done with deep neural
|
||||
networks, we can instead connect each neuron to a small part of the
|
||||
image (in all 3 RGB depth dimensions). The size of each small area is
|
||||
fixed, and known as a <a href="https://en.wikipedia.org/wiki/Receptive_field" target="_self">receptive</a>.
|
||||
<b>Note</b>: this material is thanks to Linus Ekstrøm.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -240,7 +216,7 @@ fixed, and known as a <a href="https://en.wikipedia.org/wiki/Receptive_field" ta
|
||||
<li><a href="._week43-bs021.html">22</a></li>
|
||||
<li><a href="._week43-bs022.html">23</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs014.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -197,26 +170,27 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0014"></a>
|
||||
<!-- !split -->
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="layers-of-a-cnn" class="anchor">Layers of a CNN </h2>
|
||||
The layers of a convolutional neural network arrange neurons in 3D: width, height and depth.
|
||||
The input image is typically a square matrix of depth 3.
|
||||
<h2 id="generative-adversarial-networks" class="anchor">Generative Adversarial Networks </h2>
|
||||
|
||||
<p>
|
||||
A <b>convolution</b> is performed on the image which outputs
|
||||
a 3D volume of neurons. The weights to the input are arranged in a number of 2D matrices, known as <b>filters</b>.
|
||||
<b>Generative Adversarial Networks</b> are a type of unsupervised machine learning
|
||||
algorithm proposed by <a href="https://arxiv.org/pdf/1406.2661.pdf" target="_self">Goodfellow et. al</a>
|
||||
in 2014 (short and good article).
|
||||
|
||||
<p>
|
||||
Each filter slides along the input image, taking the dot product
|
||||
between each small part of the image and the filter, in all depth
|
||||
dimensions. This is then passed through a non-linear function,
|
||||
typically the <b>Rectified Linear (ReLu)</b> function, which serves as the
|
||||
activation of the neurons in the first convolutional layer. This is
|
||||
further passed through a <b>pooling layer</b>, which reduces the size of the
|
||||
convolutional layer, e.g. by taking the maximum or average across some
|
||||
small regions, and this serves as input to the next convolutional
|
||||
layer.
|
||||
The simplest formulation of
|
||||
the model is based on a game theoretic approach, <em>zero sum game</em>, where we pit
|
||||
two neural networks against one another. We define two rival networks, one
|
||||
generator \( g \), and one discriminator \( d \). The generator directly produces
|
||||
samples
|
||||
$$
|
||||
\begin{equation}
|
||||
x = g(z; \theta^{(g)})
|
||||
\tag{1}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -244,7 +218,7 @@ layer.
|
||||
<li><a href="._week43-bs022.html">23</a></li>
|
||||
<li><a href="._week43-bs023.html">24</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs015.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,17 +172,43 @@ MathJax.Hub.Config({
|
||||
<a name="part0015"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="systematic-reduction" class="anchor">Systematic reduction </h2>
|
||||
<h2 id="discriminator" class="anchor">Discriminator </h2>
|
||||
The discriminator attempts to distinguish between samples drawn from the
|
||||
training data and samples drawn from the generator. In other words, it tries to
|
||||
tell the difference between the fake data produced by \( g \) and the actual data
|
||||
samples we want to do prediction on. The discriminator outputs a probability
|
||||
value given by
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
d(x; \theta^{(d)})
|
||||
\tag{2}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
By systematically reducing the size of the input volume, through
|
||||
convolution and pooling, the network should create representations of
|
||||
small parts of the input, and then from them assemble representations
|
||||
of larger areas. The final pooling layer is flattened to serve as
|
||||
input to a hidden layer, such that each neuron in the final pooling
|
||||
layer is connected to every single neuron in the hidden layer. This
|
||||
then serves as input to the output layer, e.g. a softmax output for
|
||||
classification.
|
||||
indicating the probability that \( x \) is a real training example rather than a
|
||||
fake sample the generator has generated. The simplest way to formulate the
|
||||
learning process in a generative adversarial network is a zero-sum game, in
|
||||
which a function
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{3}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
determines the reward for the discriminator, while the generator gets the
|
||||
conjugate reward
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
-v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{4}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<p>
|
||||
@@ -237,7 +236,7 @@ classification.
|
||||
<li><a href="._week43-bs023.html">24</a></li>
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs016.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,51 +172,26 @@ MathJax.Hub.Config({
|
||||
<a name="part0016"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="prerequisites-collect-and-pre-process-data" class="anchor">Prerequisites: Collect and pre-process data </h2>
|
||||
<h2 id="learning-process" class="anchor">Learning Process </h2>
|
||||
|
||||
<p>
|
||||
During learning both of the networks maximize their own reward function, so that
|
||||
the generator gets better and better at tricking the discriminator, while the
|
||||
discriminator gets better and better at telling the difference between the fake
|
||||
and real data. The generator and discriminator alternate on which one trains at
|
||||
one time (i.e. for one epoch). In other words, we keep the generator constant
|
||||
and train the discriminator, then we keep the discriminator constant to train
|
||||
the generator and repeat. It is this back and forth dynamic which lets GANs
|
||||
tackle otherwise intractable generative problems. As the generator improves with
|
||||
training, the discriminator's performance gets worse because it cannot easily
|
||||
tell the difference between real and fake. If the generator ends up succeeding
|
||||
perfectly, the the discriminator will do no better than random guessing i.e.
|
||||
50\%. This progression in the training poses a problem for the convergence
|
||||
criteria for GANs. The discriminator feedback gets less meaningful over time,
|
||||
if we continue training after this point then the generator is effectively
|
||||
training on junk data which can undo the learning up to that point. Therefore,
|
||||
we stop training when the discriminator starts outputting \( 1/2 \) everywhere.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #408080; font-style: italic"># import necessary packages</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn</span> <span style="color: #008000; font-weight: bold">import</span> datasets
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># ensure the same random numbers appear every time</span>
|
||||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #666666">0</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># display images in notebook</span>
|
||||
<span style="color: #666666">%</span>matplotlib inline
|
||||
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> (<span style="color: #666666">12</span>,<span style="color: #666666">12</span>)
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># download MNIST dataset</span>
|
||||
digits <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>load_digits()
|
||||
|
||||
<span style="color: #408080; font-style: italic"># define inputs and labels</span>
|
||||
inputs <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>images
|
||||
labels <span style="color: #666666">=</span> digits<span style="color: #666666">.</span>target
|
||||
|
||||
<span style="color: #408080; font-style: italic"># RGB images have a depth of 3</span>
|
||||
<span style="color: #408080; font-style: italic"># our images are grayscale so they should have a depth of 1</span>
|
||||
inputs <span style="color: #666666">=</span> inputs[:,:,:,np<span style="color: #666666">.</span>newaxis]
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"inputs = (n_inputs, pixel_width, pixel_height, depth) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(inputs<span style="color: #666666">.</span>shape))
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"labels = (n_inputs) = "</span> <span style="color: #666666">+</span> <span style="color: #008000">str</span>(labels<span style="color: #666666">.</span>shape))
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># choose some random images to display</span>
|
||||
n_inputs <span style="color: #666666">=</span> <span style="color: #008000">len</span>(inputs)
|
||||
indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>arange(n_inputs)
|
||||
random_indices <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>choice(indices, size<span style="color: #666666">=5</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, image <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(digits<span style="color: #666666">.</span>images[random_indices]):
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, i<span style="color: #666666">+1</span>)
|
||||
plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">'off'</span>)
|
||||
plt<span style="color: #666666">.</span>imshow(image, cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>gray_r, interpolation<span style="color: #666666">=</span><span style="color: #BA2121">'nearest'</span>)
|
||||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Label: </span><span style="color: #BB6688; font-weight: bold">%d</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> digits<span style="color: #666666">.</span>target[random_indices[i]])
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -270,7 +218,7 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs017.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,33 +172,43 @@ MathJax.Hub.Config({
|
||||
<a name="part0017"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="importing-keras-and-tensorflow" class="anchor">Importing Keras and Tensorflow </h2>
|
||||
<h2 id="more-about-the-learning-process" class="anchor">More about the Learning Process </h2>
|
||||
|
||||
<p>
|
||||
At convergence we have
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> datasets, layers, models
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Input
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.models</span> <span style="color: #008000; font-weight: bold">import</span> Sequential <span style="color: #408080; font-style: italic">#This allows appending layers to existing models</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.layers</span> <span style="color: #008000; font-weight: bold">import</span> Dense <span style="color: #408080; font-style: italic">#This allows defining the characteristics of a particular layer</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> optimizers <span style="color: #408080; font-style: italic">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> regularizers <span style="color: #408080; font-style: italic">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> to_categorical <span style="color: #408080; font-style: italic">#This allows using categorical cross entropy as the cost function</span>
|
||||
<span style="color: #408080; font-style: italic">#from tensorflow.keras import Conv2D</span>
|
||||
<span style="color: #408080; font-style: italic">#from tensorflow.keras import MaxPooling2D</span>
|
||||
<span style="color: #408080; font-style: italic">#from tensorflow.keras import Flatten</span>
|
||||
$$
|
||||
\begin{equation}
|
||||
g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{5}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||||
The default choice for \( v \) is
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x)
|
||||
+ \mathbb{E}_{x\sim p_\mathrm{model}}
|
||||
\log (1 - d(x))
|
||||
\tag{6}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<span style="color: #408080; font-style: italic"># representation of labels</span>
|
||||
labels <span style="color: #666666">=</span> to_categorical(labels)
|
||||
The main motivation for the design of GANs is that the learning process requires
|
||||
neither approximate inference (variational autoencoders for example) nor
|
||||
approximation of a partition function. In the case where
|
||||
$$
|
||||
\begin{equation}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{7}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is
|
||||
asymptotically consistent
|
||||
( <a href="https://arxiv.org/pdf/1804.09139.pdf" target="_self">Seth Lloyd on QuGANs</a> ).
|
||||
|
||||
<span style="color: #408080; font-style: italic"># split into train and test data</span>
|
||||
<span style="color: #408080; font-style: italic"># one-liner from scikit-learn library</span>
|
||||
train_size <span style="color: #666666">=</span> <span style="color: #666666">0.8</span>
|
||||
test_size <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">-</span> train_size
|
||||
X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_test_split(inputs, labels, train_size<span style="color: #666666">=</span>train_size,
|
||||
test_size<span style="color: #666666">=</span>test_size)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -251,8 +234,6 @@ X_train, X_test, Y_train, Y_test <span style="color: #666666">=</span> train_tes
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs018.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -197,40 +170,21 @@ MathJax.Hub.Config({
|
||||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||||
|
||||
<a name="part0018"></a>
|
||||
<!-- !split -->
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="running-with-keras" class="anchor">Running with Keras </h2>
|
||||
<h2 id="additional-references" class="anchor">Additional References </h2>
|
||||
This is in
|
||||
general not the case and it is possible to get situations where the training
|
||||
process never converges because the generator and discriminator chase one
|
||||
another around in the parameter space indefinitely. A much deeper discussion on
|
||||
the currently open research problem of GAN convergence is available
|
||||
<a href="https://www.deeplearningbook.org/contents/generative_models.html" target="_self">here</a>. To
|
||||
anyone interested in learning more about GANs it is a highly recommended read.
|
||||
Direct quote: "In this best-performing formulation, the generator aims to
|
||||
increase the log probability that the discriminator makes a mistake, rather than
|
||||
aiming to decrease the log probability that the discriminator makes the correct
|
||||
prediction." <a href="https://arxiv.org/abs/1701.00160" target="_self">Another interesting read</a>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">create_convolutional_neural_network_keras</span>(input_shape, receptive_field,
|
||||
n_filters, n_neurons_connected, n_categories,
|
||||
eta, lmbd):
|
||||
model <span style="color: #666666">=</span> Sequential()
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(n_filters, (receptive_field, receptive_field), input_shape<span style="color: #666666">=</span>input_shape, padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>, kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lmbd)))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>MaxPooling2D(pool_size<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>)))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Flatten())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(n_neurons_connected, activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>, kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lmbd)))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(n_categories, activation<span style="color: #666666">=</span><span style="color: #BA2121">'softmax'</span>, kernel_regularizer<span style="color: #666666">=</span>regularizers<span style="color: #666666">.</span>l2(lmbd)))
|
||||
|
||||
sgd <span style="color: #666666">=</span> optimizers<span style="color: #666666">.</span>SGD(lr<span style="color: #666666">=</span>eta)
|
||||
model<span style="color: #666666">.</span>compile(loss<span style="color: #666666">=</span><span style="color: #BA2121">'categorical_crossentropy'</span>, optimizer<span style="color: #666666">=</span>sgd, metrics<span style="color: #666666">=</span>[<span style="color: #BA2121">'accuracy'</span>])
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
|
||||
epochs <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
batch_size <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
input_shape <span style="color: #666666">=</span> X_train<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>:<span style="color: #666666">4</span>]
|
||||
receptive_field <span style="color: #666666">=</span> <span style="color: #666666">3</span>
|
||||
n_filters <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
n_neurons_connected <span style="color: #666666">=</span> <span style="color: #666666">50</span>
|
||||
n_categories <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||||
|
||||
eta_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.</span>logspace(<span style="color: #666666">-5</span>, <span style="color: #666666">1</span>, <span style="color: #666666">7</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -255,9 +209,6 @@ lmbd_vals <span style="color: #666666">=</span> np<span style="color: #666666">.
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs019.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,27 +172,47 @@ MathJax.Hub.Config({
|
||||
<a name="part0019"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="final-part" class="anchor">Final part </h2>
|
||||
<h2 id="writing-our-first-generative-adversarial-network" class="anchor">Writing Our First Generative Adversarial Network </h2>
|
||||
Let us now move on to actually implementing a GAN in tensorflow. We will study
|
||||
the performance of our GAN on the MNIST dataset. This code is based on and
|
||||
adapted from the
|
||||
<a href="https://www.tensorflow.org/tutorials/generative/dcgan" target="_self">google tutorial</a>
|
||||
|
||||
<p>
|
||||
First we import our libraries
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>CNN_keras <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)), dtype<span style="color: #666666">=</span><span style="color: #008000">object</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, eta <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(eta_vals):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j, lmbd <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(lmbd_vals):
|
||||
CNN <span style="color: #666666">=</span> create_convolutional_neural_network_keras(input_shape, receptive_field,
|
||||
n_filters, n_neurons_connected, n_categories,
|
||||
eta, lmbd)
|
||||
CNN<span style="color: #666666">.</span>fit(X_train, Y_train, epochs<span style="color: #666666">=</span>epochs, batch_size<span style="color: #666666">=</span>batch_size, verbose<span style="color: #666666">=0</span>)
|
||||
scores <span style="color: #666666">=</span> CNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)
|
||||
|
||||
CNN_keras[i][j] <span style="color: #666666">=</span> CNN
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Learning rate = "</span>, eta)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Lambda = "</span>, lmbd)
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test accuracy: </span><span style="color: #BB6688; font-weight: bold">%.3f</span><span style="color: #BA2121">"</span> <span style="color: #666666">%</span> scores[<span style="color: #666666">1</span>])
|
||||
<span style="color: #008000">print</span>()
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> layers
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> plot_model
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define our hyperparameters and import our data the usual way
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>BUFFER_SIZE <span style="color: #666666">=</span> <span style="color: #666666">60000</span>
|
||||
BATCH_SIZE <span style="color: #666666">=</span> <span style="color: #666666">256</span>
|
||||
EPOCHS <span style="color: #666666">=</span> <span style="color: #666666">30</span>
|
||||
|
||||
data <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>datasets<span style="color: #666666">.</span>mnist<span style="color: #666666">.</span>load_data()
|
||||
(train_images, train_labels), (test_images, test_labels) <span style="color: #666666">=</span> data
|
||||
train_images <span style="color: #666666">=</span> np<span style="color: #666666">.</span>reshape(train_images, (train_images<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>],
|
||||
<span style="color: #666666">28</span>,
|
||||
<span style="color: #666666">28</span>,
|
||||
<span style="color: #666666">1</span>))<span style="color: #666666">.</span>astype(<span style="color: #BA2121">'float32'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we normalize between -1 and 1</span>
|
||||
train_images <span style="color: #666666">=</span> (train_images <span style="color: #666666">-</span> <span style="color: #666666">127.5</span>) <span style="color: #666666">/</span> <span style="color: #666666">127.5</span>
|
||||
training_dataset <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>data<span style="color: #666666">.</span>Dataset<span style="color: #666666">.</span>from_tensor_slices(
|
||||
train_images)<span style="color: #666666">.</span>shuffle(BUFFER_SIZE)<span style="color: #666666">.</span>batch(BATCH_SIZE)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -244,10 +237,6 @@ MathJax.Hub.Config({
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs020.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,41 +172,127 @@ MathJax.Hub.Config({
|
||||
<a name="part0020"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="final-visualization" class="anchor">Final visualization </h2>
|
||||
<h2 id="mnist-and-gans" class="anchor">MNIST and GANs </h2>
|
||||
|
||||
<p>
|
||||
Let's have a quick look
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #408080; font-style: italic"># visual representation of grid search</span>
|
||||
<span style="color: #408080; font-style: italic"># uses seaborn heatmap, could probably do this in matplotlib</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">seaborn</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">sns</span>
|
||||
|
||||
sns<span style="color: #666666">.</span>set()
|
||||
|
||||
train_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
test_accuracy <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">len</span>(eta_vals), <span style="color: #008000">len</span>(lmbd_vals)))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(eta_vals)):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(lmbd_vals)):
|
||||
CNN <span style="color: #666666">=</span> CNN_keras[i][j]
|
||||
|
||||
train_accuracy[i][j] <span style="color: #666666">=</span> CNN<span style="color: #666666">.</span>evaluate(X_train, Y_train)[<span style="color: #666666">1</span>]
|
||||
test_accuracy[i][j] <span style="color: #666666">=</span> CNN<span style="color: #666666">.</span>evaluate(X_test, Y_test)[<span style="color: #666666">1</span>]
|
||||
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(train_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Training Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plt<span style="color: #666666">.</span>imshow(train_images[<span style="color: #666666">0</span>], cmap<span style="color: #666666">=</span><span style="color: #BA2121">'Greys'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we define our two models. This is where the 'magic' happens. There are a
|
||||
huge amount of possible formulations for both models. A lot of engineering and
|
||||
trial and error can be done here to try to produce better performing models. For
|
||||
more advanced GANs this is by far the step where you can 'make or break' a
|
||||
model.
|
||||
|
||||
fig, ax <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(figsize <span style="color: #666666">=</span> (<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
sns<span style="color: #666666">.</span>heatmap(test_accuracy, annot<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, ax<span style="color: #666666">=</span>ax, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"viridis"</span>)
|
||||
ax<span style="color: #666666">.</span>set_title(<span style="color: #BA2121">"Test Accuracy"</span>)
|
||||
ax<span style="color: #666666">.</span>set_ylabel(<span style="color: #BA2121">"$\eta$"</span>)
|
||||
ax<span style="color: #666666">.</span>set_xlabel(<span style="color: #BA2121">"$\lambda$"</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
<p>
|
||||
We start with the generator. As stated in the introductory text the generator
|
||||
\( g \) upsamples from a random sample to the shape of what we want to predict. In
|
||||
our case we are trying to predict MNIST images (\( 28\times 28 \) pixels).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generator_model</span>():
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to</span>
|
||||
<span style="color: #BA2121; font-style: italic"> produce an image from a random seed. We start with a Dense layer taking this</span>
|
||||
<span style="color: #BA2121; font-style: italic"> random sample as an input and subsequently upsample through multiple</span>
|
||||
<span style="color: #BA2121; font-style: italic"> convolutional layers.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we define our model</span>
|
||||
model <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>Sequential()
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># adding our input layer. Dense means that every neuron is connected and</span>
|
||||
<span style="color: #408080; font-style: italic"># the input shape is the shape of our random noise. The units need to match</span>
|
||||
<span style="color: #408080; font-style: italic"># in some sense the upsampling strides to reach our desired output shape.</span>
|
||||
<span style="color: #408080; font-style: italic"># we are using 100 random numbers as our seed</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(units<span style="color: #666666">=7*7*</span>BATCH_SIZE,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||||
input_shape<span style="color: #666666">=</span>(<span style="color: #666666">100</span>, )))
|
||||
<span style="color: #408080; font-style: italic"># we normalize the output form the Dense layer</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>BatchNormalization())
|
||||
<span style="color: #408080; font-style: italic"># and add an activation function to our 'layer'. LeakyReLU avoids vanishing</span>
|
||||
<span style="color: #408080; font-style: italic"># gradient problem</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Reshape((<span style="color: #666666">7</span>, <span style="color: #666666">7</span>, BATCH_SIZE)))
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">7</span>, <span style="color: #666666">7</span>, BATCH_SIZE)
|
||||
<span style="color: #408080; font-style: italic"># even though we just added four keras layers we think of everything above</span>
|
||||
<span style="color: #408080; font-style: italic"># as 'one' layer</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># next we add our upscaling convolutional layers</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2DTranspose(filters<span style="color: #666666">=128</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">1</span>, <span style="color: #666666">1</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>BatchNormalization())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">7</span>, <span style="color: #666666">7</span>, <span style="color: #666666">128</span>)
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2DTranspose(filters<span style="color: #666666">=64</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>BatchNormalization())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">14</span>, <span style="color: #666666">14</span>, <span style="color: #666666">64</span>)
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2DTranspose(filters<span style="color: #666666">=1</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||||
activation<span style="color: #666666">=</span><span style="color: #BA2121">'tanh'</span>))
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">28</span>, <span style="color: #666666">28</span>, <span style="color: #666666">1</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
And there we have our 'simple' generator model. Now we move on to defining our
|
||||
discriminator model \( d \), which is a convolutional neural network based image
|
||||
classifier.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">discriminator_model</span>():
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> The discriminator is a convolutional neural network based image classifier</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we define our model</span>
|
||||
model <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>Sequential()
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(filters<span style="color: #666666">=64</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
input_shape<span style="color: #666666">=</span>[<span style="color: #666666">28</span>, <span style="color: #666666">28</span>, <span style="color: #666666">1</span>]))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #408080; font-style: italic"># adding a dropout layer as you do in conv-nets</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dropout(<span style="color: #666666">0.3</span>))
|
||||
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(filters<span style="color: #666666">=128</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #408080; font-style: italic"># adding a dropout layer as you do in conv-nets</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dropout(<span style="color: #666666">0.3</span>))
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Flatten())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(<span style="color: #666666">1</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -257,11 +316,6 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="._week43-bs029.html">30</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs021.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,27 +172,66 @@ MathJax.Hub.Config({
|
||||
<a name="part0021"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="the-cifar01-data-set" class="anchor">The CIFAR01 data set </h2>
|
||||
|
||||
<p>
|
||||
The CIFAR10 dataset contains 60,000 color images in 10 classes, with
|
||||
6,000 images in each class. The dataset is divided into 50,000
|
||||
training images and 10,000 testing images. The classes are mutually
|
||||
exclusive and there is no overlap between them.
|
||||
<h2 id="other-models" class="anchor">Other Models </h2>
|
||||
Let us take a look at our models. <b>Note</b>: double click images for bigger view.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>generator <span style="color: #666666">=</span> generator_model()
|
||||
plot_model(generator, show_shapes<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, rankdir<span style="color: #666666">=</span><span style="color: #BA2121">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> datasets, layers, models
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>discriminator <span style="color: #666666">=</span> discriminator_model()
|
||||
plot_model(discriminator, show_shapes<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, rankdir<span style="color: #666666">=</span><span style="color: #BA2121">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we need a few helper objects we will use in training
|
||||
|
||||
<span style="color: #408080; font-style: italic"># We import the data set</span>
|
||||
(train_images, train_labels), (test_images, test_labels) <span style="color: #666666">=</span> datasets<span style="color: #666666">.</span>cifar10<span style="color: #666666">.</span>load_data()
|
||||
<p>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Normalize pixel values to be between 0 and 1 by dividing by 255. </span>
|
||||
train_images, test_images <span style="color: #666666">=</span> train_images <span style="color: #666666">/</span> <span style="color: #666666">255.0</span>, test_images <span style="color: #666666">/</span> <span style="color: #666666">255.0</span>
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>cross_entropy <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>losses<span style="color: #666666">.</span>BinaryCrossentropy(from_logits<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
generator_optimizer <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>optimizers<span style="color: #666666">.</span>Adam(<span style="color: #666666">1e-4</span>)
|
||||
discriminator_optimizer <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>optimizers<span style="color: #666666">.</span>Adam(<span style="color: #666666">1e-4</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
The first object, <em>cross_entropy</em> is our loss function and the two others are
|
||||
our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This
|
||||
is because they need to improve their accuracy at approximately equal speeds to
|
||||
get convergence (not necessarily exactly equal). Now we define our loss
|
||||
functions
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generator_loss</span>(fake_output):
|
||||
loss <span style="color: #666666">=</span> cross_entropy(tf<span style="color: #666666">.</span>ones_like(fake_output), fake_output)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> loss
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">discriminator_loss</span>(real_output, fake_output):
|
||||
real_loss <span style="color: #666666">=</span> cross_entropy(tf<span style="color: #666666">.</span>ones_like(real_output), real_output)
|
||||
fake_loss <span style="color: #666666">=</span> cross_entropy(tf<span style="color: #666666">.</span>zeros_liks(fake_output), fake_output)
|
||||
total_loss <span style="color: #666666">=</span> real_loss <span style="color: #666666">+</span> fake_loss
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> total_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a kind of seed to help us compare the learning process over
|
||||
multiple training epochs.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>noise_dimension <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
n_examples_to_generate <span style="color: #666666">=</span> <span style="color: #666666">16</span>
|
||||
seed_images <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal([n_examples_to_generate, noise_dimension])
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -242,12 +254,6 @@ train_images, test_images <span style="color: #666666">=</span> train_images <sp
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="._week43-bs029.html">30</a></li>
|
||||
<li><a href="._week43-bs030.html">31</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs022.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,28 +172,62 @@ MathJax.Hub.Config({
|
||||
<a name="part0022"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="verifying-the-data-set" class="anchor">Verifying the data set </h2>
|
||||
<h2 id="training-step" class="anchor">Training Step </h2>
|
||||
|
||||
<p>
|
||||
To verify that the dataset looks correct, let's plot the first 25 images from the training set and display the class name below each image.
|
||||
Now we have everything we need to define our training step, which we will apply
|
||||
for every step in our training loop. Notice the @tf.function flag signifying
|
||||
that the function is tensorflow 'compiled'. Removing this flag doubles the
|
||||
computation time.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>class_names <span style="color: #666666">=</span> [<span style="color: #BA2121">'airplane'</span>, <span style="color: #BA2121">'automobile'</span>, <span style="color: #BA2121">'bird'</span>, <span style="color: #BA2121">'cat'</span>, <span style="color: #BA2121">'deer'</span>,
|
||||
<span style="color: #BA2121">'dog'</span>, <span style="color: #BA2121">'frog'</span>, <span style="color: #BA2121">'horse'</span>, <span style="color: #BA2121">'ship'</span>, <span style="color: #BA2121">'truck'</span>]
|
||||
|
||||
plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">10</span>,<span style="color: #666666">10</span>))
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">25</span>):
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">5</span>,<span style="color: #666666">5</span>,i<span style="color: #666666">+1</span>)
|
||||
plt<span style="color: #666666">.</span>xticks([])
|
||||
plt<span style="color: #666666">.</span>yticks([])
|
||||
plt<span style="color: #666666">.</span>grid(<span style="color: #008000; font-weight: bold">False</span>)
|
||||
plt<span style="color: #666666">.</span>imshow(train_images[i], cmap<span style="color: #666666">=</span>plt<span style="color: #666666">.</span>cm<span style="color: #666666">.</span>binary)
|
||||
<span style="color: #408080; font-style: italic"># The CIFAR labels happen to be arrays, </span>
|
||||
<span style="color: #408080; font-style: italic"># which is why you need the extra index</span>
|
||||
plt<span style="color: #666666">.</span>xlabel(class_names[train_labels[i][<span style="color: #666666">0</span>]])
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #AA22FF">@tf</span><span style="color: #666666">.</span>function
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train_step</span>(images):
|
||||
noise <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal([BATCH_SIZE, noise_dimension])
|
||||
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>GradientTape() <span style="color: #008000; font-weight: bold">as</span> gen_tape, tf<span style="color: #666666">.</span>GradientTape() <span style="color: #008000; font-weight: bold">as</span> disc_tape:
|
||||
generated_images <span style="color: #666666">=</span> generator(noise, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
|
||||
real_output <span style="color: #666666">=</span> discriminator(images, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
fake_output <span style="color: #666666">=</span> discriminator(generated_images, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
|
||||
gen_loss <span style="color: #666666">=</span> generator_loss(fake_output)
|
||||
disc_loss <span style="color: #666666">=</span> discriminator_loss(real_output, fake_output)
|
||||
|
||||
gradients_of_generator <span style="color: #666666">=</span> gen_tape<span style="color: #666666">.</span>gradient(gen_loss,
|
||||
generator<span style="color: #666666">.</span>trainable_variables)
|
||||
gradients_of_discriminator <span style="color: #666666">=</span> disc_tape<span style="color: #666666">.</span>gradient(disc_loss,
|
||||
discriminator<span style="color: #666666">.</span>trainable_variables)
|
||||
generator_optimizer<span style="color: #666666">.</span>apply_gradients(<span style="color: #008000">zip</span>(gradients_of_generator,
|
||||
generator<span style="color: #666666">.</span>trainable_variables))
|
||||
discriminator_optimizer<span style="color: #666666">.</span>apply_gradients(<span style="color: #008000">zip</span>(gradients_of_discriminator,
|
||||
discriminator<span style="color: #666666">.</span>trainable_variables))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> gen_loss, disc_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a helper function to produce an output over our training epochs
|
||||
to see the predictive progression of our generator model. <b>Note</b>: I am including
|
||||
this code here, but comment it out in the training loop.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_and_save_images</span>(model, epoch, test_input):
|
||||
<span style="color: #408080; font-style: italic"># we're making inferences here</span>
|
||||
predictions <span style="color: #666666">=</span> model(test_input, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>)
|
||||
|
||||
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">4</span>, <span style="color: #666666">4</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(predictions<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]):
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">4</span>, <span style="color: #666666">4</span>, i<span style="color: #666666">+1</span>)
|
||||
plt<span style="color: #666666">.</span>imshow(predictions[i, :, :, <span style="color: #666666">0</span>] <span style="color: #666666">*</span> <span style="color: #666666">127.5</span> <span style="color: #666666">+</span> <span style="color: #666666">127.5</span>, cmap<span style="color: #666666">=</span><span style="color: #BA2121">'gray'</span>)
|
||||
plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">'off'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>savefig(<span style="color: #BA2121">f'./images_from_seed_images/image_at_epoch_</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">str</span>(epoch)<span style="color: #666666">.</span>zfill(<span style="color: #666666">3</span>)<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">.png'</span>)
|
||||
plt<span style="color: #666666">.</span>close()
|
||||
<span style="color: #408080; font-style: italic">#plt.show()</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
@@ -242,13 +249,6 @@ plt<span style="color: #666666">.</span>show()
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="._week43-bs029.html">30</a></li>
|
||||
<li><a href="._week43-bs030.html">31</a></li>
|
||||
<li><a href="._week43-bs031.html">32</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs023.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,31 +172,89 @@ MathJax.Hub.Config({
|
||||
<a name="part0023"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="set-up-the-model" class="anchor">Set up the model </h2>
|
||||
|
||||
<p>
|
||||
The 6 lines of code below define the convolutional base using a common pattern: a stack of Conv2D and MaxPooling2D layers.
|
||||
|
||||
<p>
|
||||
As input, a CNN takes tensors of shape (image_height, image_width, color_channels), ignoring the batch size. If you are new to these dimensions, color_channels refers to (R,G,B). In this example, you will configure our CNN to process inputs of shape (32, 32, 3), which is the format of CIFAR images. You can do this by passing the argument input_shape to our first layer.
|
||||
<h2 id="checkpoints" class="anchor">Checkpoints </h2>
|
||||
Setting up checkpoints to periodically save our model during training so that
|
||||
everything is not lost even if the program were to somehow terminate while
|
||||
training.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>model <span style="color: #666666">=</span> models<span style="color: #666666">.</span>Sequential()
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(<span style="color: #666666">32</span>, (<span style="color: #666666">3</span>, <span style="color: #666666">3</span>), activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>, input_shape<span style="color: #666666">=</span>(<span style="color: #666666">32</span>, <span style="color: #666666">32</span>, <span style="color: #666666">3</span>)))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>MaxPooling2D((<span style="color: #666666">2</span>, <span style="color: #666666">2</span>)))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(<span style="color: #666666">64</span>, (<span style="color: #666666">3</span>, <span style="color: #666666">3</span>), activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>MaxPooling2D((<span style="color: #666666">2</span>, <span style="color: #666666">2</span>)))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(<span style="color: #666666">64</span>, (<span style="color: #666666">3</span>, <span style="color: #666666">3</span>), activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>))
|
||||
|
||||
<span style="color: #408080; font-style: italic"># Let's display the architecture of our model so far.</span>
|
||||
|
||||
model<span style="color: #666666">.</span>summary()
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #408080; font-style: italic"># Setting up checkpoints to save model during training</span>
|
||||
checkpoint_dir <span style="color: #666666">=</span> <span style="color: #BA2121">'./training_checkpoints'</span>
|
||||
checkpoint_prefix <span style="color: #666666">=</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(checkpoint_dir, <span style="color: #BA2121">'ckpt'</span>)
|
||||
checkpoint <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>train<span style="color: #666666">.</span>Checkpoint(generator_optimizer<span style="color: #666666">=</span>generator_optimizer,
|
||||
discriminator_optimizer<span style="color: #666666">=</span>discriminator_optimizer,
|
||||
generator<span style="color: #666666">=</span>generator,
|
||||
discriminator<span style="color: #666666">=</span>discriminator)
|
||||
</pre></div>
|
||||
<p>
|
||||
You can see that the output of every Conv2D and MaxPooling2D layer is a 3D tensor of shape (height, width, channels). The width and height dimensions tend to shrink as you go deeper in the network. The number of output channels for each Conv2D layer is controlled by the first argument (e.g., 32 or 64). Typically, as the width and height shrink, you can afford (computationally) to add more output channels in each Conv2D layer.
|
||||
Now we define our training loop
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train</span>(dataset, epochs):
|
||||
generator_loss_list <span style="color: #666666">=</span> []
|
||||
discriminator_loss_list <span style="color: #666666">=</span> []
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(epochs):
|
||||
start <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> image_batch <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
gen_loss, disc_loss <span style="color: #666666">=</span> train_step(image_batch)
|
||||
generator_loss_list<span style="color: #666666">.</span>append(gen_loss<span style="color: #666666">.</span>numpy())
|
||||
discriminator_loss_list<span style="color: #666666">.</span>append(disc_loss<span style="color: #666666">.</span>numpy())
|
||||
|
||||
<span style="color: #408080; font-style: italic">#generate_and_save_images(generator, epoch + 1, seed_images)</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> (epoch <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #666666">%</span> <span style="color: #666666">15</span> <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
||||
checkpoint<span style="color: #666666">.</span>save(file_prefix<span style="color: #666666">=</span>checkpoint_prefix)
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Time for epoch </span><span style="color: #BB6688; font-weight: bold">{</span>epoch<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> is </span><span style="color: #BB6688; font-weight: bold">{</span>time<span style="color: #666666">.</span>time() <span style="color: #666666">-</span> start<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic">#generate_and_save_images(generator, epochs, seed_images)</span>
|
||||
|
||||
loss_file <span style="color: #666666">=</span> <span style="color: #BA2121">'./data/lossfile.txt'</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(loss_file, <span style="color: #BA2121">'w'</span>) <span style="color: #008000; font-weight: bold">as</span> outfile:
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #008000">str</span>(generator_loss_list))
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #008000">str</span>(discriminator_loss_list))
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
To train simply call this function. <b>Warning</b>: this might take a long time so
|
||||
there is a folder of a pretrained network already included in the repository.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>train(train_dataset, EPOCHS)
|
||||
</pre></div>
|
||||
<p>
|
||||
And here is the result of training our model for 100 epochs
|
||||
|
||||
<p>
|
||||
|
||||
<embed src="images_from_seed_images/generation.gif" autoplay="false" loop="true"></embed>
|
||||
<p><em></em></p>
|
||||
|
||||
<p>
|
||||
Now to avoid having to train and everything, which will take a while depending
|
||||
on your computer setup we now load in the model which produced the above gif.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>checkpoint<span style="color: #666666">.</span>restore(tf<span style="color: #666666">.</span>train<span style="color: #666666">.</span>latest_checkpoint(checkpoint_dir))
|
||||
restored_generator <span style="color: #666666">=</span> checkpoint<span style="color: #666666">.</span>generator
|
||||
restored_discriminator <span style="color: #666666">=</span> checkpoint<span style="color: #666666">.</span>discriminator
|
||||
|
||||
<span style="color: #008000">print</span>(restored_generator)
|
||||
<span style="color: #008000">print</span>(restored_discriminator)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -243,14 +274,6 @@ You can see that the output of every Conv2D and MaxPooling2D layer is a 3D tenso
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="._week43-bs029.html">30</a></li>
|
||||
<li><a href="._week43-bs030.html">31</a></li>
|
||||
<li><a href="._week43-bs031.html">32</a></li>
|
||||
<li><a href="._week43-bs032.html">33</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs024.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,30 +172,55 @@ MathJax.Hub.Config({
|
||||
<a name="part0024"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="add-dense-layers-on-top" class="anchor">Add Dense layers on top </h2>
|
||||
<h2 id="exploring-the-latent-space" class="anchor">Exploring the Latent Space </h2>
|
||||
|
||||
<p>
|
||||
To complete our model, you will feed the last output tensor from the
|
||||
convolutional base (of shape (4, 4, 64)) into one or more Dense layers
|
||||
to perform classification. Dense layers take vectors as input (which
|
||||
are 1D), while the current output is a 3D tensor. First, you will
|
||||
flatten (or unroll) the 3D output to 1D, then add one or more Dense
|
||||
layers on top. CIFAR has 10 output classes, so you use a final Dense
|
||||
layer with 10 outputs and a softmax activation.
|
||||
We have successfully loaded in our latest model. Let us now play around a bit
|
||||
and see what kind of things we can learn about this model. Our generator takes
|
||||
an array of 100 numbers. One idea can be to try to systematically change our
|
||||
input. Let us try and see what we get
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Flatten())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(<span style="color: #666666">64</span>, activation<span style="color: #666666">=</span><span style="color: #BA2121">'relu'</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(<span style="color: #666666">10</span>))
|
||||
Here<span style="color: #BA2121">'s the complete architecture of our model.</span>
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_latent_points</span>(number<span style="color: #666666">=100</span>, scale_means<span style="color: #666666">=1</span>, scale_stds<span style="color: #666666">=1</span>):
|
||||
latent_dim <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
means <span style="color: #666666">=</span> scale_means <span style="color: #666666">*</span> tf<span style="color: #666666">.</span>linspace(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>, num<span style="color: #666666">=</span>latent_dim)
|
||||
stds <span style="color: #666666">=</span> scale_stds <span style="color: #666666">*</span> tf<span style="color: #666666">.</span>linspace(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>, num<span style="color: #666666">=</span>latent_dim)
|
||||
latent_space_value_range <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal([number, latent_dim],
|
||||
means,
|
||||
stds,
|
||||
dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float64)
|
||||
|
||||
model<span style="color: #666666">.</span>summary()
|
||||
<span style="color: #008000; font-weight: bold">return</span> latent_space_value_range
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_images</span>(latent_points):
|
||||
<span style="color: #408080; font-style: italic"># notice we set training to false because we are making inferences</span>
|
||||
generated_images <span style="color: #666666">=</span> restored_generator<span style="color: #666666">.</span>predict(latent_points)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> generated_images
|
||||
</pre></div>
|
||||
<p>
|
||||
As you can see, our (4, 4, 64) outputs were flattened into vectors of shape (1024) before going through two Dense layers.
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_result</span>(generated_images, number<span style="color: #666666">=100</span>):
|
||||
<span style="color: #408080; font-style: italic"># obviously this assumes sqrt number is an int</span>
|
||||
fig, axs <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(<span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number)), <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number)),
|
||||
figsize<span style="color: #666666">=</span>(<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number))):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number))):
|
||||
axs[i, j]<span style="color: #666666">.</span>imshow(generated_images[i<span style="color: #666666">*</span>j], cmap<span style="color: #666666">=</span><span style="color: #BA2121">'Greys'</span>)
|
||||
axs[i, j]<span style="color: #666666">.</span>axis(<span style="color: #BA2121">'off'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points())
|
||||
plot_result(generated_images)
|
||||
</pre></div>
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -241,15 +239,6 @@ As you can see, our (4, 4, 64) outputs were flattened into vectors of shape (102
|
||||
<li class="active"><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="._week43-bs029.html">30</a></li>
|
||||
<li><a href="._week43-bs030.html">31</a></li>
|
||||
<li><a href="._week43-bs031.html">32</a></li>
|
||||
<li><a href="._week43-bs032.html">33</a></li>
|
||||
<li><a href="._week43-bs033.html">34</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs025.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,18 +172,53 @@ MathJax.Hub.Config({
|
||||
<a name="part0025"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="compile-and-train-the-model" class="anchor">Compile and train the model </h2>
|
||||
<h2 id="getting-results" class="anchor">Getting Results </h2>
|
||||
We see that the generator generates images that look like MNIST
|
||||
numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able
|
||||
to generate a similar plot where we generate every MNIST number. Let us now try
|
||||
to 'move' a bit around in the latent space. <b>Note</b>: decrease the plot number if
|
||||
these following cells take too long to run on your computer.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>model<span style="color: #666666">.</span>compile(optimizer<span style="color: #666666">=</span><span style="color: #BA2121">'adam'</span>,
|
||||
loss<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>losses<span style="color: #666666">.</span>SparseCategoricalCrossentropy(from_logits<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>),
|
||||
metrics<span style="color: #666666">=</span>[<span style="color: #BA2121">'accuracy'</span>])
|
||||
|
||||
history <span style="color: #666666">=</span> model<span style="color: #666666">.</span>fit(train_images, train_labels, epochs<span style="color: #666666">=10</span>,
|
||||
validation_data<span style="color: #666666">=</span>(test_images, test_labels))
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plot_number <span style="color: #666666">=</span> <span style="color: #666666">225</span>
|
||||
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=5</span>,
|
||||
scale_stds<span style="color: #666666">=1</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=-5</span>,
|
||||
scale_stds<span style="color: #666666">=1</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=1</span>,
|
||||
scale_stds<span style="color: #666666">=5</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
Again, we have found something interesting. <em>Moving</em> around using our means
|
||||
takes us from digit to digit, while <em>moving</em> around using our standard
|
||||
deviations seem to increase the number of different digits! In the last image
|
||||
above, we can barely make out every MNIST digit. Let us make on last plot using
|
||||
this information by upping the standard deviation of our Gaussian noises.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plot_number <span style="color: #666666">=</span> <span style="color: #666666">400</span>
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=1</span>,
|
||||
scale_stds<span style="color: #666666">=10</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
A pretty cool result! We see that our generator indeed has learned a
|
||||
distribution which qualitatively looks a whole lot like the MNIST dataset.
|
||||
|
||||
<p>
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
@@ -228,14 +236,6 @@ history <span style="color: #666666">=</span> model<span style="color: #666666">
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li class="active"><a href="._week43-bs025.html">26</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="._week43-bs029.html">30</a></li>
|
||||
<li><a href="._week43-bs030.html">31</a></li>
|
||||
<li><a href="._week43-bs031.html">32</a></li>
|
||||
<li><a href="._week43-bs032.html">33</a></li>
|
||||
<li><a href="._week43-bs033.html">34</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs026.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -7,9 +7,9 @@ Automatically generated HTML file from DocOnce source
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<meta name="description" content="Week 43: Convolutional Neural Networks and Recurrent Neural Networks">
|
||||
<meta name="description" content="Week 43: Deep Learning: Recurrent Neural Networks and other methods">
|
||||
|
||||
<title>Week 43: Convolutional Neural Networks and Recurrent Neural Networks</title>
|
||||
<title>Week 43: Deep Learning: Recurrent Neural Networks and other methods</title>
|
||||
|
||||
<!-- Bootstrap style: bootstrap -->
|
||||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||||
@@ -42,60 +42,12 @@ Automatically generated HTML file from DocOnce source
|
||||
<!-- tocinfo
|
||||
{'highest level': 2,
|
||||
'sections': [('Plans for week 43', 2, None, 'plans-for-week-43'),
|
||||
('Convolutional Neural Networks (recognizing images)',
|
||||
('Reading Recommendations', 2, None, 'reading-recommendations'),
|
||||
('Summary on Deep Learning Methods',
|
||||
2,
|
||||
None,
|
||||
'convolutional-neural-networks-recognizing-images'),
|
||||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||||
('Why CNNS for images, sound files, medical images from CT scans '
|
||||
'etc?',
|
||||
2,
|
||||
None,
|
||||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||||
('Regular NNs don’t scale well to full images',
|
||||
2,
|
||||
None,
|
||||
'regular-nns-don-t-scale-well-to-full-images'),
|
||||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||||
('Layers used to build CNNs',
|
||||
2,
|
||||
None,
|
||||
'layers-used-to-build-cnns'),
|
||||
('Transforming images', 2, None, 'transforming-images'),
|
||||
'summary-on-deep-learning-methods'),
|
||||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||||
('CNNs in more detail, building convolutional neural networks in '
|
||||
'Tensorflow and Keras',
|
||||
2,
|
||||
None,
|
||||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||||
('Setting it up', 2, None, 'setting-it-up'),
|
||||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||||
('Strong correlations', 2, None, 'strong-correlations'),
|
||||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||||
('Prerequisites: Collect and pre-process data',
|
||||
2,
|
||||
None,
|
||||
'prerequisites-collect-and-pre-process-data'),
|
||||
('Importing Keras and Tensorflow',
|
||||
2,
|
||||
None,
|
||||
'importing-keras-and-tensorflow'),
|
||||
('Running with Keras', 2, None, 'running-with-keras'),
|
||||
('Final part', 2, None, 'final-part'),
|
||||
('Final visualization', 2, None, 'final-visualization'),
|
||||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||||
('Set up the model', 2, None, 'set-up-the-model'),
|
||||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||||
('Compile and train the model',
|
||||
2,
|
||||
None,
|
||||
'compile-and-train-the-model'),
|
||||
('Finally, evaluate the model',
|
||||
2,
|
||||
None,
|
||||
'finally-evaluate-the-model'),
|
||||
('Recurrent neural networks: Overarching view',
|
||||
2,
|
||||
None,
|
||||
@@ -112,7 +64,36 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -142,7 +123,7 @@ MathJax.Hub.Config({
|
||||
<span class="icon-bar"></span>
|
||||
<span class="icon-bar"></span>
|
||||
</button>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Convolutional Neural Networks and Recurrent Neural Networks</a>
|
||||
<a class="navbar-brand" href="week43-bs.html">Week 43: Deep Learning: Recurrent Neural Networks and other methods</a>
|
||||
</div>
|
||||
|
||||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||||
@@ -151,39 +132,31 @@ MathJax.Hub.Config({
|
||||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||||
<ul class="dropdown-menu">
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs001.html#plans-for-week-43" style="font-size: 80%;">Plans for week 43</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;">Convolutional Neural Networks (recognizing images)</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#neural-networks-vs-cnns" style="font-size: 80%;">Neural Networks vs CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;">Why CNNS for images, sound files, medical images from CT scans etc?</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;">Regular NNs don’t scale well to full images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#3d-volumes-of-neurons" style="font-size: 80%;">3D volumes of neurons</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#layers-used-to-build-cnns" style="font-size: 80%;">Layers used to build CNNs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#transforming-images" style="font-size: 80%;">Transforming images</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;">CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#setting-it-up" style="font-size: 80%;">Setting it up</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#the-mnist-dataset-again" style="font-size: 80%;">The MNIST dataset again</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#strong-correlations" style="font-size: 80%;">Strong correlations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#layers-of-a-cnn" style="font-size: 80%;">Layers of a CNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#systematic-reduction" style="font-size: 80%;">Systematic reduction</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;">Prerequisites: Collect and pre-process data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#importing-keras-and-tensorflow" style="font-size: 80%;">Importing Keras and Tensorflow</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#running-with-keras" style="font-size: 80%;">Running with Keras</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#final-part" style="font-size: 80%;">Final part</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#final-visualization" style="font-size: 80%;">Final visualization</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#the-cifar01-data-set" style="font-size: 80%;">The CIFAR01 data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#verifying-the-data-set" style="font-size: 80%;">Verifying the data set</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#set-up-the-model" style="font-size: 80%;">Set up the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#add-dense-layers-on-top" style="font-size: 80%;">Add Dense layers on top</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#compile-and-train-the-model" style="font-size: 80%;">Compile and train the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#finally-evaluate-the-model" style="font-size: 80%;">Finally, evaluate the model</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs027.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs028.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs029.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs030.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs031.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs032.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs033.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs034.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs002.html#reading-recommendations" style="font-size: 80%;">Reading Recommendations</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs003.html#summary-on-deep-learning-methods" style="font-size: 80%;">Summary on Deep Learning Methods</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs004.html#cnns-in-brief" style="font-size: 80%;">CNNs in brief</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs005.html#recurrent-neural-networks-overarching-view" style="font-size: 80%;">Recurrent neural networks: Overarching view</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs006.html#set-up-of-an-rnn" style="font-size: 80%;">Set up of an RNN</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs007.html#a-simple-example" style="font-size: 80%;">A simple example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs008.html#an-extrapolation-example" style="font-size: 80%;">An extrapolation example</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs009.html#formatting-the-data" style="font-size: 80%;">Formatting the Data</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -199,23 +172,49 @@ MathJax.Hub.Config({
|
||||
<a name="part0026"></a>
|
||||
<!-- !split -->
|
||||
|
||||
<h2 id="finally-evaluate-the-model" class="anchor">Finally, evaluate the model </h2>
|
||||
<h2 id="interpolating-between-mnist-digits" class="anchor">Interpolating Between MNIST Digits </h2>
|
||||
Another interesting way to explore the latent space of our generator model is by
|
||||
interpolating between the MNIST digits. This section is largely based on
|
||||
<a href="https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/" target="_self">this excellent blogpost</a>
|
||||
by Jason Brownlee.
|
||||
|
||||
<p>
|
||||
So let us start by defining a function to interpolate between two points in the
|
||||
latent space.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plt<span style="color: #666666">.</span>plot(history<span style="color: #666666">.</span>history[<span style="color: #BA2121">'accuracy'</span>], label<span style="color: #666666">=</span><span style="color: #BA2121">'accuracy'</span>)
|
||||
plt<span style="color: #666666">.</span>plot(history<span style="color: #666666">.</span>history[<span style="color: #BA2121">'val_accuracy'</span>], label <span style="color: #666666">=</span> <span style="color: #BA2121">'val_accuracy'</span>)
|
||||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">'Epoch'</span>)
|
||||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">'Accuracy'</span>)
|
||||
plt<span style="color: #666666">.</span>ylim([<span style="color: #666666">0.5</span>, <span style="color: #666666">1</span>])
|
||||
plt<span style="color: #666666">.</span>legend(loc<span style="color: #666666">=</span><span style="color: #BA2121">'lower right'</span>)
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">interpolation</span>(point_1, point_2, n_steps<span style="color: #666666">=10</span>):
|
||||
ratios <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, num<span style="color: #666666">=</span>n_steps)
|
||||
vectors <span style="color: #666666">=</span> []
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, ratio <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(ratios):
|
||||
vectors<span style="color: #666666">.</span>append(((<span style="color: #666666">1.0</span> <span style="color: #666666">-</span> ratio) <span style="color: #666666">*</span> point_1 <span style="color: #666666">+</span> ratio <span style="color: #666666">*</span> point_2))
|
||||
|
||||
test_loss, test_acc <span style="color: #666666">=</span> model<span style="color: #666666">.</span>evaluate(test_images, test_labels, verbose<span style="color: #666666">=2</span>)
|
||||
|
||||
<span style="color: #008000">print</span>(test_acc)
|
||||
<span style="color: #008000; font-weight: bold">return</span> tf<span style="color: #666666">.</span>stack(vectors)
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we have all we need to do our interpolation analysis.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plot_number <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
latent_points <span style="color: #666666">=</span> generate_latent_points(number<span style="color: #666666">=</span>plot_number)
|
||||
results <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">0</span>, <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>sqrt(plot_number), <span style="color: #666666">2</span>):
|
||||
interpolated <span style="color: #666666">=</span> interpolation(latent_points[i], latent_points[i<span style="color: #666666">+1</span>])
|
||||
generated_images <span style="color: #666666">=</span> generate_images(interpolated)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> results <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||||
results <span style="color: #666666">=</span> generated_images
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
results <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>stack((results, generated_images))
|
||||
|
||||
plot_results(results, plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<p>
|
||||
<!-- navigation buttons at the bottom of the page -->
|
||||
<ul class="pagination">
|
||||
@@ -231,15 +230,6 @@ test_loss, test_acc <span style="color: #666666">=</span> model<span style="colo
|
||||
<li><a href="._week43-bs024.html">25</a></li>
|
||||
<li><a href="._week43-bs025.html">26</a></li>
|
||||
<li class="active"><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs027.html">28</a></li>
|
||||
<li><a href="._week43-bs028.html">29</a></li>
|
||||
<li><a href="._week43-bs029.html">30</a></li>
|
||||
<li><a href="._week43-bs030.html">31</a></li>
|
||||
<li><a href="._week43-bs031.html">32</a></li>
|
||||
<li><a href="._week43-bs032.html">33</a></li>
|
||||
<li><a href="._week43-bs033.html">34</a></li>
|
||||
<li><a href="._week43-bs034.html">35</a></li>
|
||||
<li><a href="._week43-bs027.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -64,11 +64,56 @@ Automatically generated HTML file from DocOnce source
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- Bootstrap navigation bar -->
|
||||
<div class="navbar navbar-default navbar-fixed-top">
|
||||
@@ -98,6 +143,20 @@ end of tocinfo -->
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs010.html#predicting-new-points-with-a-trained-recurrent-neural-network" style="font-size: 80%;">Predicting New Points With A Trained Recurrent Neural Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs011.html#other-things-to-try" style="font-size: 80%;">Other Things to Try</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs012.html#other-types-of-recurrent-neural-networks" style="font-size: 80%;">Other Types of Recurrent Neural Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs013.html#generative-models" style="font-size: 80%;">Generative Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs014.html#generative-adversarial-networks" style="font-size: 80%;">Generative Adversarial Networks</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs015.html#discriminator" style="font-size: 80%;">Discriminator</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs016.html#learning-process" style="font-size: 80%;">Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs017.html#more-about-the-learning-process" style="font-size: 80%;">More about the Learning Process</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs018.html#additional-references" style="font-size: 80%;">Additional References</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs019.html#writing-our-first-generative-adversarial-network" style="font-size: 80%;">Writing Our First Generative Adversarial Network</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs020.html#mnist-and-gans" style="font-size: 80%;">MNIST and GANs</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs021.html#other-models" style="font-size: 80%;">Other Models</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs022.html#training-step" style="font-size: 80%;">Training Step</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs023.html#checkpoints" style="font-size: 80%;">Checkpoints</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs024.html#exploring-the-latent-space" style="font-size: 80%;">Exploring the Latent Space</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs025.html#getting-results" style="font-size: 80%;">Getting Results</a></li>
|
||||
<!-- navigation toc: --> <li><a href="._week43-bs026.html#interpolating-between-mnist-digits" style="font-size: 80%;">Interpolating Between MNIST Digits</a></li>
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
@@ -156,7 +215,7 @@ end of tocinfo -->
|
||||
<li><a href="._week43-bs008.html">9</a></li>
|
||||
<li><a href="._week43-bs009.html">10</a></li>
|
||||
<li><a href="">...</a></li>
|
||||
<li><a href="._week43-bs012.html">13</a></li>
|
||||
<li><a href="._week43-bs026.html">27</a></li>
|
||||
<li><a href="._week43-bs001.html">»</a></li>
|
||||
</ul>
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
@@ -107,6 +107,22 @@ td.padding {
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "none" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -242,14 +258,14 @@ fixed-sized inputs like all the nets we have discussed so far. For
|
||||
example, they can take sentences, documents, or audio samples as
|
||||
input, making them extremely useful for natural language processing
|
||||
systems such as automatic translation and speech-to-text.
|
||||
|
||||
<p>
|
||||
RNNs are also used in "text classification":"Text Classification Algorithms: A Survey"
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="set-up-of-an-rnn">Set up of an RNN </h2>
|
||||
|
||||
<p>
|
||||
More to text to be added by Wednesday October 27.
|
||||
</section>
|
||||
|
||||
|
||||
@@ -918,6 +934,717 @@ end = timer()
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="generative-models">Generative Models </h2>
|
||||
|
||||
<p>
|
||||
<b>Generative models</b> describe a class of statistical models that are a contrast
|
||||
to <b>discriminative models</b>. Informally we say that generative models can
|
||||
generate new data instances while discriminative models discriminate between
|
||||
different kinds of data instances. A generative model could generate new photos
|
||||
of animals that look like 'real' animals while a discriminative model could tell
|
||||
a dog from a cat. More formally, given a data set \( x \) and a set of labels /
|
||||
targets \( y \). Generative models capture the joint probability \( p(x, y) \), or
|
||||
just \( p(x) \) if there are no labels, while discriminative models capture the
|
||||
conditional probability \( p(y | x) \). Discriminative models generally try to draw
|
||||
boundaries in the data space (often high dimensional), while generative models
|
||||
try to model how data is placed throughout the space.
|
||||
|
||||
<p>
|
||||
<b>Note</b>: this material is thanks to Linus Ekstrøm.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="generative-adversarial-networks">Generative Adversarial Networks </h2>
|
||||
|
||||
<p>
|
||||
<b>Generative Adversarial Networks</b> are a type of unsupervised machine learning
|
||||
algorithm proposed by <a href="https://arxiv.org/pdf/1406.2661.pdf" target="_blank">Goodfellow et. al</a>
|
||||
in 2014 (short and good article).
|
||||
|
||||
<p>
|
||||
The simplest formulation of
|
||||
the model is based on a game theoretic approach, <em>zero sum game</em>, where we pit
|
||||
two neural networks against one another. We define two rival networks, one
|
||||
generator \( g \), and one discriminator \( d \). The generator directly produces
|
||||
samples
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
x = g(z; \theta^{(g)})
|
||||
\tag{1}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="discriminator">Discriminator </h2>
|
||||
The discriminator attempts to distinguish between samples drawn from the
|
||||
training data and samples drawn from the generator. In other words, it tries to
|
||||
tell the difference between the fake data produced by \( g \) and the actual data
|
||||
samples we want to do prediction on. The discriminator outputs a probability
|
||||
value given by
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
d(x; \theta^{(d)})
|
||||
\tag{2}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
indicating the probability that \( x \) is a real training example rather than a
|
||||
fake sample the generator has generated. The simplest way to formulate the
|
||||
learning process in a generative adversarial network is a zero-sum game, in
|
||||
which a function
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{3}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
<p>
|
||||
determines the reward for the discriminator, while the generator gets the
|
||||
conjugate reward
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
-v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{4}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="learning-process">Learning Process </h2>
|
||||
|
||||
<p>
|
||||
During learning both of the networks maximize their own reward function, so that
|
||||
the generator gets better and better at tricking the discriminator, while the
|
||||
discriminator gets better and better at telling the difference between the fake
|
||||
and real data. The generator and discriminator alternate on which one trains at
|
||||
one time (i.e. for one epoch). In other words, we keep the generator constant
|
||||
and train the discriminator, then we keep the discriminator constant to train
|
||||
the generator and repeat. It is this back and forth dynamic which lets GANs
|
||||
tackle otherwise intractable generative problems. As the generator improves with
|
||||
training, the discriminator's performance gets worse because it cannot easily
|
||||
tell the difference between real and fake. If the generator ends up succeeding
|
||||
perfectly, the the discriminator will do no better than random guessing i.e.
|
||||
50\%. This progression in the training poses a problem for the convergence
|
||||
criteria for GANs. The discriminator feedback gets less meaningful over time,
|
||||
if we continue training after this point then the generator is effectively
|
||||
training on junk data which can undo the learning up to that point. Therefore,
|
||||
we stop training when the discriminator starts outputting \( 1/2 \) everywhere.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="more-about-the-learning-process">More about the Learning Process </h2>
|
||||
|
||||
<p>
|
||||
At convergence we have
|
||||
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{5}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
The default choice for \( v \) is
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x)
|
||||
+ \mathbb{E}_{x\sim p_\mathrm{model}}
|
||||
\log (1 - d(x))
|
||||
\tag{6}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
The main motivation for the design of GANs is that the learning process requires
|
||||
neither approximate inference (variational autoencoders for example) nor
|
||||
approximation of a partition function. In the case where
|
||||
<p> <br>
|
||||
$$
|
||||
\begin{equation}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\tag{7}
|
||||
\end{equation}
|
||||
$$
|
||||
<p> <br>
|
||||
|
||||
is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is
|
||||
asymptotically consistent
|
||||
( <a href="https://arxiv.org/pdf/1804.09139.pdf" target="_blank">Seth Lloyd on QuGANs</a> ).
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="additional-references">Additional References </h2>
|
||||
This is in
|
||||
general not the case and it is possible to get situations where the training
|
||||
process never converges because the generator and discriminator chase one
|
||||
another around in the parameter space indefinitely. A much deeper discussion on
|
||||
the currently open research problem of GAN convergence is available
|
||||
<a href="https://www.deeplearningbook.org/contents/generative_models.html" target="_blank">here</a>. To
|
||||
anyone interested in learning more about GANs it is a highly recommended read.
|
||||
Direct quote: "In this best-performing formulation, the generator aims to
|
||||
increase the log probability that the discriminator makes a mistake, rather than
|
||||
aiming to decrease the log probability that the discriminator makes the correct
|
||||
prediction." <a href="https://arxiv.org/abs/1701.00160" target="_blank">Another interesting read</a>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="writing-our-first-generative-adversarial-network">Writing Our First Generative Adversarial Network </h2>
|
||||
Let us now move on to actually implementing a GAN in tensorflow. We will study
|
||||
the performance of our GAN on the MNIST dataset. This code is based on and
|
||||
adapted from the
|
||||
<a href="https://www.tensorflow.org/tutorials/generative/dcgan" target="_blank">google tutorial</a>
|
||||
|
||||
<p>
|
||||
First we import our libraries
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">time</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> layers
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.utils</span> <span style="color: #8B008B; font-weight: bold">import</span> plot_model
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define our hyperparameters and import our data the usual way
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>BUFFER_SIZE = <span style="color: #B452CD">60000</span>
|
||||
BATCH_SIZE = <span style="color: #B452CD">256</span>
|
||||
EPOCHS = <span style="color: #B452CD">30</span>
|
||||
|
||||
data = tf.keras.datasets.mnist.load_data()
|
||||
(train_images, train_labels), (test_images, test_labels) = data
|
||||
train_images = np.reshape(train_images, (train_images.shape[<span style="color: #B452CD">0</span>],
|
||||
<span style="color: #B452CD">28</span>,
|
||||
<span style="color: #B452CD">28</span>,
|
||||
<span style="color: #B452CD">1</span>)).astype(<span style="color: #CD5555">'float32'</span>)
|
||||
|
||||
<span style="color: #228B22"># we normalize between -1 and 1</span>
|
||||
train_images = (train_images - <span style="color: #B452CD">127.5</span>) / <span style="color: #B452CD">127.5</span>
|
||||
training_dataset = tf.data.Dataset.from_tensor_slices(
|
||||
train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="mnist-and-gans">MNIST and GANs </h2>
|
||||
|
||||
<p>
|
||||
Let's have a quick look
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>plt.imshow(train_images[<span style="color: #B452CD">0</span>], cmap=<span style="color: #CD5555">'Greys'</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we define our two models. This is where the 'magic' happens. There are a
|
||||
huge amount of possible formulations for both models. A lot of engineering and
|
||||
trial and error can be done here to try to produce better performing models. For
|
||||
more advanced GANs this is by far the step where you can 'make or break' a
|
||||
model.
|
||||
|
||||
<p>
|
||||
We start with the generator. As stated in the introductory text the generator
|
||||
\( g \) upsamples from a random sample to the shape of what we want to predict. In
|
||||
our case we are trying to predict MNIST images (\( 28\times 28 \) pixels).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generator_model</span>():
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to</span>
|
||||
<span style="color: #CD5555"> produce an image from a random seed. We start with a Dense layer taking this</span>
|
||||
<span style="color: #CD5555"> random sample as an input and subsequently upsample through multiple</span>
|
||||
<span style="color: #CD5555"> convolutional layers.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
<span style="color: #228B22"># we define our model</span>
|
||||
model = tf.keras.Sequential()
|
||||
|
||||
|
||||
<span style="color: #228B22"># adding our input layer. Dense means that every neuron is connected and</span>
|
||||
<span style="color: #228B22"># the input shape is the shape of our random noise. The units need to match</span>
|
||||
<span style="color: #228B22"># in some sense the upsampling strides to reach our desired output shape.</span>
|
||||
<span style="color: #228B22"># we are using 100 random numbers as our seed</span>
|
||||
model.add(layers.Dense(units=<span style="color: #B452CD">7</span>*<span style="color: #B452CD">7</span>*BATCH_SIZE,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
input_shape=(<span style="color: #B452CD">100</span>, )))
|
||||
<span style="color: #228B22"># we normalize the output form the Dense layer</span>
|
||||
model.add(layers.BatchNormalization())
|
||||
<span style="color: #228B22"># and add an activation function to our 'layer'. LeakyReLU avoids vanishing</span>
|
||||
<span style="color: #228B22"># gradient problem</span>
|
||||
model.add(layers.LeakyReLU())
|
||||
model.add(layers.Reshape((<span style="color: #B452CD">7</span>, <span style="color: #B452CD">7</span>, BATCH_SIZE)))
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">7</span>, <span style="color: #B452CD">7</span>, BATCH_SIZE)
|
||||
<span style="color: #228B22"># even though we just added four keras layers we think of everything above</span>
|
||||
<span style="color: #228B22"># as 'one' layer</span>
|
||||
|
||||
<span style="color: #228B22"># next we add our upscaling convolutional layers</span>
|
||||
model.add(layers.Conv2DTranspose(filters=<span style="color: #B452CD">128</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>))
|
||||
model.add(layers.BatchNormalization())
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">7</span>, <span style="color: #B452CD">7</span>, <span style="color: #B452CD">128</span>)
|
||||
|
||||
model.add(layers.Conv2DTranspose(filters=<span style="color: #B452CD">64</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>))
|
||||
model.add(layers.BatchNormalization())
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">14</span>, <span style="color: #B452CD">14</span>, <span style="color: #B452CD">64</span>)
|
||||
|
||||
model.add(layers.Conv2DTranspose(filters=<span style="color: #B452CD">1</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
activation=<span style="color: #CD5555">'tanh'</span>))
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">28</span>, <span style="color: #B452CD">28</span>, <span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
And there we have our 'simple' generator model. Now we move on to defining our
|
||||
discriminator model \( d \), which is a convolutional neural network based image
|
||||
classifier.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">discriminator_model</span>():
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> The discriminator is a convolutional neural network based image classifier</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
<span style="color: #228B22"># we define our model</span>
|
||||
model = tf.keras.Sequential()
|
||||
model.add(layers.Conv2D(filters=<span style="color: #B452CD">64</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
input_shape=[<span style="color: #B452CD">28</span>, <span style="color: #B452CD">28</span>, <span style="color: #B452CD">1</span>]))
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #228B22"># adding a dropout layer as you do in conv-nets</span>
|
||||
model.add(layers.Dropout(<span style="color: #B452CD">0.3</span>))
|
||||
|
||||
|
||||
model.add(layers.Conv2D(filters=<span style="color: #B452CD">128</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>))
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #228B22"># adding a dropout layer as you do in conv-nets</span>
|
||||
model.add(layers.Dropout(<span style="color: #B452CD">0.3</span>))
|
||||
|
||||
model.add(layers.Flatten())
|
||||
model.add(layers.Dense(<span style="color: #B452CD">1</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="other-models">Other Models </h2>
|
||||
Let us take a look at our models. <b>Note</b>: double click images for bigger view.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>generator = generator_model()
|
||||
plot_model(generator, show_shapes=<span style="color: #8B008B; font-weight: bold">True</span>, rankdir=<span style="color: #CD5555">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>discriminator = discriminator_model()
|
||||
plot_model(discriminator, show_shapes=<span style="color: #8B008B; font-weight: bold">True</span>, rankdir=<span style="color: #CD5555">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we need a few helper objects we will use in training
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
generator_optimizer = tf.keras.optimizers.Adam(<span style="color: #B452CD">1e-4</span>)
|
||||
discriminator_optimizer = tf.keras.optimizers.Adam(<span style="color: #B452CD">1e-4</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
The first object, <em>cross_entropy</em> is our loss function and the two others are
|
||||
our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This
|
||||
is because they need to improve their accuracy at approximately equal speeds to
|
||||
get convergence (not necessarily exactly equal). Now we define our loss
|
||||
functions
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generator_loss</span>(fake_output):
|
||||
loss = cross_entropy(tf.ones_like(fake_output), fake_output)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> loss
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">discriminator_loss</span>(real_output, fake_output):
|
||||
real_loss = cross_entropy(tf.ones_like(real_output), real_output)
|
||||
fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
|
||||
total_loss = real_loss + fake_loss
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> total_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a kind of seed to help us compare the learning process over
|
||||
multiple training epochs.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>noise_dimension = <span style="color: #B452CD">100</span>
|
||||
n_examples_to_generate = <span style="color: #B452CD">16</span>
|
||||
seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="training-step">Training Step </h2>
|
||||
|
||||
<p>
|
||||
Now we have everything we need to define our training step, which we will apply
|
||||
for every step in our training loop. Notice the @tf.function flag signifying
|
||||
that the function is tensorflow 'compiled'. Removing this flag doubles the
|
||||
computation time.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #707a7c">@tf</span>.function
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">train_step</span>(images):
|
||||
noise = tf.random.normal([BATCH_SIZE, noise_dimension])
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> tf.GradientTape() <span style="color: #8B008B; font-weight: bold">as</span> gen_tape, tf.GradientTape() <span style="color: #8B008B; font-weight: bold">as</span> disc_tape:
|
||||
generated_images = generator(noise, training=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
|
||||
real_output = discriminator(images, training=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
fake_output = discriminator(generated_images, training=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
|
||||
gen_loss = generator_loss(fake_output)
|
||||
disc_loss = discriminator_loss(real_output, fake_output)
|
||||
|
||||
gradients_of_generator = gen_tape.gradient(gen_loss,
|
||||
generator.trainable_variables)
|
||||
gradients_of_discriminator = disc_tape.gradient(disc_loss,
|
||||
discriminator.trainable_variables)
|
||||
generator_optimizer.apply_gradients(<span style="color: #658b00">zip</span>(gradients_of_generator,
|
||||
generator.trainable_variables))
|
||||
discriminator_optimizer.apply_gradients(<span style="color: #658b00">zip</span>(gradients_of_discriminator,
|
||||
discriminator.trainable_variables))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> gen_loss, disc_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a helper function to produce an output over our training epochs
|
||||
to see the predictive progression of our generator model. <b>Note</b>: I am including
|
||||
this code here, but comment it out in the training loop.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_and_save_images</span>(model, epoch, test_input):
|
||||
<span style="color: #228B22"># we're making inferences here</span>
|
||||
predictions = model(test_input, training=<span style="color: #8B008B; font-weight: bold">False</span>)
|
||||
|
||||
fig = plt.figure(figsize=(<span style="color: #B452CD">4</span>, <span style="color: #B452CD">4</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(predictions.shape[<span style="color: #B452CD">0</span>]):
|
||||
plt.subplot(<span style="color: #B452CD">4</span>, <span style="color: #B452CD">4</span>, i+<span style="color: #B452CD">1</span>)
|
||||
plt.imshow(predictions[i, :, :, <span style="color: #B452CD">0</span>] * <span style="color: #B452CD">127.5</span> + <span style="color: #B452CD">127.5</span>, cmap=<span style="color: #CD5555">'gray'</span>)
|
||||
plt.axis(<span style="color: #CD5555">'off'</span>)
|
||||
|
||||
plt.savefig(<span style="color: #CD5555">f'./images_from_seed_images/image_at_epoch_{</span><span style="color: #658b00">str</span>(epoch).zfill(<span style="color: #B452CD">3</span>)<span style="color: #CD5555">}.png'</span>)
|
||||
plt.close()
|
||||
<span style="color: #228B22">#plt.show()</span>
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="checkpoints">Checkpoints </h2>
|
||||
Setting up checkpoints to periodically save our model during training so that
|
||||
everything is not lost even if the program were to somehow terminate while
|
||||
training.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #228B22"># Setting up checkpoints to save model during training</span>
|
||||
checkpoint_dir = <span style="color: #CD5555">'./training_checkpoints'</span>
|
||||
checkpoint_prefix = os.path.join(checkpoint_dir, <span style="color: #CD5555">'ckpt'</span>)
|
||||
checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
|
||||
discriminator_optimizer=discriminator_optimizer,
|
||||
generator=generator,
|
||||
discriminator=discriminator)
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we define our training loop
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">train</span>(dataset, epochs):
|
||||
generator_loss_list = []
|
||||
discriminator_loss_list = []
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(epochs):
|
||||
start = time.time()
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> image_batch <span style="color: #8B008B">in</span> dataset:
|
||||
gen_loss, disc_loss = train_step(image_batch)
|
||||
generator_loss_list.append(gen_loss.numpy())
|
||||
discriminator_loss_list.append(disc_loss.numpy())
|
||||
|
||||
<span style="color: #228B22">#generate_and_save_images(generator, epoch + 1, seed_images)</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> (epoch + <span style="color: #B452CD">1</span>) % <span style="color: #B452CD">15</span> == <span style="color: #B452CD">0</span>:
|
||||
checkpoint.save(file_prefix=checkpoint_prefix)
|
||||
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Time for epoch {</span>epoch<span style="color: #CD5555">} is {</span>time.time() - start<span style="color: #CD5555">}'</span>)
|
||||
|
||||
<span style="color: #228B22">#generate_and_save_images(generator, epochs, seed_images)</span>
|
||||
|
||||
loss_file = <span style="color: #CD5555">'./data/lossfile.txt'</span>
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> <span style="color: #658b00">open</span>(loss_file, <span style="color: #CD5555">'w'</span>) <span style="color: #8B008B; font-weight: bold">as</span> outfile:
|
||||
outfile.write(<span style="color: #658b00">str</span>(generator_loss_list))
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
outfile.write(<span style="color: #658b00">str</span>(discriminator_loss_list))
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
To train simply call this function. <b>Warning</b>: this might take a long time so
|
||||
there is a folder of a pretrained network already included in the repository.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>train(train_dataset, EPOCHS)
|
||||
</pre></div>
|
||||
<p>
|
||||
And here is the result of training our model for 100 epochs
|
||||
|
||||
<p>
|
||||
|
||||
<embed src="images_from_seed_images/generation.gif" autoplay="false" loop="true"></embed>
|
||||
<p><em></em></p>
|
||||
|
||||
<p>
|
||||
Now to avoid having to train and everything, which will take a while depending
|
||||
on your computer setup we now load in the model which produced the above gif.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
|
||||
restored_generator = checkpoint.generator
|
||||
restored_discriminator = checkpoint.discriminator
|
||||
|
||||
<span style="color: #658b00">print</span>(restored_generator)
|
||||
<span style="color: #658b00">print</span>(restored_discriminator)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="exploring-the-latent-space">Exploring the Latent Space </h2>
|
||||
|
||||
<p>
|
||||
We have successfully loaded in our latest model. Let us now play around a bit
|
||||
and see what kind of things we can learn about this model. Our generator takes
|
||||
an array of 100 numbers. One idea can be to try to systematically change our
|
||||
input. Let us try and see what we get
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_latent_points</span>(number=<span style="color: #B452CD">100</span>, scale_means=<span style="color: #B452CD">1</span>, scale_stds=<span style="color: #B452CD">1</span>):
|
||||
latent_dim = <span style="color: #B452CD">100</span>
|
||||
means = scale_means * tf.linspace(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, num=latent_dim)
|
||||
stds = scale_stds * tf.linspace(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, num=latent_dim)
|
||||
latent_space_value_range = tf.random.normal([number, latent_dim],
|
||||
means,
|
||||
stds,
|
||||
dtype=tf.float64)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> latent_space_value_range
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_images</span>(latent_points):
|
||||
<span style="color: #228B22"># notice we set training to false because we are making inferences</span>
|
||||
generated_images = restored_generator.predict(latent_points)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> generated_images
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_result</span>(generated_images, number=<span style="color: #B452CD">100</span>):
|
||||
<span style="color: #228B22"># obviously this assumes sqrt number is an int</span>
|
||||
fig, axs = plt.subplots(<span style="color: #658b00">int</span>(np.sqrt(number)), <span style="color: #658b00">int</span>(np.sqrt(number)),
|
||||
figsize=(<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">int</span>(np.sqrt(number))):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">int</span>(np.sqrt(number))):
|
||||
axs[i, j].imshow(generated_images[i*j], cmap=<span style="color: #CD5555">'Greys'</span>)
|
||||
axs[i, j].axis(<span style="color: #CD5555">'off'</span>)
|
||||
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>generated_images = generate_images(generate_latent_points())
|
||||
plot_result(generated_images)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="getting-results">Getting Results </h2>
|
||||
We see that the generator generates images that look like MNIST
|
||||
numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able
|
||||
to generate a similar plot where we generate every MNIST number. Let us now try
|
||||
to 'move' a bit around in the latent space. <b>Note</b>: decrease the plot number if
|
||||
these following cells take too long to run on your computer.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>plot_number = <span style="color: #B452CD">225</span>
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=<span style="color: #B452CD">5</span>,
|
||||
scale_stds=<span style="color: #B452CD">1</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=-<span style="color: #B452CD">5</span>,
|
||||
scale_stds=<span style="color: #B452CD">1</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=<span style="color: #B452CD">1</span>,
|
||||
scale_stds=<span style="color: #B452CD">5</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
Again, we have found something interesting. <em>Moving</em> around using our means
|
||||
takes us from digit to digit, while <em>moving</em> around using our standard
|
||||
deviations seem to increase the number of different digits! In the last image
|
||||
above, we can barely make out every MNIST digit. Let us make on last plot using
|
||||
this information by upping the standard deviation of our Gaussian noises.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>plot_number = <span style="color: #B452CD">400</span>
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=<span style="color: #B452CD">1</span>,
|
||||
scale_stds=<span style="color: #B452CD">10</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
A pretty cool result! We see that our generator indeed has learned a
|
||||
distribution which qualitatively looks a whole lot like the MNIST dataset.
|
||||
</section>
|
||||
|
||||
|
||||
<section>
|
||||
<h2 id="interpolating-between-mnist-digits">Interpolating Between MNIST Digits </h2>
|
||||
Another interesting way to explore the latent space of our generator model is by
|
||||
interpolating between the MNIST digits. This section is largely based on
|
||||
<a href="https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/" target="_blank">this excellent blogpost</a>
|
||||
by Jason Brownlee.
|
||||
|
||||
<p>
|
||||
So let us start by defining a function to interpolate between two points in the
|
||||
latent space.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">interpolation</span>(point_1, point_2, n_steps=<span style="color: #B452CD">10</span>):
|
||||
ratios = np.linspace(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, num=n_steps)
|
||||
vectors = []
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i, ratio <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(ratios):
|
||||
vectors.append(((<span style="color: #B452CD">1.0</span> - ratio) * point_1 + ratio * point_2))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> tf.stack(vectors)
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we have all we need to do our interpolation analysis.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="font-size: 80%; line-height: 125%;"><span></span>plot_number = <span style="color: #B452CD">100</span>
|
||||
latent_points = generate_latent_points(number=plot_number)
|
||||
results = <span style="color: #8B008B; font-weight: bold">None</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">2</span>*np.sqrt(plot_number), <span style="color: #B452CD">2</span>):
|
||||
interpolated = interpolation(latent_points[i], latent_points[i+<span style="color: #B452CD">1</span>])
|
||||
generated_images = generate_images(interpolated)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> results <span style="color: #8B008B">is</span> <span style="color: #8B008B; font-weight: bold">None</span>:
|
||||
results = generated_images
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
results = tf.stack((results, generated_images))
|
||||
|
||||
plot_results(results, plot_number)
|
||||
</pre></div>
|
||||
</section>
|
||||
|
||||
|
||||
|
||||
</div> <!-- class="slides" -->
|
||||
</div> <!-- class="reveal" -->
|
||||
|
||||
@@ -84,11 +84,56 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "AMS" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
|
||||
@@ -218,14 +263,14 @@ example, they can take sentences, documents, or audio samples as
|
||||
input, making them extremely useful for natural language processing
|
||||
systems such as automatic translation and speech-to-text.
|
||||
|
||||
<p>
|
||||
RNNs are also used in "text classification":"Text Classification Algorithms: A Survey"
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="set-up-of-an-rnn">Set up of an RNN </h2>
|
||||
|
||||
<p>
|
||||
More to text to be added by Wednesday October 27.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
@@ -886,6 +931,696 @@ end = timer()
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Time: '</span>, end-start)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="generative-models">Generative Models </h2>
|
||||
|
||||
<p>
|
||||
<b>Generative models</b> describe a class of statistical models that are a contrast
|
||||
to <b>discriminative models</b>. Informally we say that generative models can
|
||||
generate new data instances while discriminative models discriminate between
|
||||
different kinds of data instances. A generative model could generate new photos
|
||||
of animals that look like 'real' animals while a discriminative model could tell
|
||||
a dog from a cat. More formally, given a data set \( x \) and a set of labels /
|
||||
targets \( y \). Generative models capture the joint probability \( p(x, y) \), or
|
||||
just \( p(x) \) if there are no labels, while discriminative models capture the
|
||||
conditional probability \( p(y | x) \). Discriminative models generally try to draw
|
||||
boundaries in the data space (often high dimensional), while generative models
|
||||
try to model how data is placed throughout the space.
|
||||
|
||||
<p>
|
||||
<b>Note</b>: this material is thanks to Linus Ekstrøm.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="generative-adversarial-networks">Generative Adversarial Networks </h2>
|
||||
|
||||
<p>
|
||||
<b>Generative Adversarial Networks</b> are a type of unsupervised machine learning
|
||||
algorithm proposed by <a href="https://arxiv.org/pdf/1406.2661.pdf" target="_blank">Goodfellow et. al</a>
|
||||
in 2014 (short and good article).
|
||||
|
||||
<p>
|
||||
The simplest formulation of
|
||||
the model is based on a game theoretic approach, <em>zero sum game</em>, where we pit
|
||||
two neural networks against one another. We define two rival networks, one
|
||||
generator \( g \), and one discriminator \( d \). The generator directly produces
|
||||
samples
|
||||
$$
|
||||
\begin{equation}
|
||||
x = g(z; \theta^{(g)})
|
||||
\label{_auto1}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="discriminator">Discriminator </h2>
|
||||
The discriminator attempts to distinguish between samples drawn from the
|
||||
training data and samples drawn from the generator. In other words, it tries to
|
||||
tell the difference between the fake data produced by \( g \) and the actual data
|
||||
samples we want to do prediction on. The discriminator outputs a probability
|
||||
value given by
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
d(x; \theta^{(d)})
|
||||
\label{_auto2}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
indicating the probability that \( x \) is a real training example rather than a
|
||||
fake sample the generator has generated. The simplest way to formulate the
|
||||
learning process in a generative adversarial network is a zero-sum game, in
|
||||
which a function
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto3}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
determines the reward for the discriminator, while the generator gets the
|
||||
conjugate reward
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
-v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto4}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="learning-process">Learning Process </h2>
|
||||
|
||||
<p>
|
||||
During learning both of the networks maximize their own reward function, so that
|
||||
the generator gets better and better at tricking the discriminator, while the
|
||||
discriminator gets better and better at telling the difference between the fake
|
||||
and real data. The generator and discriminator alternate on which one trains at
|
||||
one time (i.e. for one epoch). In other words, we keep the generator constant
|
||||
and train the discriminator, then we keep the discriminator constant to train
|
||||
the generator and repeat. It is this back and forth dynamic which lets GANs
|
||||
tackle otherwise intractable generative problems. As the generator improves with
|
||||
training, the discriminator's performance gets worse because it cannot easily
|
||||
tell the difference between real and fake. If the generator ends up succeeding
|
||||
perfectly, the the discriminator will do no better than random guessing i.e.
|
||||
50\%. This progression in the training poses a problem for the convergence
|
||||
criteria for GANs. The discriminator feedback gets less meaningful over time,
|
||||
if we continue training after this point then the generator is effectively
|
||||
training on junk data which can undo the learning up to that point. Therefore,
|
||||
we stop training when the discriminator starts outputting \( 1/2 \) everywhere.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="more-about-the-learning-process">More about the Learning Process </h2>
|
||||
|
||||
<p>
|
||||
At convergence we have
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto5}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
The default choice for \( v \) is
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x)
|
||||
+ \mathbb{E}_{x\sim p_\mathrm{model}}
|
||||
\log (1 - d(x))
|
||||
\label{_auto6}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
The main motivation for the design of GANs is that the learning process requires
|
||||
neither approximate inference (variational autoencoders for example) nor
|
||||
approximation of a partition function. In the case where
|
||||
$$
|
||||
\begin{equation}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto7}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is
|
||||
asymptotically consistent
|
||||
( <a href="https://arxiv.org/pdf/1804.09139.pdf" target="_blank">Seth Lloyd on QuGANs</a> ).
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="additional-references">Additional References </h2>
|
||||
This is in
|
||||
general not the case and it is possible to get situations where the training
|
||||
process never converges because the generator and discriminator chase one
|
||||
another around in the parameter space indefinitely. A much deeper discussion on
|
||||
the currently open research problem of GAN convergence is available
|
||||
<a href="https://www.deeplearningbook.org/contents/generative_models.html" target="_blank">here</a>. To
|
||||
anyone interested in learning more about GANs it is a highly recommended read.
|
||||
Direct quote: "In this best-performing formulation, the generator aims to
|
||||
increase the log probability that the discriminator makes a mistake, rather than
|
||||
aiming to decrease the log probability that the discriminator makes the correct
|
||||
prediction." <a href="https://arxiv.org/abs/1701.00160" target="_blank">Another interesting read</a>
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="writing-our-first-generative-adversarial-network">Writing Our First Generative Adversarial Network </h2>
|
||||
Let us now move on to actually implementing a GAN in tensorflow. We will study
|
||||
the performance of our GAN on the MNIST dataset. This code is based on and
|
||||
adapted from the
|
||||
<a href="https://www.tensorflow.org/tutorials/generative/dcgan" target="_blank">google tutorial</a>
|
||||
|
||||
<p>
|
||||
First we import our libraries
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">os</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">time</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">numpy</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">np</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">tensorflow</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">tf</span>
|
||||
<span style="color: #8B008B; font-weight: bold">import</span> <span style="color: #008b45; text-decoration: underline">matplotlib.pyplot</span> <span style="color: #8B008B; font-weight: bold">as</span> <span style="color: #008b45; text-decoration: underline">plt</span>
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras</span> <span style="color: #8B008B; font-weight: bold">import</span> layers
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">tensorflow.keras.utils</span> <span style="color: #8B008B; font-weight: bold">import</span> plot_model
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define our hyperparameters and import our data the usual way
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>BUFFER_SIZE = <span style="color: #B452CD">60000</span>
|
||||
BATCH_SIZE = <span style="color: #B452CD">256</span>
|
||||
EPOCHS = <span style="color: #B452CD">30</span>
|
||||
|
||||
data = tf.keras.datasets.mnist.load_data()
|
||||
(train_images, train_labels), (test_images, test_labels) = data
|
||||
train_images = np.reshape(train_images, (train_images.shape[<span style="color: #B452CD">0</span>],
|
||||
<span style="color: #B452CD">28</span>,
|
||||
<span style="color: #B452CD">28</span>,
|
||||
<span style="color: #B452CD">1</span>)).astype(<span style="color: #CD5555">'float32'</span>)
|
||||
|
||||
<span style="color: #228B22"># we normalize between -1 and 1</span>
|
||||
train_images = (train_images - <span style="color: #B452CD">127.5</span>) / <span style="color: #B452CD">127.5</span>
|
||||
training_dataset = tf.data.Dataset.from_tensor_slices(
|
||||
train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="mnist-and-gans">MNIST and GANs </h2>
|
||||
|
||||
<p>
|
||||
Let's have a quick look
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>plt.imshow(train_images[<span style="color: #B452CD">0</span>], cmap=<span style="color: #CD5555">'Greys'</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we define our two models. This is where the 'magic' happens. There are a
|
||||
huge amount of possible formulations for both models. A lot of engineering and
|
||||
trial and error can be done here to try to produce better performing models. For
|
||||
more advanced GANs this is by far the step where you can 'make or break' a
|
||||
model.
|
||||
|
||||
<p>
|
||||
We start with the generator. As stated in the introductory text the generator
|
||||
\( g \) upsamples from a random sample to the shape of what we want to predict. In
|
||||
our case we are trying to predict MNIST images (\( 28\times 28 \) pixels).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generator_model</span>():
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to</span>
|
||||
<span style="color: #CD5555"> produce an image from a random seed. We start with a Dense layer taking this</span>
|
||||
<span style="color: #CD5555"> random sample as an input and subsequently upsample through multiple</span>
|
||||
<span style="color: #CD5555"> convolutional layers.</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
<span style="color: #228B22"># we define our model</span>
|
||||
model = tf.keras.Sequential()
|
||||
|
||||
|
||||
<span style="color: #228B22"># adding our input layer. Dense means that every neuron is connected and</span>
|
||||
<span style="color: #228B22"># the input shape is the shape of our random noise. The units need to match</span>
|
||||
<span style="color: #228B22"># in some sense the upsampling strides to reach our desired output shape.</span>
|
||||
<span style="color: #228B22"># we are using 100 random numbers as our seed</span>
|
||||
model.add(layers.Dense(units=<span style="color: #B452CD">7</span>*<span style="color: #B452CD">7</span>*BATCH_SIZE,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
input_shape=(<span style="color: #B452CD">100</span>, )))
|
||||
<span style="color: #228B22"># we normalize the output form the Dense layer</span>
|
||||
model.add(layers.BatchNormalization())
|
||||
<span style="color: #228B22"># and add an activation function to our 'layer'. LeakyReLU avoids vanishing</span>
|
||||
<span style="color: #228B22"># gradient problem</span>
|
||||
model.add(layers.LeakyReLU())
|
||||
model.add(layers.Reshape((<span style="color: #B452CD">7</span>, <span style="color: #B452CD">7</span>, BATCH_SIZE)))
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">7</span>, <span style="color: #B452CD">7</span>, BATCH_SIZE)
|
||||
<span style="color: #228B22"># even though we just added four keras layers we think of everything above</span>
|
||||
<span style="color: #228B22"># as 'one' layer</span>
|
||||
|
||||
<span style="color: #228B22"># next we add our upscaling convolutional layers</span>
|
||||
model.add(layers.Conv2DTranspose(filters=<span style="color: #B452CD">128</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>))
|
||||
model.add(layers.BatchNormalization())
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">7</span>, <span style="color: #B452CD">7</span>, <span style="color: #B452CD">128</span>)
|
||||
|
||||
model.add(layers.Conv2DTranspose(filters=<span style="color: #B452CD">64</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>))
|
||||
model.add(layers.BatchNormalization())
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">14</span>, <span style="color: #B452CD">14</span>, <span style="color: #B452CD">64</span>)
|
||||
|
||||
model.add(layers.Conv2DTranspose(filters=<span style="color: #B452CD">1</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
use_bias=<span style="color: #8B008B; font-weight: bold">False</span>,
|
||||
activation=<span style="color: #CD5555">'tanh'</span>))
|
||||
<span style="color: #8B008B; font-weight: bold">assert</span> model.output_shape == (<span style="color: #8B008B; font-weight: bold">None</span>, <span style="color: #B452CD">28</span>, <span style="color: #B452CD">28</span>, <span style="color: #B452CD">1</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
And there we have our 'simple' generator model. Now we move on to defining our
|
||||
discriminator model \( d \), which is a convolutional neural network based image
|
||||
classifier.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">discriminator_model</span>():
|
||||
<span style="color: #CD5555">"""</span>
|
||||
<span style="color: #CD5555"> The discriminator is a convolutional neural network based image classifier</span>
|
||||
<span style="color: #CD5555"> """</span>
|
||||
|
||||
<span style="color: #228B22"># we define our model</span>
|
||||
model = tf.keras.Sequential()
|
||||
model.add(layers.Conv2D(filters=<span style="color: #B452CD">64</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>,
|
||||
input_shape=[<span style="color: #B452CD">28</span>, <span style="color: #B452CD">28</span>, <span style="color: #B452CD">1</span>]))
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #228B22"># adding a dropout layer as you do in conv-nets</span>
|
||||
model.add(layers.Dropout(<span style="color: #B452CD">0.3</span>))
|
||||
|
||||
|
||||
model.add(layers.Conv2D(filters=<span style="color: #B452CD">128</span>,
|
||||
kernel_size=(<span style="color: #B452CD">5</span>, <span style="color: #B452CD">5</span>),
|
||||
strides=(<span style="color: #B452CD">2</span>, <span style="color: #B452CD">2</span>),
|
||||
padding=<span style="color: #CD5555">'same'</span>))
|
||||
model.add(layers.LeakyReLU())
|
||||
<span style="color: #228B22"># adding a dropout layer as you do in conv-nets</span>
|
||||
model.add(layers.Dropout(<span style="color: #B452CD">0.3</span>))
|
||||
|
||||
model.add(layers.Flatten())
|
||||
model.add(layers.Dense(<span style="color: #B452CD">1</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="other-models">Other Models </h2>
|
||||
Let us take a look at our models. <b>Note</b>: double click images for bigger view.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>generator = generator_model()
|
||||
plot_model(generator, show_shapes=<span style="color: #8B008B; font-weight: bold">True</span>, rankdir=<span style="color: #CD5555">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>discriminator = discriminator_model()
|
||||
plot_model(discriminator, show_shapes=<span style="color: #8B008B; font-weight: bold">True</span>, rankdir=<span style="color: #CD5555">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we need a few helper objects we will use in training
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
generator_optimizer = tf.keras.optimizers.Adam(<span style="color: #B452CD">1e-4</span>)
|
||||
discriminator_optimizer = tf.keras.optimizers.Adam(<span style="color: #B452CD">1e-4</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
The first object, <em>cross_entropy</em> is our loss function and the two others are
|
||||
our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This
|
||||
is because they need to improve their accuracy at approximately equal speeds to
|
||||
get convergence (not necessarily exactly equal). Now we define our loss
|
||||
functions
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generator_loss</span>(fake_output):
|
||||
loss = cross_entropy(tf.ones_like(fake_output), fake_output)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> loss
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">discriminator_loss</span>(real_output, fake_output):
|
||||
real_loss = cross_entropy(tf.ones_like(real_output), real_output)
|
||||
fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
|
||||
total_loss = real_loss + fake_loss
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> total_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a kind of seed to help us compare the learning process over
|
||||
multiple training epochs.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>noise_dimension = <span style="color: #B452CD">100</span>
|
||||
n_examples_to_generate = <span style="color: #B452CD">16</span>
|
||||
seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="training-step">Training Step </h2>
|
||||
|
||||
<p>
|
||||
Now we have everything we need to define our training step, which we will apply
|
||||
for every step in our training loop. Notice the @tf.function flag signifying
|
||||
that the function is tensorflow 'compiled'. Removing this flag doubles the
|
||||
computation time.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #707a7c">@tf</span>.function
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">train_step</span>(images):
|
||||
noise = tf.random.normal([BATCH_SIZE, noise_dimension])
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> tf.GradientTape() <span style="color: #8B008B; font-weight: bold">as</span> gen_tape, tf.GradientTape() <span style="color: #8B008B; font-weight: bold">as</span> disc_tape:
|
||||
generated_images = generator(noise, training=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
|
||||
real_output = discriminator(images, training=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
fake_output = discriminator(generated_images, training=<span style="color: #8B008B; font-weight: bold">True</span>)
|
||||
|
||||
gen_loss = generator_loss(fake_output)
|
||||
disc_loss = discriminator_loss(real_output, fake_output)
|
||||
|
||||
gradients_of_generator = gen_tape.gradient(gen_loss,
|
||||
generator.trainable_variables)
|
||||
gradients_of_discriminator = disc_tape.gradient(disc_loss,
|
||||
discriminator.trainable_variables)
|
||||
generator_optimizer.apply_gradients(<span style="color: #658b00">zip</span>(gradients_of_generator,
|
||||
generator.trainable_variables))
|
||||
discriminator_optimizer.apply_gradients(<span style="color: #658b00">zip</span>(gradients_of_discriminator,
|
||||
discriminator.trainable_variables))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> gen_loss, disc_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a helper function to produce an output over our training epochs
|
||||
to see the predictive progression of our generator model. <b>Note</b>: I am including
|
||||
this code here, but comment it out in the training loop.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_and_save_images</span>(model, epoch, test_input):
|
||||
<span style="color: #228B22"># we're making inferences here</span>
|
||||
predictions = model(test_input, training=<span style="color: #8B008B; font-weight: bold">False</span>)
|
||||
|
||||
fig = plt.figure(figsize=(<span style="color: #B452CD">4</span>, <span style="color: #B452CD">4</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(predictions.shape[<span style="color: #B452CD">0</span>]):
|
||||
plt.subplot(<span style="color: #B452CD">4</span>, <span style="color: #B452CD">4</span>, i+<span style="color: #B452CD">1</span>)
|
||||
plt.imshow(predictions[i, :, :, <span style="color: #B452CD">0</span>] * <span style="color: #B452CD">127.5</span> + <span style="color: #B452CD">127.5</span>, cmap=<span style="color: #CD5555">'gray'</span>)
|
||||
plt.axis(<span style="color: #CD5555">'off'</span>)
|
||||
|
||||
plt.savefig(<span style="color: #CD5555">f'./images_from_seed_images/image_at_epoch_{</span><span style="color: #658b00">str</span>(epoch).zfill(<span style="color: #B452CD">3</span>)<span style="color: #CD5555">}.png'</span>)
|
||||
plt.close()
|
||||
<span style="color: #228B22">#plt.show()</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="checkpoints">Checkpoints </h2>
|
||||
Setting up checkpoints to periodically save our model during training so that
|
||||
everything is not lost even if the program were to somehow terminate while
|
||||
training.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #228B22"># Setting up checkpoints to save model during training</span>
|
||||
checkpoint_dir = <span style="color: #CD5555">'./training_checkpoints'</span>
|
||||
checkpoint_prefix = os.path.join(checkpoint_dir, <span style="color: #CD5555">'ckpt'</span>)
|
||||
checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
|
||||
discriminator_optimizer=discriminator_optimizer,
|
||||
generator=generator,
|
||||
discriminator=discriminator)
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we define our training loop
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">train</span>(dataset, epochs):
|
||||
generator_loss_list = []
|
||||
discriminator_loss_list = []
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> epoch <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(epochs):
|
||||
start = time.time()
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> image_batch <span style="color: #8B008B">in</span> dataset:
|
||||
gen_loss, disc_loss = train_step(image_batch)
|
||||
generator_loss_list.append(gen_loss.numpy())
|
||||
discriminator_loss_list.append(disc_loss.numpy())
|
||||
|
||||
<span style="color: #228B22">#generate_and_save_images(generator, epoch + 1, seed_images)</span>
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> (epoch + <span style="color: #B452CD">1</span>) % <span style="color: #B452CD">15</span> == <span style="color: #B452CD">0</span>:
|
||||
checkpoint.save(file_prefix=checkpoint_prefix)
|
||||
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">f'Time for epoch {</span>epoch<span style="color: #CD5555">} is {</span>time.time() - start<span style="color: #CD5555">}'</span>)
|
||||
|
||||
<span style="color: #228B22">#generate_and_save_images(generator, epochs, seed_images)</span>
|
||||
|
||||
loss_file = <span style="color: #CD5555">'./data/lossfile.txt'</span>
|
||||
<span style="color: #8B008B; font-weight: bold">with</span> <span style="color: #658b00">open</span>(loss_file, <span style="color: #CD5555">'w'</span>) <span style="color: #8B008B; font-weight: bold">as</span> outfile:
|
||||
outfile.write(<span style="color: #658b00">str</span>(generator_loss_list))
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
outfile.write(<span style="color: #658b00">str</span>(discriminator_loss_list))
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
outfile.write(<span style="color: #CD5555">'\n'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
To train simply call this function. <b>Warning</b>: this might take a long time so
|
||||
there is a folder of a pretrained network already included in the repository.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>train(train_dataset, EPOCHS)
|
||||
</pre></div>
|
||||
<p>
|
||||
And here is the result of training our model for 100 epochs
|
||||
|
||||
<p>
|
||||
|
||||
<embed src="images_from_seed_images/generation.gif" autoplay="false" loop="true"></embed>
|
||||
<p><em></em></p>
|
||||
|
||||
<p>
|
||||
Now to avoid having to train and everything, which will take a while depending
|
||||
on your computer setup we now load in the model which produced the above gif.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
|
||||
restored_generator = checkpoint.generator
|
||||
restored_discriminator = checkpoint.discriminator
|
||||
|
||||
<span style="color: #658b00">print</span>(restored_generator)
|
||||
<span style="color: #658b00">print</span>(restored_discriminator)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="exploring-the-latent-space">Exploring the Latent Space </h2>
|
||||
|
||||
<p>
|
||||
We have successfully loaded in our latest model. Let us now play around a bit
|
||||
and see what kind of things we can learn about this model. Our generator takes
|
||||
an array of 100 numbers. One idea can be to try to systematically change our
|
||||
input. Let us try and see what we get
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_latent_points</span>(number=<span style="color: #B452CD">100</span>, scale_means=<span style="color: #B452CD">1</span>, scale_stds=<span style="color: #B452CD">1</span>):
|
||||
latent_dim = <span style="color: #B452CD">100</span>
|
||||
means = scale_means * tf.linspace(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, num=latent_dim)
|
||||
stds = scale_stds * tf.linspace(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>, num=latent_dim)
|
||||
latent_space_value_range = tf.random.normal([number, latent_dim],
|
||||
means,
|
||||
stds,
|
||||
dtype=tf.float64)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> latent_space_value_range
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">generate_images</span>(latent_points):
|
||||
<span style="color: #228B22"># notice we set training to false because we are making inferences</span>
|
||||
generated_images = restored_generator.predict(latent_points)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> generated_images
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_result</span>(generated_images, number=<span style="color: #B452CD">100</span>):
|
||||
<span style="color: #228B22"># obviously this assumes sqrt number is an int</span>
|
||||
fig, axs = plt.subplots(<span style="color: #658b00">int</span>(np.sqrt(number)), <span style="color: #658b00">int</span>(np.sqrt(number)),
|
||||
figsize=(<span style="color: #B452CD">10</span>, <span style="color: #B452CD">10</span>))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">int</span>(np.sqrt(number))):
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> j <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #658b00">int</span>(np.sqrt(number))):
|
||||
axs[i, j].imshow(generated_images[i*j], cmap=<span style="color: #CD5555">'Greys'</span>)
|
||||
axs[i, j].axis(<span style="color: #CD5555">'off'</span>)
|
||||
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>generated_images = generate_images(generate_latent_points())
|
||||
plot_result(generated_images)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="getting-results">Getting Results </h2>
|
||||
We see that the generator generates images that look like MNIST
|
||||
numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able
|
||||
to generate a similar plot where we generate every MNIST number. Let us now try
|
||||
to 'move' a bit around in the latent space. <b>Note</b>: decrease the plot number if
|
||||
these following cells take too long to run on your computer.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>plot_number = <span style="color: #B452CD">225</span>
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=<span style="color: #B452CD">5</span>,
|
||||
scale_stds=<span style="color: #B452CD">1</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=-<span style="color: #B452CD">5</span>,
|
||||
scale_stds=<span style="color: #B452CD">1</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=<span style="color: #B452CD">1</span>,
|
||||
scale_stds=<span style="color: #B452CD">5</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
Again, we have found something interesting. <em>Moving</em> around using our means
|
||||
takes us from digit to digit, while <em>moving</em> around using our standard
|
||||
deviations seem to increase the number of different digits! In the last image
|
||||
above, we can barely make out every MNIST digit. Let us make on last plot using
|
||||
this information by upping the standard deviation of our Gaussian noises.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>plot_number = <span style="color: #B452CD">400</span>
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=<span style="color: #B452CD">1</span>,
|
||||
scale_stds=<span style="color: #B452CD">10</span>))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
A pretty cool result! We see that our generator indeed has learned a
|
||||
distribution which qualitatively looks a whole lot like the MNIST dataset.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="interpolating-between-mnist-digits">Interpolating Between MNIST Digits </h2>
|
||||
Another interesting way to explore the latent space of our generator model is by
|
||||
interpolating between the MNIST digits. This section is largely based on
|
||||
<a href="https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/" target="_blank">this excellent blogpost</a>
|
||||
by Jason Brownlee.
|
||||
|
||||
<p>
|
||||
So let us start by defining a function to interpolate between two points in the
|
||||
latent space.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span><span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">interpolation</span>(point_1, point_2, n_steps=<span style="color: #B452CD">10</span>):
|
||||
ratios = np.linspace(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">1</span>, num=n_steps)
|
||||
vectors = []
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i, ratio <span style="color: #8B008B">in</span> <span style="color: #658b00">enumerate</span>(ratios):
|
||||
vectors.append(((<span style="color: #B452CD">1.0</span> - ratio) * point_1 + ratio * point_2))
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">return</span> tf.stack(vectors)
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we have all we need to do our interpolation analysis.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%;"><span></span>plot_number = <span style="color: #B452CD">100</span>
|
||||
latent_points = generate_latent_points(number=plot_number)
|
||||
results = <span style="color: #8B008B; font-weight: bold">None</span>
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">0</span>, <span style="color: #B452CD">2</span>*np.sqrt(plot_number), <span style="color: #B452CD">2</span>):
|
||||
interpolated = interpolation(latent_points[i], latent_points[i+<span style="color: #B452CD">1</span>])
|
||||
generated_images = generate_images(interpolated)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> results <span style="color: #8B008B">is</span> <span style="color: #8B008B; font-weight: bold">None</span>:
|
||||
results = generated_images
|
||||
<span style="color: #8B008B; font-weight: bold">else</span>:
|
||||
results = tf.stack((results, generated_images))
|
||||
|
||||
plot_results(results, plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
@@ -89,11 +89,56 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('Other Types of Recurrent Neural Networks',
|
||||
2,
|
||||
None,
|
||||
'other-types-of-recurrent-neural-networks')]}
|
||||
'other-types-of-recurrent-neural-networks'),
|
||||
('Generative Models', 2, None, 'generative-models'),
|
||||
('Generative Adversarial Networks',
|
||||
2,
|
||||
None,
|
||||
'generative-adversarial-networks'),
|
||||
('Discriminator', 2, None, 'discriminator'),
|
||||
('Learning Process', 2, None, 'learning-process'),
|
||||
('More about the Learning Process',
|
||||
2,
|
||||
None,
|
||||
'more-about-the-learning-process'),
|
||||
('Additional References', 2, None, 'additional-references'),
|
||||
('Writing Our First Generative Adversarial Network',
|
||||
2,
|
||||
None,
|
||||
'writing-our-first-generative-adversarial-network'),
|
||||
('MNIST and GANs', 2, None, 'mnist-and-gans'),
|
||||
('Other Models', 2, None, 'other-models'),
|
||||
('Training Step', 2, None, 'training-step'),
|
||||
('Checkpoints', 2, None, 'checkpoints'),
|
||||
('Exploring the Latent Space',
|
||||
2,
|
||||
None,
|
||||
'exploring-the-latent-space'),
|
||||
('Getting Results', 2, None, 'getting-results'),
|
||||
('Interpolating Between MNIST Digits',
|
||||
2,
|
||||
None,
|
||||
'interpolating-between-mnist-digits')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
|
||||
|
||||
|
||||
<script type="text/x-mathjax-config">
|
||||
MathJax.Hub.Config({
|
||||
TeX: {
|
||||
equationNumbers: { autoNumber: "AMS" },
|
||||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||||
}
|
||||
});
|
||||
</script>
|
||||
<script type="text/javascript" async
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- ------------------- main content ---------------------- -->
|
||||
|
||||
@@ -223,14 +268,14 @@ example, they can take sentences, documents, or audio samples as
|
||||
input, making them extremely useful for natural language processing
|
||||
systems such as automatic translation and speech-to-text.
|
||||
|
||||
<p>
|
||||
RNNs are also used in "text classification":"Text Classification Algorithms: A Survey"
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="set-up-of-an-rnn">Set up of an RNN </h2>
|
||||
|
||||
<p>
|
||||
More to text to be added by Wednesday October 27.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
@@ -891,6 +936,696 @@ end <span style="color: #666666">=</span> timer()
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Time: '</span>, end<span style="color: #666666">-</span>start)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="generative-models">Generative Models </h2>
|
||||
|
||||
<p>
|
||||
<b>Generative models</b> describe a class of statistical models that are a contrast
|
||||
to <b>discriminative models</b>. Informally we say that generative models can
|
||||
generate new data instances while discriminative models discriminate between
|
||||
different kinds of data instances. A generative model could generate new photos
|
||||
of animals that look like 'real' animals while a discriminative model could tell
|
||||
a dog from a cat. More formally, given a data set \( x \) and a set of labels /
|
||||
targets \( y \). Generative models capture the joint probability \( p(x, y) \), or
|
||||
just \( p(x) \) if there are no labels, while discriminative models capture the
|
||||
conditional probability \( p(y | x) \). Discriminative models generally try to draw
|
||||
boundaries in the data space (often high dimensional), while generative models
|
||||
try to model how data is placed throughout the space.
|
||||
|
||||
<p>
|
||||
<b>Note</b>: this material is thanks to Linus Ekstrøm.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="generative-adversarial-networks">Generative Adversarial Networks </h2>
|
||||
|
||||
<p>
|
||||
<b>Generative Adversarial Networks</b> are a type of unsupervised machine learning
|
||||
algorithm proposed by <a href="https://arxiv.org/pdf/1406.2661.pdf" target="_blank">Goodfellow et. al</a>
|
||||
in 2014 (short and good article).
|
||||
|
||||
<p>
|
||||
The simplest formulation of
|
||||
the model is based on a game theoretic approach, <em>zero sum game</em>, where we pit
|
||||
two neural networks against one another. We define two rival networks, one
|
||||
generator \( g \), and one discriminator \( d \). The generator directly produces
|
||||
samples
|
||||
$$
|
||||
\begin{equation}
|
||||
x = g(z; \theta^{(g)})
|
||||
\label{_auto1}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="discriminator">Discriminator </h2>
|
||||
The discriminator attempts to distinguish between samples drawn from the
|
||||
training data and samples drawn from the generator. In other words, it tries to
|
||||
tell the difference between the fake data produced by \( g \) and the actual data
|
||||
samples we want to do prediction on. The discriminator outputs a probability
|
||||
value given by
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
d(x; \theta^{(d)})
|
||||
\label{_auto2}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
indicating the probability that \( x \) is a real training example rather than a
|
||||
fake sample the generator has generated. The simplest way to formulate the
|
||||
learning process in a generative adversarial network is a zero-sum game, in
|
||||
which a function
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto3}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
determines the reward for the discriminator, while the generator gets the
|
||||
conjugate reward
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
-v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto4}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="learning-process">Learning Process </h2>
|
||||
|
||||
<p>
|
||||
During learning both of the networks maximize their own reward function, so that
|
||||
the generator gets better and better at tricking the discriminator, while the
|
||||
discriminator gets better and better at telling the difference between the fake
|
||||
and real data. The generator and discriminator alternate on which one trains at
|
||||
one time (i.e. for one epoch). In other words, we keep the generator constant
|
||||
and train the discriminator, then we keep the discriminator constant to train
|
||||
the generator and repeat. It is this back and forth dynamic which lets GANs
|
||||
tackle otherwise intractable generative problems. As the generator improves with
|
||||
training, the discriminator's performance gets worse because it cannot easily
|
||||
tell the difference between real and fake. If the generator ends up succeeding
|
||||
perfectly, the the discriminator will do no better than random guessing i.e.
|
||||
50\%. This progression in the training poses a problem for the convergence
|
||||
criteria for GANs. The discriminator feedback gets less meaningful over time,
|
||||
if we continue training after this point then the generator is effectively
|
||||
training on junk data which can undo the learning up to that point. Therefore,
|
||||
we stop training when the discriminator starts outputting \( 1/2 \) everywhere.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="more-about-the-learning-process">More about the Learning Process </h2>
|
||||
|
||||
<p>
|
||||
At convergence we have
|
||||
|
||||
$$
|
||||
\begin{equation}
|
||||
g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto5}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
The default choice for \( v \) is
|
||||
$$
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x)
|
||||
+ \mathbb{E}_{x\sim p_\mathrm{model}}
|
||||
\log (1 - d(x))
|
||||
\label{_auto6}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
The main motivation for the design of GANs is that the learning process requires
|
||||
neither approximate inference (variational autoencoders for example) nor
|
||||
approximation of a partition function. In the case where
|
||||
$$
|
||||
\begin{equation}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\label{_auto7}
|
||||
\end{equation}
|
||||
$$
|
||||
|
||||
is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is
|
||||
asymptotically consistent
|
||||
( <a href="https://arxiv.org/pdf/1804.09139.pdf" target="_blank">Seth Lloyd on QuGANs</a> ).
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="additional-references">Additional References </h2>
|
||||
This is in
|
||||
general not the case and it is possible to get situations where the training
|
||||
process never converges because the generator and discriminator chase one
|
||||
another around in the parameter space indefinitely. A much deeper discussion on
|
||||
the currently open research problem of GAN convergence is available
|
||||
<a href="https://www.deeplearningbook.org/contents/generative_models.html" target="_blank">here</a>. To
|
||||
anyone interested in learning more about GANs it is a highly recommended read.
|
||||
Direct quote: "In this best-performing formulation, the generator aims to
|
||||
increase the log probability that the discriminator makes a mistake, rather than
|
||||
aiming to decrease the log probability that the discriminator makes the correct
|
||||
prediction." <a href="https://arxiv.org/abs/1701.00160" target="_blank">Another interesting read</a>
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="writing-our-first-generative-adversarial-network">Writing Our First Generative Adversarial Network </h2>
|
||||
Let us now move on to actually implementing a GAN in tensorflow. We will study
|
||||
the performance of our GAN on the MNIST dataset. This code is based on and
|
||||
adapted from the
|
||||
<a href="https://www.tensorflow.org/tutorials/generative/dcgan" target="_blank">google tutorial</a>
|
||||
|
||||
<p>
|
||||
First we import our libraries
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">tensorflow</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">tf</span>
|
||||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras</span> <span style="color: #008000; font-weight: bold">import</span> layers
|
||||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">tensorflow.keras.utils</span> <span style="color: #008000; font-weight: bold">import</span> plot_model
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define our hyperparameters and import our data the usual way
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>BUFFER_SIZE <span style="color: #666666">=</span> <span style="color: #666666">60000</span>
|
||||
BATCH_SIZE <span style="color: #666666">=</span> <span style="color: #666666">256</span>
|
||||
EPOCHS <span style="color: #666666">=</span> <span style="color: #666666">30</span>
|
||||
|
||||
data <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>datasets<span style="color: #666666">.</span>mnist<span style="color: #666666">.</span>load_data()
|
||||
(train_images, train_labels), (test_images, test_labels) <span style="color: #666666">=</span> data
|
||||
train_images <span style="color: #666666">=</span> np<span style="color: #666666">.</span>reshape(train_images, (train_images<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>],
|
||||
<span style="color: #666666">28</span>,
|
||||
<span style="color: #666666">28</span>,
|
||||
<span style="color: #666666">1</span>))<span style="color: #666666">.</span>astype(<span style="color: #BA2121">'float32'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we normalize between -1 and 1</span>
|
||||
train_images <span style="color: #666666">=</span> (train_images <span style="color: #666666">-</span> <span style="color: #666666">127.5</span>) <span style="color: #666666">/</span> <span style="color: #666666">127.5</span>
|
||||
training_dataset <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>data<span style="color: #666666">.</span>Dataset<span style="color: #666666">.</span>from_tensor_slices(
|
||||
train_images)<span style="color: #666666">.</span>shuffle(BUFFER_SIZE)<span style="color: #666666">.</span>batch(BATCH_SIZE)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="mnist-and-gans">MNIST and GANs </h2>
|
||||
|
||||
<p>
|
||||
Let's have a quick look
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plt<span style="color: #666666">.</span>imshow(train_images[<span style="color: #666666">0</span>], cmap<span style="color: #666666">=</span><span style="color: #BA2121">'Greys'</span>)
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we define our two models. This is where the 'magic' happens. There are a
|
||||
huge amount of possible formulations for both models. A lot of engineering and
|
||||
trial and error can be done here to try to produce better performing models. For
|
||||
more advanced GANs this is by far the step where you can 'make or break' a
|
||||
model.
|
||||
|
||||
<p>
|
||||
We start with the generator. As stated in the introductory text the generator
|
||||
\( g \) upsamples from a random sample to the shape of what we want to predict. In
|
||||
our case we are trying to predict MNIST images (\( 28\times 28 \) pixels).
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generator_model</span>():
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to</span>
|
||||
<span style="color: #BA2121; font-style: italic"> produce an image from a random seed. We start with a Dense layer taking this</span>
|
||||
<span style="color: #BA2121; font-style: italic"> random sample as an input and subsequently upsample through multiple</span>
|
||||
<span style="color: #BA2121; font-style: italic"> convolutional layers.</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we define our model</span>
|
||||
model <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>Sequential()
|
||||
|
||||
|
||||
<span style="color: #408080; font-style: italic"># adding our input layer. Dense means that every neuron is connected and</span>
|
||||
<span style="color: #408080; font-style: italic"># the input shape is the shape of our random noise. The units need to match</span>
|
||||
<span style="color: #408080; font-style: italic"># in some sense the upsampling strides to reach our desired output shape.</span>
|
||||
<span style="color: #408080; font-style: italic"># we are using 100 random numbers as our seed</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(units<span style="color: #666666">=7*7*</span>BATCH_SIZE,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||||
input_shape<span style="color: #666666">=</span>(<span style="color: #666666">100</span>, )))
|
||||
<span style="color: #408080; font-style: italic"># we normalize the output form the Dense layer</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>BatchNormalization())
|
||||
<span style="color: #408080; font-style: italic"># and add an activation function to our 'layer'. LeakyReLU avoids vanishing</span>
|
||||
<span style="color: #408080; font-style: italic"># gradient problem</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Reshape((<span style="color: #666666">7</span>, <span style="color: #666666">7</span>, BATCH_SIZE)))
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">7</span>, <span style="color: #666666">7</span>, BATCH_SIZE)
|
||||
<span style="color: #408080; font-style: italic"># even though we just added four keras layers we think of everything above</span>
|
||||
<span style="color: #408080; font-style: italic"># as 'one' layer</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># next we add our upscaling convolutional layers</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2DTranspose(filters<span style="color: #666666">=128</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">1</span>, <span style="color: #666666">1</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>BatchNormalization())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">7</span>, <span style="color: #666666">7</span>, <span style="color: #666666">128</span>)
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2DTranspose(filters<span style="color: #666666">=64</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>BatchNormalization())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">14</span>, <span style="color: #666666">14</span>, <span style="color: #666666">64</span>)
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2DTranspose(filters<span style="color: #666666">=1</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
use_bias<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||||
activation<span style="color: #666666">=</span><span style="color: #BA2121">'tanh'</span>))
|
||||
<span style="color: #008000; font-weight: bold">assert</span> model<span style="color: #666666">.</span>output_shape <span style="color: #666666">==</span> (<span style="color: #008000; font-weight: bold">None</span>, <span style="color: #666666">28</span>, <span style="color: #666666">28</span>, <span style="color: #666666">1</span>)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
And there we have our 'simple' generator model. Now we move on to defining our
|
||||
discriminator model \( d \), which is a convolutional neural network based image
|
||||
classifier.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">discriminator_model</span>():
|
||||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||||
<span style="color: #BA2121; font-style: italic"> The discriminator is a convolutional neural network based image classifier</span>
|
||||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||||
|
||||
<span style="color: #408080; font-style: italic"># we define our model</span>
|
||||
model <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>Sequential()
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(filters<span style="color: #666666">=64</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>,
|
||||
input_shape<span style="color: #666666">=</span>[<span style="color: #666666">28</span>, <span style="color: #666666">28</span>, <span style="color: #666666">1</span>]))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #408080; font-style: italic"># adding a dropout layer as you do in conv-nets</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dropout(<span style="color: #666666">0.3</span>))
|
||||
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Conv2D(filters<span style="color: #666666">=128</span>,
|
||||
kernel_size<span style="color: #666666">=</span>(<span style="color: #666666">5</span>, <span style="color: #666666">5</span>),
|
||||
strides<span style="color: #666666">=</span>(<span style="color: #666666">2</span>, <span style="color: #666666">2</span>),
|
||||
padding<span style="color: #666666">=</span><span style="color: #BA2121">'same'</span>))
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>LeakyReLU())
|
||||
<span style="color: #408080; font-style: italic"># adding a dropout layer as you do in conv-nets</span>
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dropout(<span style="color: #666666">0.3</span>))
|
||||
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Flatten())
|
||||
model<span style="color: #666666">.</span>add(layers<span style="color: #666666">.</span>Dense(<span style="color: #666666">1</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> model
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="other-models">Other Models </h2>
|
||||
Let us take a look at our models. <b>Note</b>: double click images for bigger view.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>generator <span style="color: #666666">=</span> generator_model()
|
||||
plot_model(generator, show_shapes<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, rankdir<span style="color: #666666">=</span><span style="color: #BA2121">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>discriminator <span style="color: #666666">=</span> discriminator_model()
|
||||
plot_model(discriminator, show_shapes<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>, rankdir<span style="color: #666666">=</span><span style="color: #BA2121">'LR'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we need a few helper objects we will use in training
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>cross_entropy <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>losses<span style="color: #666666">.</span>BinaryCrossentropy(from_logits<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
generator_optimizer <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>optimizers<span style="color: #666666">.</span>Adam(<span style="color: #666666">1e-4</span>)
|
||||
discriminator_optimizer <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>keras<span style="color: #666666">.</span>optimizers<span style="color: #666666">.</span>Adam(<span style="color: #666666">1e-4</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
The first object, <em>cross_entropy</em> is our loss function and the two others are
|
||||
our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This
|
||||
is because they need to improve their accuracy at approximately equal speeds to
|
||||
get convergence (not necessarily exactly equal). Now we define our loss
|
||||
functions
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generator_loss</span>(fake_output):
|
||||
loss <span style="color: #666666">=</span> cross_entropy(tf<span style="color: #666666">.</span>ones_like(fake_output), fake_output)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> loss
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">discriminator_loss</span>(real_output, fake_output):
|
||||
real_loss <span style="color: #666666">=</span> cross_entropy(tf<span style="color: #666666">.</span>ones_like(real_output), real_output)
|
||||
fake_loss <span style="color: #666666">=</span> cross_entropy(tf<span style="color: #666666">.</span>zeros_liks(fake_output), fake_output)
|
||||
total_loss <span style="color: #666666">=</span> real_loss <span style="color: #666666">+</span> fake_loss
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> total_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a kind of seed to help us compare the learning process over
|
||||
multiple training epochs.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>noise_dimension <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
n_examples_to_generate <span style="color: #666666">=</span> <span style="color: #666666">16</span>
|
||||
seed_images <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal([n_examples_to_generate, noise_dimension])
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="training-step">Training Step </h2>
|
||||
|
||||
<p>
|
||||
Now we have everything we need to define our training step, which we will apply
|
||||
for every step in our training loop. Notice the @tf.function flag signifying
|
||||
that the function is tensorflow 'compiled'. Removing this flag doubles the
|
||||
computation time.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #AA22FF">@tf</span><span style="color: #666666">.</span>function
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train_step</span>(images):
|
||||
noise <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal([BATCH_SIZE, noise_dimension])
|
||||
|
||||
<span style="color: #008000; font-weight: bold">with</span> tf<span style="color: #666666">.</span>GradientTape() <span style="color: #008000; font-weight: bold">as</span> gen_tape, tf<span style="color: #666666">.</span>GradientTape() <span style="color: #008000; font-weight: bold">as</span> disc_tape:
|
||||
generated_images <span style="color: #666666">=</span> generator(noise, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
|
||||
real_output <span style="color: #666666">=</span> discriminator(images, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
fake_output <span style="color: #666666">=</span> discriminator(generated_images, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||||
|
||||
gen_loss <span style="color: #666666">=</span> generator_loss(fake_output)
|
||||
disc_loss <span style="color: #666666">=</span> discriminator_loss(real_output, fake_output)
|
||||
|
||||
gradients_of_generator <span style="color: #666666">=</span> gen_tape<span style="color: #666666">.</span>gradient(gen_loss,
|
||||
generator<span style="color: #666666">.</span>trainable_variables)
|
||||
gradients_of_discriminator <span style="color: #666666">=</span> disc_tape<span style="color: #666666">.</span>gradient(disc_loss,
|
||||
discriminator<span style="color: #666666">.</span>trainable_variables)
|
||||
generator_optimizer<span style="color: #666666">.</span>apply_gradients(<span style="color: #008000">zip</span>(gradients_of_generator,
|
||||
generator<span style="color: #666666">.</span>trainable_variables))
|
||||
discriminator_optimizer<span style="color: #666666">.</span>apply_gradients(<span style="color: #008000">zip</span>(gradients_of_discriminator,
|
||||
discriminator<span style="color: #666666">.</span>trainable_variables))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> gen_loss, disc_loss
|
||||
</pre></div>
|
||||
<p>
|
||||
Next we define a helper function to produce an output over our training epochs
|
||||
to see the predictive progression of our generator model. <b>Note</b>: I am including
|
||||
this code here, but comment it out in the training loop.
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_and_save_images</span>(model, epoch, test_input):
|
||||
<span style="color: #408080; font-style: italic"># we're making inferences here</span>
|
||||
predictions <span style="color: #666666">=</span> model(test_input, training<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>)
|
||||
|
||||
fig <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>figure(figsize<span style="color: #666666">=</span>(<span style="color: #666666">4</span>, <span style="color: #666666">4</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(predictions<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]):
|
||||
plt<span style="color: #666666">.</span>subplot(<span style="color: #666666">4</span>, <span style="color: #666666">4</span>, i<span style="color: #666666">+1</span>)
|
||||
plt<span style="color: #666666">.</span>imshow(predictions[i, :, :, <span style="color: #666666">0</span>] <span style="color: #666666">*</span> <span style="color: #666666">127.5</span> <span style="color: #666666">+</span> <span style="color: #666666">127.5</span>, cmap<span style="color: #666666">=</span><span style="color: #BA2121">'gray'</span>)
|
||||
plt<span style="color: #666666">.</span>axis(<span style="color: #BA2121">'off'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>savefig(<span style="color: #BA2121">f'./images_from_seed_images/image_at_epoch_</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">str</span>(epoch)<span style="color: #666666">.</span>zfill(<span style="color: #666666">3</span>)<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">.png'</span>)
|
||||
plt<span style="color: #666666">.</span>close()
|
||||
<span style="color: #408080; font-style: italic">#plt.show()</span>
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="checkpoints">Checkpoints </h2>
|
||||
Setting up checkpoints to periodically save our model during training so that
|
||||
everything is not lost even if the program were to somehow terminate while
|
||||
training.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #408080; font-style: italic"># Setting up checkpoints to save model during training</span>
|
||||
checkpoint_dir <span style="color: #666666">=</span> <span style="color: #BA2121">'./training_checkpoints'</span>
|
||||
checkpoint_prefix <span style="color: #666666">=</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(checkpoint_dir, <span style="color: #BA2121">'ckpt'</span>)
|
||||
checkpoint <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>train<span style="color: #666666">.</span>Checkpoint(generator_optimizer<span style="color: #666666">=</span>generator_optimizer,
|
||||
discriminator_optimizer<span style="color: #666666">=</span>discriminator_optimizer,
|
||||
generator<span style="color: #666666">=</span>generator,
|
||||
discriminator<span style="color: #666666">=</span>discriminator)
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we define our training loop
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">train</span>(dataset, epochs):
|
||||
generator_loss_list <span style="color: #666666">=</span> []
|
||||
discriminator_loss_list <span style="color: #666666">=</span> []
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(epochs):
|
||||
start <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> image_batch <span style="color: #AA22FF; font-weight: bold">in</span> dataset:
|
||||
gen_loss, disc_loss <span style="color: #666666">=</span> train_step(image_batch)
|
||||
generator_loss_list<span style="color: #666666">.</span>append(gen_loss<span style="color: #666666">.</span>numpy())
|
||||
discriminator_loss_list<span style="color: #666666">.</span>append(disc_loss<span style="color: #666666">.</span>numpy())
|
||||
|
||||
<span style="color: #408080; font-style: italic">#generate_and_save_images(generator, epoch + 1, seed_images)</span>
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> (epoch <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #666666">%</span> <span style="color: #666666">15</span> <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
||||
checkpoint<span style="color: #666666">.</span>save(file_prefix<span style="color: #666666">=</span>checkpoint_prefix)
|
||||
|
||||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Time for epoch </span><span style="color: #BB6688; font-weight: bold">{</span>epoch<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> is </span><span style="color: #BB6688; font-weight: bold">{</span>time<span style="color: #666666">.</span>time() <span style="color: #666666">-</span> start<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">'</span>)
|
||||
|
||||
<span style="color: #408080; font-style: italic">#generate_and_save_images(generator, epochs, seed_images)</span>
|
||||
|
||||
loss_file <span style="color: #666666">=</span> <span style="color: #BA2121">'./data/lossfile.txt'</span>
|
||||
<span style="color: #008000; font-weight: bold">with</span> <span style="color: #008000">open</span>(loss_file, <span style="color: #BA2121">'w'</span>) <span style="color: #008000; font-weight: bold">as</span> outfile:
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #008000">str</span>(generator_loss_list))
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #008000">str</span>(discriminator_loss_list))
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
outfile<span style="color: #666666">.</span>write(<span style="color: #BA2121">'</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">'</span>)
|
||||
</pre></div>
|
||||
<p>
|
||||
To train simply call this function. <b>Warning</b>: this might take a long time so
|
||||
there is a folder of a pretrained network already included in the repository.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>train(train_dataset, EPOCHS)
|
||||
</pre></div>
|
||||
<p>
|
||||
And here is the result of training our model for 100 epochs
|
||||
|
||||
<p>
|
||||
|
||||
<embed src="images_from_seed_images/generation.gif" autoplay="false" loop="true"></embed>
|
||||
<p><em></em></p>
|
||||
|
||||
<p>
|
||||
Now to avoid having to train and everything, which will take a while depending
|
||||
on your computer setup we now load in the model which produced the above gif.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>checkpoint<span style="color: #666666">.</span>restore(tf<span style="color: #666666">.</span>train<span style="color: #666666">.</span>latest_checkpoint(checkpoint_dir))
|
||||
restored_generator <span style="color: #666666">=</span> checkpoint<span style="color: #666666">.</span>generator
|
||||
restored_discriminator <span style="color: #666666">=</span> checkpoint<span style="color: #666666">.</span>discriminator
|
||||
|
||||
<span style="color: #008000">print</span>(restored_generator)
|
||||
<span style="color: #008000">print</span>(restored_discriminator)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="exploring-the-latent-space">Exploring the Latent Space </h2>
|
||||
|
||||
<p>
|
||||
We have successfully loaded in our latest model. Let us now play around a bit
|
||||
and see what kind of things we can learn about this model. Our generator takes
|
||||
an array of 100 numbers. One idea can be to try to systematically change our
|
||||
input. Let us try and see what we get
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_latent_points</span>(number<span style="color: #666666">=100</span>, scale_means<span style="color: #666666">=1</span>, scale_stds<span style="color: #666666">=1</span>):
|
||||
latent_dim <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
means <span style="color: #666666">=</span> scale_means <span style="color: #666666">*</span> tf<span style="color: #666666">.</span>linspace(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>, num<span style="color: #666666">=</span>latent_dim)
|
||||
stds <span style="color: #666666">=</span> scale_stds <span style="color: #666666">*</span> tf<span style="color: #666666">.</span>linspace(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>, num<span style="color: #666666">=</span>latent_dim)
|
||||
latent_space_value_range <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal([number, latent_dim],
|
||||
means,
|
||||
stds,
|
||||
dtype<span style="color: #666666">=</span>tf<span style="color: #666666">.</span>float64)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> latent_space_value_range
|
||||
|
||||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_images</span>(latent_points):
|
||||
<span style="color: #408080; font-style: italic"># notice we set training to false because we are making inferences</span>
|
||||
generated_images <span style="color: #666666">=</span> restored_generator<span style="color: #666666">.</span>predict(latent_points)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> generated_images
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_result</span>(generated_images, number<span style="color: #666666">=100</span>):
|
||||
<span style="color: #408080; font-style: italic"># obviously this assumes sqrt number is an int</span>
|
||||
fig, axs <span style="color: #666666">=</span> plt<span style="color: #666666">.</span>subplots(<span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number)), <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number)),
|
||||
figsize<span style="color: #666666">=</span>(<span style="color: #666666">10</span>, <span style="color: #666666">10</span>))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number))):
|
||||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">int</span>(np<span style="color: #666666">.</span>sqrt(number))):
|
||||
axs[i, j]<span style="color: #666666">.</span>imshow(generated_images[i<span style="color: #666666">*</span>j], cmap<span style="color: #666666">=</span><span style="color: #BA2121">'Greys'</span>)
|
||||
axs[i, j]<span style="color: #666666">.</span>axis(<span style="color: #BA2121">'off'</span>)
|
||||
|
||||
plt<span style="color: #666666">.</span>show()
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points())
|
||||
plot_result(generated_images)
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="getting-results">Getting Results </h2>
|
||||
We see that the generator generates images that look like MNIST
|
||||
numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able
|
||||
to generate a similar plot where we generate every MNIST number. Let us now try
|
||||
to 'move' a bit around in the latent space. <b>Note</b>: decrease the plot number if
|
||||
these following cells take too long to run on your computer.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plot_number <span style="color: #666666">=</span> <span style="color: #666666">225</span>
|
||||
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=5</span>,
|
||||
scale_stds<span style="color: #666666">=1</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=-5</span>,
|
||||
scale_stds<span style="color: #666666">=1</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=1</span>,
|
||||
scale_stds<span style="color: #666666">=5</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
Again, we have found something interesting. <em>Moving</em> around using our means
|
||||
takes us from digit to digit, while <em>moving</em> around using our standard
|
||||
deviations seem to increase the number of different digits! In the last image
|
||||
above, we can barely make out every MNIST digit. Let us make on last plot using
|
||||
this information by upping the standard deviation of our Gaussian noises.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plot_number <span style="color: #666666">=</span> <span style="color: #666666">400</span>
|
||||
generated_images <span style="color: #666666">=</span> generate_images(generate_latent_points(number<span style="color: #666666">=</span>plot_number,
|
||||
scale_means<span style="color: #666666">=1</span>,
|
||||
scale_stds<span style="color: #666666">=10</span>))
|
||||
plot_result(generated_images, number<span style="color: #666666">=</span>plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
A pretty cool result! We see that our generator indeed has learned a
|
||||
distribution which qualitatively looks a whole lot like the MNIST dataset.
|
||||
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="interpolating-between-mnist-digits">Interpolating Between MNIST Digits </h2>
|
||||
Another interesting way to explore the latent space of our generator model is by
|
||||
interpolating between the MNIST digits. This section is largely based on
|
||||
<a href="https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/" target="_blank">this excellent blogpost</a>
|
||||
by Jason Brownlee.
|
||||
|
||||
<p>
|
||||
So let us start by defining a function to interpolate between two points in the
|
||||
latent space.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">interpolation</span>(point_1, point_2, n_steps<span style="color: #666666">=10</span>):
|
||||
ratios <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">0</span>, <span style="color: #666666">1</span>, num<span style="color: #666666">=</span>n_steps)
|
||||
vectors <span style="color: #666666">=</span> []
|
||||
<span style="color: #008000; font-weight: bold">for</span> i, ratio <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">enumerate</span>(ratios):
|
||||
vectors<span style="color: #666666">.</span>append(((<span style="color: #666666">1.0</span> <span style="color: #666666">-</span> ratio) <span style="color: #666666">*</span> point_1 <span style="color: #666666">+</span> ratio <span style="color: #666666">*</span> point_2))
|
||||
|
||||
<span style="color: #008000; font-weight: bold">return</span> tf<span style="color: #666666">.</span>stack(vectors)
|
||||
</pre></div>
|
||||
<p>
|
||||
Now we have all we need to do our interpolation analysis.
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||||
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span>plot_number <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
||||
latent_points <span style="color: #666666">=</span> generate_latent_points(number<span style="color: #666666">=</span>plot_number)
|
||||
results <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">0</span>, <span style="color: #666666">2*</span>np<span style="color: #666666">.</span>sqrt(plot_number), <span style="color: #666666">2</span>):
|
||||
interpolated <span style="color: #666666">=</span> interpolation(latent_points[i], latent_points[i<span style="color: #666666">+1</span>])
|
||||
generated_images <span style="color: #666666">=</span> generate_images(interpolated)
|
||||
|
||||
<span style="color: #008000; font-weight: bold">if</span> results <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||||
results <span style="color: #666666">=</span> generated_images
|
||||
<span style="color: #008000; font-weight: bold">else</span>:
|
||||
results <span style="color: #666666">=</span> tf<span style="color: #666666">.</span>stack((results, generated_images))
|
||||
|
||||
plot_results(results, plot_number)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- ------------------- end of main content --------------- -->
|
||||
|
||||
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -77,13 +77,11 @@ example, they can take sentences, documents, or audio samples as
|
||||
input, making them extremely useful for natural language processing
|
||||
systems such as automatic translation and speech-to-text.
|
||||
|
||||
RNNs are also used in "text classification":"Text Classification Algorithms: A Survey"
|
||||
|
||||
|
||||
!split
|
||||
===== Set up of an RNN =====
|
||||
|
||||
|
||||
More to text to be added by Wednesday October 27.
|
||||
|
||||
!split
|
||||
===== A simple example =====
|
||||
@@ -731,4 +729,607 @@ print('Time: ', end-start)
|
||||
|
||||
|
||||
|
||||
!split
|
||||
===== Generative Models =====
|
||||
|
||||
_Generative models_ describe a class of statistical models that are a contrast
|
||||
to _discriminative models_. Informally we say that generative models can
|
||||
generate new data instances while discriminative models discriminate between
|
||||
different kinds of data instances. A generative model could generate new photos
|
||||
of animals that look like 'real' animals while a discriminative model could tell
|
||||
a dog from a cat. More formally, given a data set $x$ and a set of labels /
|
||||
targets $y$. Generative models capture the joint probability $p(x, y)$, or
|
||||
just $p(x)$ if there are no labels, while discriminative models capture the
|
||||
conditional probability $p(y | x)$. Discriminative models generally try to draw
|
||||
boundaries in the data space (often high dimensional), while generative models
|
||||
try to model how data is placed throughout the space.
|
||||
|
||||
_Note_: this material is thanks to Linus Ekstrøm.
|
||||
|
||||
!split
|
||||
===== Generative Adversarial Networks =====
|
||||
|
||||
_Generative Adversarial Networks_ are a type of unsupervised machine learning
|
||||
algorithm proposed by "Goodfellow et. al": "https://arxiv.org/pdf/1406.2661.pdf"
|
||||
in 2014 (short and good article).
|
||||
|
||||
The simplest formulation of
|
||||
the model is based on a game theoretic approach, *zero sum game*, where we pit
|
||||
two neural networks against one another. We define two rival networks, one
|
||||
generator $g$, and one discriminator $d$. The generator directly produces
|
||||
samples
|
||||
!bt
|
||||
\begin{equation}
|
||||
x = g(z; \theta^{(g)})
|
||||
\end{equation}
|
||||
!et
|
||||
|
||||
|
||||
!split
|
||||
===== Discriminator =====
|
||||
The discriminator attempts to distinguish between samples drawn from the
|
||||
training data and samples drawn from the generator. In other words, it tries to
|
||||
tell the difference between the fake data produced by $g$ and the actual data
|
||||
samples we want to do prediction on. The discriminator outputs a probability
|
||||
value given by
|
||||
|
||||
!bt
|
||||
\begin{equation}
|
||||
d(x; \theta^{(d)})
|
||||
\end{equation}
|
||||
!et
|
||||
|
||||
indicating the probability that $x$ is a real training example rather than a
|
||||
fake sample the generator has generated. The simplest way to formulate the
|
||||
learning process in a generative adversarial network is a zero-sum game, in
|
||||
which a function
|
||||
|
||||
!bt
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)})
|
||||
\end{equation}
|
||||
!et
|
||||
|
||||
determines the reward for the discriminator, while the generator gets the
|
||||
conjugate reward
|
||||
|
||||
!bt
|
||||
\begin{equation}
|
||||
-v(\theta^{(g)}, \theta^{(d)})
|
||||
\end{equation}
|
||||
!et
|
||||
|
||||
|
||||
!split
|
||||
===== Learning Process =====
|
||||
|
||||
During learning both of the networks maximize their own reward function, so that
|
||||
the generator gets better and better at tricking the discriminator, while the
|
||||
discriminator gets better and better at telling the difference between the fake
|
||||
and real data. The generator and discriminator alternate on which one trains at
|
||||
one time (i.e. for one epoch). In other words, we keep the generator constant
|
||||
and train the discriminator, then we keep the discriminator constant to train
|
||||
the generator and repeat. It is this back and forth dynamic which lets GANs
|
||||
tackle otherwise intractable generative problems. As the generator improves with
|
||||
training, the discriminator's performance gets worse because it cannot easily
|
||||
tell the difference between real and fake. If the generator ends up succeeding
|
||||
perfectly, the the discriminator will do no better than random guessing i.e.
|
||||
50\%. This progression in the training poses a problem for the convergence
|
||||
criteria for GANs. The discriminator feedback gets less meaningful over time,
|
||||
if we continue training after this point then the generator is effectively
|
||||
training on junk data which can undo the learning up to that point. Therefore,
|
||||
we stop training when the discriminator starts outputting $1/2$ everywhere.
|
||||
|
||||
|
||||
!split
|
||||
===== More about the Learning Process =====
|
||||
|
||||
At convergence we have
|
||||
|
||||
!bt
|
||||
\begin{equation}
|
||||
g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\end{equation}
|
||||
!et
|
||||
The default choice for $v$ is
|
||||
!bt
|
||||
\begin{equation}
|
||||
v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x)
|
||||
+ \mathbb{E}_{x\sim p_\mathrm{model}}
|
||||
\log (1 - d(x))
|
||||
\end{equation}
|
||||
!et
|
||||
The main motivation for the design of GANs is that the learning process requires
|
||||
neither approximate inference (variational autoencoders for example) nor
|
||||
approximation of a partition function. In the case where
|
||||
!bt
|
||||
\begin{equation}
|
||||
\underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)})
|
||||
\end{equation}
|
||||
!et
|
||||
is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is
|
||||
asymptotically consistent
|
||||
( "Seth Lloyd on QuGANs": "https://arxiv.org/pdf/1804.09139.pdf" ).
|
||||
|
||||
!split
|
||||
===== Additional References =====
|
||||
This is in
|
||||
general not the case and it is possible to get situations where the training
|
||||
process never converges because the generator and discriminator chase one
|
||||
another around in the parameter space indefinitely. A much deeper discussion on
|
||||
the currently open research problem of GAN convergence is available
|
||||
"here": "https://www.deeplearningbook.org/contents/generative_models.html". To
|
||||
anyone interested in learning more about GANs it is a highly recommended read.
|
||||
Direct quote: "In this best-performing formulation, the generator aims to
|
||||
increase the log probability that the discriminator makes a mistake, rather than
|
||||
aiming to decrease the log probability that the discriminator makes the correct
|
||||
prediction." "Another interesting read": "https://arxiv.org/abs/1701.00160"
|
||||
|
||||
|
||||
!split
|
||||
===== Writing Our First Generative Adversarial Network =====
|
||||
Let us now move on to actually implementing a GAN in tensorflow. We will study
|
||||
the performance of our GAN on the MNIST dataset. This code is based on and
|
||||
adapted from the
|
||||
"google tutorial": "https://www.tensorflow.org/tutorials/generative/dcgan"
|
||||
|
||||
First we import our libraries
|
||||
|
||||
!bc pycod
|
||||
import os
|
||||
import time
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
import matplotlib.pyplot as plt
|
||||
from tensorflow.keras import layers
|
||||
from tensorflow.keras.utils import plot_model
|
||||
!ec
|
||||
|
||||
Next we define our hyperparameters and import our data the usual way
|
||||
|
||||
!bc pycod
|
||||
BUFFER_SIZE = 60000
|
||||
BATCH_SIZE = 256
|
||||
EPOCHS = 30
|
||||
|
||||
data = tf.keras.datasets.mnist.load_data()
|
||||
(train_images, train_labels), (test_images, test_labels) = data
|
||||
train_images = np.reshape(train_images, (train_images.shape[0],
|
||||
28,
|
||||
28,
|
||||
1)).astype('float32')
|
||||
|
||||
# we normalize between -1 and 1
|
||||
train_images = (train_images - 127.5) / 127.5
|
||||
training_dataset = tf.data.Dataset.from_tensor_slices(
|
||||
train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== MNIST and GANs =====
|
||||
|
||||
Let's have a quick look
|
||||
|
||||
!bc pycod
|
||||
plt.imshow(train_images[0], cmap='Greys')
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
Now we define our two models. This is where the 'magic' happens. There are a
|
||||
huge amount of possible formulations for both models. A lot of engineering and
|
||||
trial and error can be done here to try to produce better performing models. For
|
||||
more advanced GANs this is by far the step where you can 'make or break' a
|
||||
model.
|
||||
|
||||
We start with the generator. As stated in the introductory text the generator
|
||||
$g$ upsamples from a random sample to the shape of what we want to predict. In
|
||||
our case we are trying to predict MNIST images ($28\times 28$ pixels).
|
||||
|
||||
!bc pycod
|
||||
def generator_model():
|
||||
"""
|
||||
The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to
|
||||
produce an image from a random seed. We start with a Dense layer taking this
|
||||
random sample as an input and subsequently upsample through multiple
|
||||
convolutional layers.
|
||||
"""
|
||||
|
||||
# we define our model
|
||||
model = tf.keras.Sequential()
|
||||
|
||||
|
||||
# adding our input layer. Dense means that every neuron is connected and
|
||||
# the input shape is the shape of our random noise. The units need to match
|
||||
# in some sense the upsampling strides to reach our desired output shape.
|
||||
# we are using 100 random numbers as our seed
|
||||
model.add(layers.Dense(units=7*7*BATCH_SIZE,
|
||||
use_bias=False,
|
||||
input_shape=(100, )))
|
||||
# we normalize the output form the Dense layer
|
||||
model.add(layers.BatchNormalization())
|
||||
# and add an activation function to our 'layer'. LeakyReLU avoids vanishing
|
||||
# gradient problem
|
||||
model.add(layers.LeakyReLU())
|
||||
model.add(layers.Reshape((7, 7, BATCH_SIZE)))
|
||||
assert model.output_shape == (None, 7, 7, BATCH_SIZE)
|
||||
# even though we just added four keras layers we think of everything above
|
||||
# as 'one' layer
|
||||
|
||||
# next we add our upscaling convolutional layers
|
||||
model.add(layers.Conv2DTranspose(filters=128,
|
||||
kernel_size=(5, 5),
|
||||
strides=(1, 1),
|
||||
padding='same',
|
||||
use_bias=False))
|
||||
model.add(layers.BatchNormalization())
|
||||
model.add(layers.LeakyReLU())
|
||||
assert model.output_shape == (None, 7, 7, 128)
|
||||
|
||||
model.add(layers.Conv2DTranspose(filters=64,
|
||||
kernel_size=(5, 5),
|
||||
strides=(2, 2),
|
||||
padding='same',
|
||||
use_bias=False))
|
||||
model.add(layers.BatchNormalization())
|
||||
model.add(layers.LeakyReLU())
|
||||
assert model.output_shape == (None, 14, 14, 64)
|
||||
|
||||
model.add(layers.Conv2DTranspose(filters=1,
|
||||
kernel_size=(5, 5),
|
||||
strides=(2, 2),
|
||||
padding='same',
|
||||
use_bias=False,
|
||||
activation='tanh'))
|
||||
assert model.output_shape == (None, 28, 28, 1)
|
||||
|
||||
return model
|
||||
|
||||
!ec
|
||||
|
||||
And there we have our 'simple' generator model. Now we move on to defining our
|
||||
discriminator model $d$, which is a convolutional neural network based image
|
||||
classifier.
|
||||
|
||||
!bc pycod
|
||||
def discriminator_model():
|
||||
"""
|
||||
The discriminator is a convolutional neural network based image classifier
|
||||
"""
|
||||
|
||||
# we define our model
|
||||
model = tf.keras.Sequential()
|
||||
model.add(layers.Conv2D(filters=64,
|
||||
kernel_size=(5, 5),
|
||||
strides=(2, 2),
|
||||
padding='same',
|
||||
input_shape=[28, 28, 1]))
|
||||
model.add(layers.LeakyReLU())
|
||||
# adding a dropout layer as you do in conv-nets
|
||||
model.add(layers.Dropout(0.3))
|
||||
|
||||
|
||||
model.add(layers.Conv2D(filters=128,
|
||||
kernel_size=(5, 5),
|
||||
strides=(2, 2),
|
||||
padding='same'))
|
||||
model.add(layers.LeakyReLU())
|
||||
# adding a dropout layer as you do in conv-nets
|
||||
model.add(layers.Dropout(0.3))
|
||||
|
||||
model.add(layers.Flatten())
|
||||
model.add(layers.Dense(1))
|
||||
|
||||
return model
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Other Models =====
|
||||
Let us take a look at our models. _Note_: double click images for bigger view.
|
||||
|
||||
!bc pycod
|
||||
generator = generator_model()
|
||||
plot_model(generator, show_shapes=True, rankdir='LR')
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
discriminator = discriminator_model()
|
||||
plot_model(discriminator, show_shapes=True, rankdir='LR')
|
||||
!ec
|
||||
|
||||
Next we need a few helper objects we will use in training
|
||||
|
||||
!bc pycod
|
||||
cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)
|
||||
generator_optimizer = tf.keras.optimizers.Adam(1e-4)
|
||||
discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)
|
||||
!ec
|
||||
|
||||
The first object, *cross_entropy* is our loss function and the two others are
|
||||
our optimizers. Notice we use the same learning rate for both $g$ and $d$. This
|
||||
is because they need to improve their accuracy at approximately equal speeds to
|
||||
get convergence (not necessarily exactly equal). Now we define our loss
|
||||
functions
|
||||
|
||||
!bc pycod
|
||||
def generator_loss(fake_output):
|
||||
loss = cross_entropy(tf.ones_like(fake_output), fake_output)
|
||||
|
||||
return loss
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
def discriminator_loss(real_output, fake_output):
|
||||
real_loss = cross_entropy(tf.ones_like(real_output), real_output)
|
||||
fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
|
||||
total_loss = real_loss + fake_loss
|
||||
|
||||
return total_loss
|
||||
!ec
|
||||
|
||||
Next we define a kind of seed to help us compare the learning process over
|
||||
multiple training epochs.
|
||||
|
||||
!bc pycod
|
||||
noise_dimension = 100
|
||||
n_examples_to_generate = 16
|
||||
seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Training Step =====
|
||||
|
||||
Now we have everything we need to define our training step, which we will apply
|
||||
for every step in our training loop. Notice the @tf.function flag signifying
|
||||
that the function is tensorflow 'compiled'. Removing this flag doubles the
|
||||
computation time.
|
||||
|
||||
!bc pycod
|
||||
@tf.function
|
||||
def train_step(images):
|
||||
noise = tf.random.normal([BATCH_SIZE, noise_dimension])
|
||||
|
||||
with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
|
||||
generated_images = generator(noise, training=True)
|
||||
|
||||
real_output = discriminator(images, training=True)
|
||||
fake_output = discriminator(generated_images, training=True)
|
||||
|
||||
gen_loss = generator_loss(fake_output)
|
||||
disc_loss = discriminator_loss(real_output, fake_output)
|
||||
|
||||
gradients_of_generator = gen_tape.gradient(gen_loss,
|
||||
generator.trainable_variables)
|
||||
gradients_of_discriminator = disc_tape.gradient(disc_loss,
|
||||
discriminator.trainable_variables)
|
||||
generator_optimizer.apply_gradients(zip(gradients_of_generator,
|
||||
generator.trainable_variables))
|
||||
discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator,
|
||||
discriminator.trainable_variables))
|
||||
|
||||
return gen_loss, disc_loss
|
||||
!ec
|
||||
|
||||
|
||||
Next we define a helper function to produce an output over our training epochs
|
||||
to see the predictive progression of our generator model. _Note_: I am including
|
||||
this code here, but comment it out in the training loop.
|
||||
!bc pycod
|
||||
def generate_and_save_images(model, epoch, test_input):
|
||||
# we're making inferences here
|
||||
predictions = model(test_input, training=False)
|
||||
|
||||
fig = plt.figure(figsize=(4, 4))
|
||||
|
||||
for i in range(predictions.shape[0]):
|
||||
plt.subplot(4, 4, i+1)
|
||||
plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')
|
||||
plt.axis('off')
|
||||
|
||||
plt.savefig(f'./images_from_seed_images/image_at_epoch_{str(epoch).zfill(3)}.png')
|
||||
plt.close()
|
||||
#plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Checkpoints =====
|
||||
Setting up checkpoints to periodically save our model during training so that
|
||||
everything is not lost even if the program were to somehow terminate while
|
||||
training.
|
||||
|
||||
!bc pycod
|
||||
# Setting up checkpoints to save model during training
|
||||
checkpoint_dir = './training_checkpoints'
|
||||
checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')
|
||||
checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
|
||||
discriminator_optimizer=discriminator_optimizer,
|
||||
generator=generator,
|
||||
discriminator=discriminator)
|
||||
!ec
|
||||
|
||||
Now we define our training loop
|
||||
|
||||
!bc pycod
|
||||
def train(dataset, epochs):
|
||||
generator_loss_list = []
|
||||
discriminator_loss_list = []
|
||||
|
||||
for epoch in range(epochs):
|
||||
start = time.time()
|
||||
|
||||
for image_batch in dataset:
|
||||
gen_loss, disc_loss = train_step(image_batch)
|
||||
generator_loss_list.append(gen_loss.numpy())
|
||||
discriminator_loss_list.append(disc_loss.numpy())
|
||||
|
||||
#generate_and_save_images(generator, epoch + 1, seed_images)
|
||||
|
||||
if (epoch + 1) % 15 == 0:
|
||||
checkpoint.save(file_prefix=checkpoint_prefix)
|
||||
|
||||
print(f'Time for epoch {epoch} is {time.time() - start}')
|
||||
|
||||
#generate_and_save_images(generator, epochs, seed_images)
|
||||
|
||||
loss_file = './data/lossfile.txt'
|
||||
with open(loss_file, 'w') as outfile:
|
||||
outfile.write(str(generator_loss_list))
|
||||
outfile.write('\n')
|
||||
outfile.write('\n')
|
||||
outfile.write(str(discriminator_loss_list))
|
||||
outfile.write('\n')
|
||||
outfile.write('\n')
|
||||
!ec
|
||||
|
||||
|
||||
To train simply call this function. _Warning_: this might take a long time so
|
||||
there is a folder of a pretrained network already included in the repository.
|
||||
|
||||
!bc pycod
|
||||
train(train_dataset, EPOCHS)
|
||||
!ec
|
||||
|
||||
And here is the result of training our model for 100 epochs
|
||||
|
||||
MOVIE: [images_from_seed_images/generation.gif]
|
||||
|
||||
Now to avoid having to train and everything, which will take a while depending
|
||||
on your computer setup we now load in the model which produced the above gif.
|
||||
|
||||
!bc pycod
|
||||
checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
|
||||
restored_generator = checkpoint.generator
|
||||
restored_discriminator = checkpoint.discriminator
|
||||
|
||||
print(restored_generator)
|
||||
print(restored_discriminator)
|
||||
!ec
|
||||
|
||||
|
||||
!split
|
||||
===== Exploring the Latent Space =====
|
||||
|
||||
We have successfully loaded in our latest model. Let us now play around a bit
|
||||
and see what kind of things we can learn about this model. Our generator takes
|
||||
an array of 100 numbers. One idea can be to try to systematically change our
|
||||
input. Let us try and see what we get
|
||||
|
||||
!bc pycod
|
||||
def generate_latent_points(number=100, scale_means=1, scale_stds=1):
|
||||
latent_dim = 100
|
||||
means = scale_means * tf.linspace(-1, 1, num=latent_dim)
|
||||
stds = scale_stds * tf.linspace(-1, 1, num=latent_dim)
|
||||
latent_space_value_range = tf.random.normal([number, latent_dim],
|
||||
means,
|
||||
stds,
|
||||
dtype=tf.float64)
|
||||
|
||||
return latent_space_value_range
|
||||
|
||||
def generate_images(latent_points):
|
||||
# notice we set training to false because we are making inferences
|
||||
generated_images = restored_generator.predict(latent_points)
|
||||
|
||||
return generated_images
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
def plot_result(generated_images, number=100):
|
||||
# obviously this assumes sqrt number is an int
|
||||
fig, axs = plt.subplots(int(np.sqrt(number)), int(np.sqrt(number)),
|
||||
figsize=(10, 10))
|
||||
|
||||
for i in range(int(np.sqrt(number))):
|
||||
for j in range(int(np.sqrt(number))):
|
||||
axs[i, j].imshow(generated_images[i*j], cmap='Greys')
|
||||
axs[i, j].axis('off')
|
||||
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
!bc pycod
|
||||
generated_images = generate_images(generate_latent_points())
|
||||
plot_result(generated_images)
|
||||
!ec
|
||||
|
||||
!split
|
||||
===== Getting Results =====
|
||||
We see that the generator generates images that look like MNIST
|
||||
numbers: $1, 4, 7, 9$. Let's try to tweak it a bit more to see if we are able
|
||||
to generate a similar plot where we generate every MNIST number. Let us now try
|
||||
to 'move' a bit around in the latent space. _Note_: decrease the plot number if
|
||||
these following cells take too long to run on your computer.
|
||||
|
||||
!bc pycod
|
||||
plot_number = 225
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=5,
|
||||
scale_stds=1))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=-5,
|
||||
scale_stds=1))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=1,
|
||||
scale_stds=5))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
!ec
|
||||
|
||||
Again, we have found something interesting. *Moving* around using our means
|
||||
takes us from digit to digit, while *moving* around using our standard
|
||||
deviations seem to increase the number of different digits! In the last image
|
||||
above, we can barely make out every MNIST digit. Let us make on last plot using
|
||||
this information by upping the standard deviation of our Gaussian noises.
|
||||
|
||||
!bc pycod
|
||||
plot_number = 400
|
||||
generated_images = generate_images(generate_latent_points(number=plot_number,
|
||||
scale_means=1,
|
||||
scale_stds=10))
|
||||
plot_result(generated_images, number=plot_number)
|
||||
!ec
|
||||
A pretty cool result! We see that our generator indeed has learned a
|
||||
distribution which qualitatively looks a whole lot like the MNIST dataset.
|
||||
|
||||
!split
|
||||
===== Interpolating Between MNIST Digits =====
|
||||
Another interesting way to explore the latent space of our generator model is by
|
||||
interpolating between the MNIST digits. This section is largely based on
|
||||
"this excellent blogpost": "https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/"
|
||||
by Jason Brownlee.
|
||||
|
||||
So let us start by defining a function to interpolate between two points in the
|
||||
latent space.
|
||||
|
||||
!bc pycod
|
||||
def interpolation(point_1, point_2, n_steps=10):
|
||||
ratios = np.linspace(0, 1, num=n_steps)
|
||||
vectors = []
|
||||
for i, ratio in enumerate(ratios):
|
||||
vectors.append(((1.0 - ratio) * point_1 + ratio * point_2))
|
||||
|
||||
return tf.stack(vectors)
|
||||
!ec
|
||||
|
||||
Now we have all we need to do our interpolation analysis.
|
||||
|
||||
!bc pycod
|
||||
plot_number = 100
|
||||
latent_points = generate_latent_points(number=plot_number)
|
||||
results = None
|
||||
for i in range(0, 2*np.sqrt(plot_number), 2):
|
||||
interpolated = interpolation(latent_points[i], latent_points[i+1])
|
||||
generated_images = generate_images(interpolated)
|
||||
|
||||
if results is None:
|
||||
results = generated_images
|
||||
else:
|
||||
results = tf.stack((results, generated_images))
|
||||
|
||||
plot_results(results, plot_number)
|
||||
!ec
|
||||
|
||||
Reference in New Issue
Block a user