Morten Hjorth-Jensen
1673 lines
125 KiB
HTML
1673 lines
125 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>16. Convolutional Neural Networks — Applied Data Analysis and Machine Learning</title>
|
||
|
||
<link href="_static/css/theme.css" rel="stylesheet">
|
||
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-book-theme.css?digest=c3fdc42140077d1ad13ad2f1588a4309" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||
|
||
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
|
||
|
||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||
<script src="_static/jquery.js"></script>
|
||
<script src="_static/underscore.js"></script>
|
||
<script src="_static/doctools.js"></script>
|
||
<script src="_static/clipboard.min.js"></script>
|
||
<script src="_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||
const thebe_selector = ".thebe,.cell"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output, .cell_output"
|
||
</script>
|
||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="17. Recurrent neural networks: Overarching view" href="chapter13.html" />
|
||
<link rel="prev" title="15. Solving Differential Equations with Deep Learning" href="chapter11.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta name="docsearch:language" content="None">
|
||
|
||
|
||
<!-- Google Analytics -->
|
||
|
||
</head>
|
||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||
|
||
<div class="container-fluid" id="banner"></div>
|
||
|
||
|
||
|
||
<div class="container-xl">
|
||
<div class="row">
|
||
|
||
<div class="col-12 col-md-3 bd-sidebar site-navigation show" id="site-navigation">
|
||
|
||
<div class="navbar-brand-box">
|
||
<a class="navbar-brand text-wrap" href="index.html">
|
||
|
||
<!-- `logo` is deprecated in Sphinx 4.0, so remove this when we stop supporting 3 -->
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo" alt="logo">
|
||
|
||
|
||
<h1 class="site-logo" id="site-title">Applied Data Analysis and Machine Learning</h1>
|
||
|
||
</a>
|
||
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
|
||
<i class="icon fas fa-search"></i>
|
||
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search this book..." aria-label="Search this book..." autocomplete="off" >
|
||
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item active">
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
About the course
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="schedule.html">
|
||
Teaching schedule with links to material
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="teachers.html">
|
||
Teachers and Grading
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="textbooks.html">
|
||
Textbooks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Review of Statistics with Resampling Techniques and Linear Algebra
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="statistics.html">
|
||
1. Elements of Probability Theory and Statistical Data Analysis
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="linalg.html">
|
||
2. Linear Algebra, Handling of Arrays and more Python Features
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
From Regression to Support Vector Machines
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter1.html">
|
||
3. Linear Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter2.html">
|
||
4. Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter3.html">
|
||
5. Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter4.html">
|
||
6. Logistic Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapteroptimization.html">
|
||
7. Optimization, the central part of any Machine Learning algortithm
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter5.html">
|
||
8. Support Vector Machines, overarching aims
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Decision Trees, Ensemble Methods and Boosting
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter6.html">
|
||
9. Decision trees, overarching aims
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter7.html">
|
||
10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Dimensionality Reduction
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter8.html">
|
||
11. Basic ideas of the Principal Component Analysis (PCA)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="clustering.html">
|
||
12. Clustering and Unsupervised Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Deep Learning Methods
|
||
</span>
|
||
</p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter9.html">
|
||
13. Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter10.html">
|
||
14. Building a Feed Forward Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter11.html">
|
||
15. Solving Differential Equations with Deep Learning
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1 current active">
|
||
<a class="current reference internal" href="#">
|
||
16. Convolutional Neural Networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter13.html">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Weekly material, notes and exercises
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek34.html">
|
||
Exercises week 34
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week34.html">
|
||
Week 34: Introduction to the course, Logistics and Practicalities
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek35.html">
|
||
Exercises week 35
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week35.html">
|
||
Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek36.html">
|
||
Exercises week 36
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week36.html">
|
||
Week 36: Statistical interpretation of Linear Regression and Resampling techniques
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek37.html">
|
||
Exercises week 37
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week37.html">
|
||
Week 37: Statistical interpretations and Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek38.html">
|
||
Exercises week 38
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week38.html">
|
||
Week 38: Logistic Regression and Optimization
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek39.html">
|
||
Exercises week 39
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week39.html">
|
||
Week 39: Optimization and Gradient Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week40.html">
|
||
Week 40: Gradient descent methods (continued) and start Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek41.html">
|
||
Exercises week 41
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week41.html">
|
||
Week 41 Neural networks and constructing a neural network code
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek42.html">
|
||
Exercises week 42
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week42.html">
|
||
Week 42 Constructing a Neural Network code with introduction to Tensor flow
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek43.html">
|
||
Exercises weeks 43 and 44
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week43.html">
|
||
Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week44.html">
|
||
Week 44, Convolutional Neural Networks (CNN)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week45.html">
|
||
Week 45, Recurrent Neural Networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week46.html">
|
||
Week 46: Decision Trees, Ensemble methods and Random Forests
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week47.html">
|
||
Week 47: From Decision Trees to Ensemble Methods, Random Forests and Boosting Methods and Summary of Course
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Projects
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project1.html">
|
||
Project 1 on Machine Learning, deadline October 9 (midnight), 2023
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project2.html">
|
||
Project 2 on Machine Learning, deadline November 17 (Midnight)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="project3.html">
|
||
Project 3 on Machine Learning, deadline December 18 (midnight), 2023
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav> <!-- To handle the deprecated key -->
|
||
|
||
<div class="navbar_extra_footer">
|
||
Powered by <a href="https://jupyterbook.org">Jupyter Book</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
|
||
|
||
<div class="topbar container-xl fixed-top">
|
||
<div class="topbar-contents row">
|
||
<div class="col-12 col-md-3 bd-topbar-whitespace site-navigation show"></div>
|
||
<div class="col pl-md-4 topbar-main">
|
||
|
||
<button id="navbar-toggler" class="navbar-toggler ml-0" type="button" data-toggle="collapse"
|
||
data-toggle="tooltip" data-placement="bottom" data-target=".site-navigation" aria-controls="navbar-menu"
|
||
aria-expanded="true" aria-label="Toggle navigation" aria-controls="site-navigation"
|
||
title="Toggle navigation" data-toggle="tooltip" data-placement="left">
|
||
<i class="fas fa-bars"></i>
|
||
<i class="fas fa-arrow-left"></i>
|
||
<i class="fas fa-arrow-up"></i>
|
||
</button>
|
||
|
||
|
||
<div class="dropdown-buttons-trigger">
|
||
<button id="dropdown-buttons-trigger" class="btn btn-secondary topbarbtn" aria-label="Download this page"><i
|
||
class="fas fa-download"></i></button>
|
||
|
||
<div class="dropdown-buttons">
|
||
<!-- ipynb file if we had a myst markdown file -->
|
||
|
||
<!-- Download raw file -->
|
||
<a class="dropdown-buttons" href="_sources/chapter12.ipynb"><button type="button"
|
||
class="btn btn-secondary topbarbtn" title="Download source file" data-toggle="tooltip"
|
||
data-placement="left">.ipynb</button></a>
|
||
<!-- Download PDF via print -->
|
||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Source interaction buttons -->
|
||
|
||
<!-- Full screen (wrap in <a> to have style consistency -->
|
||
|
||
<a class="full-screen-button"><button type="button" class="btn btn-secondary topbarbtn" data-toggle="tooltip"
|
||
data-placement="bottom" onclick="toggleFullScreen()" aria-label="Fullscreen mode"
|
||
title="Fullscreen mode"><i
|
||
class="fas fa-expand"></i></button></a>
|
||
|
||
<!-- Launch buttons -->
|
||
|
||
</div>
|
||
|
||
<!-- Table of contents -->
|
||
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
|
||
|
||
<div class="tocsection onthispage pt-5 pb-3">
|
||
<i class="fas fa-list"></i> Contents
|
||
</div>
|
||
<nav id="bd-toc-nav" aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#neural-networks-vs-cnns">
|
||
16.1. Neural Networks vs CNNs
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#layers-used-to-build-cnns">
|
||
16.2. Layers used to build CNNs
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mathematics-of-cnns">
|
||
16.3. Mathematics of CNNs
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#convolution-examples-polynomial-multiplication">
|
||
16.3.1. Convolution Examples: Polynomial multiplication
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms">
|
||
16.3.2. Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#two-dimensional-objects">
|
||
16.4. Two-dimensional Objects
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-on-dimensionalities">
|
||
16.5. More on Dimensionalities
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#further-dimensionality-remarks">
|
||
16.6. Further Dimensionality Remarks
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras">
|
||
16.7. CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-mnist-dataset-again">
|
||
16.7.1. The MNIST dataset again
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#systematic-reduction">
|
||
16.7.2. Systematic reduction
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#prerequisites-collect-and-pre-process-data">
|
||
16.7.3. Prerequisites: Collect and pre-process data
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-cifar01-data-set">
|
||
16.8. The CIFAR01 data set
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="main-content" class="row">
|
||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||
<!-- Table of contents that is only displayed when printing the page -->
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Convolutional Neural Networks</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#neural-networks-vs-cnns">
|
||
16.1. Neural Networks vs CNNs
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#layers-used-to-build-cnns">
|
||
16.2. Layers used to build CNNs
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mathematics-of-cnns">
|
||
16.3. Mathematics of CNNs
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#convolution-examples-polynomial-multiplication">
|
||
16.3.1. Convolution Examples: Polynomial multiplication
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms">
|
||
16.3.2. Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#two-dimensional-objects">
|
||
16.4. Two-dimensional Objects
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-on-dimensionalities">
|
||
16.5. More on Dimensionalities
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#further-dimensionality-remarks">
|
||
16.6. Further Dimensionality Remarks
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras">
|
||
16.7. CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-mnist-dataset-again">
|
||
16.7.1. The MNIST dataset again
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#systematic-reduction">
|
||
16.7.2. Systematic reduction
|
||
</a>
|
||
</li>
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#prerequisites-collect-and-pre-process-data">
|
||
16.7.3. Prerequisites: Collect and pre-process data
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-cifar01-data-set">
|
||
16.8. The CIFAR01 data set
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html chapter12.do.txt --><div class="tex2jax_ignore mathjax_ignore section" id="convolutional-neural-networks">
|
||
<h1><span class="section-number">16. </span>Convolutional Neural Networks<a class="headerlink" href="#convolutional-neural-networks" title="Permalink to this headline">¶</a></h1>
|
||
<p>Convolutional neural networks (CNNs) were developed during the last
|
||
decade of the previous century, with a focus on character recognition
|
||
tasks. Nowadays, CNNs are a central element in the spectacular success
|
||
of deep learning methods. The success in for example image
|
||
classifications have made them a central tool for most machine
|
||
learning practitioners.</p>
|
||
<p>CNNs are very similar to ordinary Neural Networks.
|
||
They are made up of neurons that have learnable weights and
|
||
biases. Each neuron receives some inputs, performs a dot product and
|
||
optionally follows it with a non-linearity. The whole network still
|
||
expresses a single differentiable score function: from the raw image
|
||
pixels on one end to class scores at the other. And they still have a
|
||
loss function (for example Softmax) on the last (fully-connected) layer
|
||
and all the tips/tricks we developed for learning regular Neural
|
||
Networks still apply (back propagation, gradient descent etc etc).</p>
|
||
<p><strong>CNN architectures make the explicit assumption that
|
||
the inputs are images, which allows us to encode certain properties
|
||
into the architecture. These then make the forward function more
|
||
efficient to implement and vastly reduce the amount of parameters in
|
||
the network.</strong></p>
|
||
<p>Here we provide only a superficial overview, for the more interested, we recommend highly the course
|
||
<a class="reference external" href="https://www.uio.no/studier/emner/matnat/ifi/IN5400/index-eng.html">IN5400 – Machine Learning for Image Analysis</a>
|
||
and the slides of <a class="reference external" href="http://cs231n.github.io/convolutional-networks/">CS231</a>.</p>
|
||
<p>Another good read is the article here <a class="reference external" href="https://arxiv.org/pdf/1603.07285.pdf">https://arxiv.org/pdf/1603.07285.pdf</a>.</p>
|
||
<div class="section" id="neural-networks-vs-cnns">
|
||
<h2><span class="section-number">16.1. </span>Neural Networks vs CNNs<a class="headerlink" href="#neural-networks-vs-cnns" title="Permalink to this headline">¶</a></h2>
|
||
<p>Neural networks are defined as <strong>affine transformations</strong>, that is
|
||
a vector is received as input and is multiplied with a matrix of so-called weights (our unknown paramters) to produce an
|
||
output (to which a bias vector is usually added before passing the result
|
||
through a nonlinear activation function). This is applicable to any type of input, be it an
|
||
image, a sound clip or an unordered collection of features: whatever their
|
||
dimensionality, their representation can always be flattened into a vector
|
||
before the transformation.</p>
|
||
<p>However, when we consider images, sound clips and many other similar kinds of data, these data have an intrinsic
|
||
structure. More formally, they share these important properties:</p>
|
||
<ul class="simple">
|
||
<li><p>They are stored as multi-dimensional arrays (think of the pixels of a figure) .</p></li>
|
||
<li><p>They feature one or more axes for which ordering matters (e.g., width and height axes for an image, time axis for a sound clip).</p></li>
|
||
<li><p>One axis, called the channel axis, is used to access different views of the data (e.g., the red, green and blue channels of a color image, or the left and right channels of a stereo audio track).</p></li>
|
||
</ul>
|
||
<p>These properties are not exploited when an affine transformation is applied; in
|
||
fact, all the axes are treated in the same way and the topological information
|
||
is not taken into account. Still, taking advantage of the implicit structure of
|
||
the data may prove very handy in solving some tasks, like computer vision and
|
||
speech recognition, and in these cases it would be best to preserve it. This is
|
||
where discrete convolutions come into play.</p>
|
||
<p>A discrete convolution is a linear transformation that preserves this notion of
|
||
ordering. It is sparse (only a few input units contribute to a given output
|
||
unit) and reuses parameters (the same weights are applied to multiple locations
|
||
in the input).</p>
|
||
<p>As an example, consider
|
||
an image of size <span class="math notranslate nohighlight">\(32\times 32\times 3\)</span> (32 wide, 32 high, 3 color channels), so a
|
||
single fully-connected neuron in a first hidden layer of a regular
|
||
Neural Network would have <span class="math notranslate nohighlight">\(32\times 32\times 3 = 3072\)</span> weights. This amount still
|
||
seems manageable, but clearly this fully-connected structure does not
|
||
scale to larger images. For example, an image of more respectable
|
||
size, say <span class="math notranslate nohighlight">\(200\times 200\times 3\)</span>, would lead to neurons that have
|
||
<span class="math notranslate nohighlight">\(200\times 200\times 3 = 120,000\)</span> weights.</p>
|
||
<p>We could have
|
||
several such neurons, and the parameters would add up quickly! Clearly,
|
||
this full connectivity is wasteful and the huge number of parameters
|
||
would quickly lead to possible overfitting.</p>
|
||
<!-- dom:FIGURE: [figslides/nn.jpeg, width=500 frac=0.6] A regular 3-layer Neural Network. -->
|
||
<!-- begin figure -->
|
||
<p><img src="figslides/nn.jpeg" width="500"><p style="font-size: 0.9em"><i>Figure 1: A regular 3-layer Neural Network.</i></p></p>
|
||
<!-- end figure -->
|
||
<p>Convolutional Neural Networks take advantage of the fact that the
|
||
input consists of images and they constrain the architecture in a more
|
||
sensible way.</p>
|
||
<p>In particular, unlike a regular Neural Network, the
|
||
layers of a CNN have neurons arranged in 3 dimensions: width,
|
||
height, depth. (Note that the word depth here refers to the third
|
||
dimension of an activation volume, not to the depth of a full Neural
|
||
Network, which can refer to the total number of layers in a network.)</p>
|
||
<p>To understand it better, the above example of an image
|
||
with an input volume of
|
||
activations has dimensions <span class="math notranslate nohighlight">\(32\times 32\times 3\)</span> (width, height,
|
||
depth respectively).</p>
|
||
<p>The neurons in a layer will
|
||
only be connected to a small region of the layer before it, instead of
|
||
all of the neurons in a fully-connected manner. Moreover, the final
|
||
output layer could for this specific image have dimensions <span class="math notranslate nohighlight">\(1\times 1 \times 10\)</span>,
|
||
because by the
|
||
end of the CNN architecture we will reduce the full image into a
|
||
single vector of class scores, arranged along the depth
|
||
dimension.</p>
|
||
<!-- dom:FIGURE: [figslides/cnn.jpeg, width=500 frac=0.6] A CNN arranges its neurons in three dimensions (width, height, depth), as visualized in one of the layers. Every layer of a CNN transforms the 3D input volume to a 3D output volume of neuron activations. In this example, the red input layer holds the image, so its width and height would be the dimensions of the image, and the depth would be 3 (Red, Green, Blue channels). -->
|
||
<!-- begin figure -->
|
||
<p><img src="figslides/cnn.jpeg" width="500"><p style="font-size: 0.9em"><i>Figure 1: A CNN arranges its neurons in three dimensions (width, height, depth), as visualized in one of the layers. Every layer of a CNN transforms the 3D input volume to a 3D output volume of neuron activations. In this example, the red input layer holds the image, so its width and height would be the dimensions of the image, and the depth would be 3 (Red, Green, Blue channels).</i></p></p>
|
||
<!-- end figure --></div>
|
||
<div class="section" id="layers-used-to-build-cnns">
|
||
<h2><span class="section-number">16.2. </span>Layers used to build CNNs<a class="headerlink" href="#layers-used-to-build-cnns" title="Permalink to this headline">¶</a></h2>
|
||
<p>A simple CNN is a sequence of layers, and every layer of a CNN
|
||
transforms one volume of activations to another through a
|
||
differentiable function. We use three main types of layers to build
|
||
CNN architectures: Convolutional Layer, Pooling Layer, and
|
||
Fully-Connected Layer (exactly as seen in regular Neural Networks). We
|
||
will stack these layers to form a full CNN architecture.</p>
|
||
<p>A simple CNN for image classification could have the architecture:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>INPUT</strong> (<span class="math notranslate nohighlight">\(32\times 32 \times 3\)</span>) will hold the raw pixel values of the image, in this case an image of width 32, height 32, and with three color channels R,G,B.</p></li>
|
||
<li><p><strong>CONV</strong> (convolutional )layer will compute the output of neurons that are connected to local regions in the input, each computing a dot product between their weights and a small region they are connected to in the input volume. This may result in volume such as <span class="math notranslate nohighlight">\([32\times 32\times 12]\)</span> if we decided to use 12 filters.</p></li>
|
||
<li><p><strong>RELU</strong> layer will apply an elementwise activation function, such as the <span class="math notranslate nohighlight">\(max(0,x)\)</span> thresholding at zero. This leaves the size of the volume unchanged (<span class="math notranslate nohighlight">\([32\times 32\times 12]\)</span>).</p></li>
|
||
<li><p><strong>POOL</strong> (pooling) layer will perform a downsampling operation along the spatial dimensions (width, height), resulting in volume such as <span class="math notranslate nohighlight">\([16\times 16\times 12]\)</span>.</p></li>
|
||
<li><p><strong>FC</strong> (i.e. fully-connected) layer will compute the class scores, resulting in volume of size <span class="math notranslate nohighlight">\([1\times 1\times 10]\)</span>, where each of the 10 numbers correspond to a class score, such as among the 10 categories of the MNIST images we considered above . As with ordinary Neural Networks and as the name implies, each neuron in this layer will be connected to all the numbers in the previous volume.</p></li>
|
||
</ul>
|
||
<p>CNNs transform the original image layer by layer from the original
|
||
pixel values to the final class scores.</p>
|
||
<p>Observe that some layers contain
|
||
parameters and other don’t. In particular, the CNN layers perform
|
||
transformations that are a function of not only the activations in the
|
||
input volume, but also of the parameters (the weights and biases of
|
||
the neurons). On the other hand, the RELU/POOL layers will implement a
|
||
fixed function. The parameters in the CONV/FC layers will be trained
|
||
with gradient descent so that the class scores that the CNN computes
|
||
are consistent with the labels in the training set for each image.</p>
|
||
<p>In summary:</p>
|
||
<ul class="simple">
|
||
<li><p>A CNN architecture is in the simplest case a list of Layers that transform the image volume into an output volume (e.g. holding the class scores)</p></li>
|
||
<li><p>There are a few distinct types of Layers (e.g. CONV/FC/RELU/POOL are by far the most popular)</p></li>
|
||
<li><p>Each Layer accepts an input 3D volume and transforms it to an output 3D volume through a differentiable function</p></li>
|
||
<li><p>Each Layer may or may not have parameters (e.g. CONV/FC do, RELU/POOL don’t)</p></li>
|
||
<li><p>Each Layer may or may not have additional hyperparameters (e.g. CONV/FC/POOL do, RELU doesn’t)</p></li>
|
||
</ul>
|
||
<p>A dense neural network is representd by an affine operation (like matrix-matrix multiplication) where all parameters are included.</p>
|
||
<p>The key idea in CNNs for say imaging is that in images neighbor pixels tend to be related! So we connect
|
||
only neighboring neurons in the input instead of connecting all with the first hidden layer.</p>
|
||
<p>We say we perform a filtering (convolution is the mathematical operation).</p>
|
||
</div>
|
||
<div class="section" id="mathematics-of-cnns">
|
||
<h2><span class="section-number">16.3. </span>Mathematics of CNNs<a class="headerlink" href="#mathematics-of-cnns" title="Permalink to this headline">¶</a></h2>
|
||
<p>The mathematics of CNNs is based on the mathematical operation of
|
||
<strong>convolution</strong>. In mathematics (in particular in functional analysis),
|
||
convolution is represented by matheematical operation (integration,
|
||
summation etc) on two function in order to produce a third function
|
||
that expresses how the shape of one gets modified by the other.
|
||
Convolution has a plethora of applications in a variety of disciplines, spanning from statistics to signal processing, computer vision, solutions of differential equations,linear algebra, engineering, and yes, machine learning.</p>
|
||
<p>Mathematically, convolution is defined as follows (one-dimensional example):
|
||
Let us define a continuous function <span class="math notranslate nohighlight">\(y(t)\)</span> given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y(t) = \int x(a) w(t-a) da,
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(x(a)\)</span> represents a so-called input and <span class="math notranslate nohighlight">\(w(t-a)\)</span> is normally called the weight function or kernel.</p>
|
||
<p>The above integral is written in a more compact form as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y(t) = \left(x * w\right)(t).
|
||
\]</div>
|
||
<p>The discretized version reads</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y(t) = \sum_{a=-\infty}^{a=\infty}x(a)w(t-a).
|
||
\]</div>
|
||
<p>Computing the inverse of the above convolution operations is known as deconvolution.</p>
|
||
<p>How can we use this? And what does it mean? Let us study some familiar examples first.</p>
|
||
<div class="section" id="convolution-examples-polynomial-multiplication">
|
||
<h3><span class="section-number">16.3.1. </span>Convolution Examples: Polynomial multiplication<a class="headerlink" href="#convolution-examples-polynomial-multiplication" title="Permalink to this headline">¶</a></h3>
|
||
<p>We have already met such an example in project 1 when we tried to set
|
||
up the design matrix for a two-dimensional function. This was an
|
||
example of polynomial multiplication. Let us recast such a problem in terms of the convolution operation.
|
||
Let us look a the following polynomials to second and third order, respectively:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
p(t) = \alpha_0+\alpha_1 t+\alpha_2 t^2,
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
s(t) = \beta_0+\beta_1 t+\beta_2 t^2+\beta_3 t^3.
|
||
\]</div>
|
||
<p>The polynomial multiplication gives us a new polynomial of degree <span class="math notranslate nohighlight">\(5\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
z(t) = \delta_0+\delta_1 t+\delta_2 t^2+\delta_3 t^3+\delta_4 t^4+\delta_5 t^5.
|
||
\]</div>
|
||
<p>Computing polynomial products can be implemented efficiently if we rewrite the more brute force multiplications using convolution.
|
||
We note first that the new coefficients are given as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{split}
|
||
\delta_0=&\alpha_0\beta_0\\
|
||
\delta_1=&\alpha_1\beta_0+\alpha_1\beta_0\\
|
||
\delta_2=&\alpha_0\beta_2+\alpha_1\beta_1+\alpha_2\beta_0\\
|
||
\delta_3=&\alpha_1\beta_2+\alpha_2\beta_1+\alpha_0\beta_3\\
|
||
\delta_4=&\alpha_2\beta_2+\alpha_1\beta_3\\
|
||
\delta_5=&\alpha_2\beta_3.\\
|
||
\end{split}
|
||
\end{split}\]</div>
|
||
<p>We note that <span class="math notranslate nohighlight">\(\alpha_i=0\)</span> except for <span class="math notranslate nohighlight">\(i\in \left\{0,1,2\right\}\)</span> and <span class="math notranslate nohighlight">\(\beta_i=0\)</span> except for <span class="math notranslate nohighlight">\(i\in\left\{0,1,2,3\right\}\)</span>.</p>
|
||
<p>We can then rewrite the coefficients <span class="math notranslate nohighlight">\(\delta_j\)</span> using a discrete convolution as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\delta_j = \sum_{i=-\infty}^{i=\infty}\alpha_i\beta_{j-i}=(\alpha * \beta)_j,
|
||
\]</div>
|
||
<p>or as a double sum with restriction <span class="math notranslate nohighlight">\(l=i+j\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\delta_l = \sum_{ij}\alpha_i\beta_{j}.
|
||
\]</div>
|
||
<p>Do you see a potential drawback with these equations?</p>
|
||
<p>Since we only have a finite number of <span class="math notranslate nohighlight">\(\alpha\)</span> and <span class="math notranslate nohighlight">\(\beta\)</span> values
|
||
which are non-zero, we can rewrite the above convolution expressions
|
||
as a matrix-vector multiplication</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\delta}=\begin{bmatrix}\alpha_0 & 0 & 0 & 0 \\
|
||
\alpha_1 & \alpha_0 & 0 & 0 \\
|
||
\alpha_2 & \alpha_1 & \alpha_0 & 0 \\
|
||
0 & \alpha_2 & \alpha_1 & \alpha_0 \\
|
||
0 & 0 & \alpha_2 & \alpha_1 \\
|
||
0 & 0 & 0 & \alpha_2
|
||
\end{bmatrix}\begin{bmatrix} \beta_0 \\ \beta_1 \\ \beta_2 \\ \beta_3\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>The process is commutative and we can easily see that we can rewrite the multiplication in terms of a matrix holding <span class="math notranslate nohighlight">\(\beta\)</span> and a vector holding <span class="math notranslate nohighlight">\(\alpha\)</span>.
|
||
In this case we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\delta}=\begin{bmatrix}\beta_0 & 0 & 0 \\
|
||
\beta_1 & \beta_0 & 0 \\
|
||
\beta_2 & \beta_1 & \beta_0 \\
|
||
\beta_3 & \beta_2 & \beta_1 \\
|
||
0 & \beta_3 & \beta_2 \\
|
||
0 & 0 & \beta_3
|
||
\end{bmatrix}\begin{bmatrix} \alpha_0 \\ \alpha_1 \\ \alpha_2\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>Note that the use of these matrices is for mathematical purposes only and not implementation purposes.
|
||
When implementing the above equation we do not encode (and allocate memory) the matrices explicitely.
|
||
We rather code the convolutions in the minimal memory footprint that they require.</p>
|
||
<p>Does the number of floating point operations change here when we use the commutative property?</p>
|
||
</div>
|
||
<div class="section" id="convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms">
|
||
<h3><span class="section-number">16.3.2. </span>Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)<a class="headerlink" href="#convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms" title="Permalink to this headline">¶</a></h3>
|
||
<p>For problems with so-called harmonic oscillations, given by for example the following differential equation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
m\frac{d^2x}{dt^2}+\eta\frac{dx}{dt}+x(t)=F(t),
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(F(t)\)</span> is an applied external force acting on the system (often called a driving force), one can use the theory of Fourier transformations to find the solutions of this type of equations.</p>
|
||
<p>If one has several driving forces, <span class="math notranslate nohighlight">\(F(t)=\sum_n F_n(t)\)</span>, one can find
|
||
the particular solution to each <span class="math notranslate nohighlight">\(F_n\)</span>, <span class="math notranslate nohighlight">\(x_{pn}(t)\)</span>, and the particular
|
||
solution for the entire driving force is then given by a series like</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
x_p(t)=\sum_nx_{pn}(t).
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>This is known as the principle of superposition. It only applies when
|
||
the homogenous equation is linear. If there were an anharmonic term
|
||
such as <span class="math notranslate nohighlight">\(x^3\)</span> in the homogenous equation, then when one summed various
|
||
solutions, <span class="math notranslate nohighlight">\(x=(\sum_n x_n)^2\)</span>, one would get cross
|
||
terms. Superposition is especially useful when <span class="math notranslate nohighlight">\(F(t)\)</span> can be written
|
||
as a sum of sinusoidal terms, because the solutions for each
|
||
sinusoidal (sine or cosine) term is analytic.</p>
|
||
<p>Driving forces are often periodic, even when they are not
|
||
sinusoidal. Periodicity implies that for some time <span class="math notranslate nohighlight">\(\tau\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{eqnarray}
|
||
F(t+\tau)=F(t).
|
||
\end{eqnarray}
|
||
\]</div>
|
||
<p>One example of a non-sinusoidal periodic force is a square wave. Many
|
||
components in electric circuits are non-linear, e.g. diodes, which
|
||
makes many wave forms non-sinusoidal even when the circuits are being
|
||
driven by purely sinusoidal sources.</p>
|
||
<p>The code here shows a typical example of such a square wave generated using the functionality included in the <strong>scipy</strong> Python package. We have used a period of <span class="math notranslate nohighlight">\(\tau=0.2\)</span>.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
|
||
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">math</span>
|
||
<span class="kn">from</span> <span class="nn">scipy</span> <span class="kn">import</span> <span class="n">signal</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="c1"># number of points </span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">500</span>
|
||
<span class="c1"># start and final times </span>
|
||
<span class="n">t0</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="n">tn</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
<span class="c1"># Period </span>
|
||
<span class="n">t</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="n">t0</span><span class="p">,</span> <span class="n">tn</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">endpoint</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
<span class="n">SqrSignal</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">SqrSignal</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">+</span><span class="n">signal</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">pi</span><span class="o">*</span><span class="mi">5</span><span class="o">*</span><span class="n">t</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">SqrSignal</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylim</span><span class="p">(</span><span class="o">-</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">2.5</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<img alt="_images/chapter12_35_0.png" src="_images/chapter12_35_0.png" />
|
||
</div>
|
||
</div>
|
||
<p>For the sinusoidal example the
|
||
period is <span class="math notranslate nohighlight">\(\tau=2\pi/\omega\)</span>. However, higher harmonics can also
|
||
satisfy the periodicity requirement. In general, any force that
|
||
satisfies the periodicity requirement can be expressed as a sum over
|
||
harmonics,</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto2"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
F(t)=\frac{f_0}{2}+\sum_{n>0} f_n\cos(2n\pi t/\tau)+g_n\sin(2n\pi t/\tau).
|
||
\label{_auto2} \tag{2}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>We can write down the answer for
|
||
<span class="math notranslate nohighlight">\(x_{pn}(t)\)</span>, by substituting <span class="math notranslate nohighlight">\(f_n/m\)</span> or <span class="math notranslate nohighlight">\(g_n/m\)</span> for <span class="math notranslate nohighlight">\(F_0/m\)</span>. By
|
||
writing each factor <span class="math notranslate nohighlight">\(2n\pi t/\tau\)</span> as <span class="math notranslate nohighlight">\(n\omega t\)</span>, with <span class="math notranslate nohighlight">\(\omega\equiv
|
||
2\pi/\tau\)</span>,</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:fourierdef1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\label{eq:fourierdef1} \tag{3}
|
||
F(t)=\frac{f_0}{2}+\sum_{n>0}f_n\cos(n\omega t)+g_n\sin(n\omega t).
|
||
\end{equation}
|
||
\]</div>
|
||
<p>The solutions for <span class="math notranslate nohighlight">\(x(t)\)</span> then come from replacing <span class="math notranslate nohighlight">\(\omega\)</span> with
|
||
<span class="math notranslate nohighlight">\(n\omega\)</span> for each term in the particular solution,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{eqnarray}
|
||
x_p(t)&=&\frac{f_0}{2k}+\sum_{n>0} \alpha_n\cos(n\omega t-\delta_n)+\beta_n\sin(n\omega t-\delta_n),\\
|
||
\nonumber
|
||
\alpha_n&=&\frac{f_n/m}{\sqrt{((n\omega)^2-\omega_0^2)+4\beta^2n^2\omega^2}},\\
|
||
\nonumber
|
||
\beta_n&=&\frac{g_n/m}{\sqrt{((n\omega)^2-\omega_0^2)+4\beta^2n^2\omega^2}},\\
|
||
\nonumber
|
||
\delta_n&=&\tan^{-1}\left(\frac{2\beta n\omega}{\omega_0^2-n^2\omega^2}\right).
|
||
\end{eqnarray}
|
||
\end{split}\]</div>
|
||
<p>Because the forces have been applied for a long time, any non-zero
|
||
damping eliminates the homogenous parts of the solution, so one need
|
||
only consider the particular solution for each <span class="math notranslate nohighlight">\(n\)</span>.</p>
|
||
<p>The problem is considered solved if one can find expressions for the
|
||
coefficients <span class="math notranslate nohighlight">\(f_n\)</span> and <span class="math notranslate nohighlight">\(g_n\)</span>, even though the solutions are expressed
|
||
as an infinite sum. The coefficients can be extracted from the
|
||
function <span class="math notranslate nohighlight">\(F(t)\)</span> by</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:fourierdef2"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{eqnarray}
|
||
\label{eq:fourierdef2} \tag{4}
|
||
f_n&=&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~F(t)\cos(2n\pi t/\tau),\\
|
||
\nonumber
|
||
g_n&=&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~F(t)\sin(2n\pi t/\tau).
|
||
\end{eqnarray}
|
||
\end{split}\]</div>
|
||
<p>To check the consistency of these expressions and to verify
|
||
Eq. (<a class="reference external" href="#eq:fourierdef2">4</a>), one can insert the expansion of <span class="math notranslate nohighlight">\(F(t)\)</span> in
|
||
Eq. (<a class="reference external" href="#eq:fourierdef1">3</a>) into the expression for the coefficients in
|
||
Eq. (<a class="reference external" href="#eq:fourierdef2">4</a>) and see whether</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{eqnarray}
|
||
f_n&=?&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~\left\{
|
||
\frac{f_0}{2}+\sum_{m>0}f_m\cos(m\omega t)+g_m\sin(m\omega t)
|
||
\right\}\cos(n\omega t).
|
||
\end{eqnarray}
|
||
\]</div>
|
||
<p>Immediately, one can throw away all the terms with <span class="math notranslate nohighlight">\(g_m\)</span> because they
|
||
convolute an even and an odd function. The term with <span class="math notranslate nohighlight">\(f_0/2\)</span>
|
||
disappears because <span class="math notranslate nohighlight">\(\cos(n\omega t)\)</span> is equally positive and negative
|
||
over the interval and will integrate to zero. For all the terms
|
||
<span class="math notranslate nohighlight">\(f_m\cos(m\omega t)\)</span> appearing in the sum, one can use angle addition
|
||
formulas to see that <span class="math notranslate nohighlight">\(\cos(m\omega t)\cos(n\omega
|
||
t)=(1/2)(\cos[(m+n)\omega t]+\cos[(m-n)\omega t]\)</span>. This will integrate
|
||
to zero unless <span class="math notranslate nohighlight">\(m=n\)</span>. In that case the <span class="math notranslate nohighlight">\(m=n\)</span> term gives</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto3"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\int_{-\tau/2}^{\tau/2}dt~\cos^2(m\omega t)=\frac{\tau}{2},
|
||
\label{_auto3} \tag{5}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{eqnarray}
|
||
f_n&=?&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~f_n/2\\
|
||
\nonumber
|
||
&=&f_n~\checkmark.
|
||
\end{eqnarray}
|
||
\end{split}\]</div>
|
||
<p>The same method can be used to check for the consistency of <span class="math notranslate nohighlight">\(g_n\)</span>.</p>
|
||
<p>The code here uses the Fourier series applied to a
|
||
square wave signal. The code here
|
||
visualizes the various approximations given by Fourier series compared
|
||
with a square wave with period <span class="math notranslate nohighlight">\(T=0.2\)</span> (dimensionless time), width <span class="math notranslate nohighlight">\(0.1\)</span> and max value of the force <span class="math notranslate nohighlight">\(F=2\)</span>. We
|
||
see that when we increase the number of components in the Fourier
|
||
series, the Fourier series approximation gets closer and closer to the
|
||
square wave signal.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">math</span>
|
||
<span class="kn">from</span> <span class="nn">scipy</span> <span class="kn">import</span> <span class="n">signal</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="c1"># number of points </span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">500</span>
|
||
<span class="c1"># start and final times </span>
|
||
<span class="n">t0</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||
<span class="n">tn</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
<span class="c1"># Period </span>
|
||
<span class="n">T</span> <span class="o">=</span><span class="mf">0.2</span>
|
||
<span class="c1"># Max value of square signal </span>
|
||
<span class="n">Fmax</span><span class="o">=</span> <span class="mf">2.0</span>
|
||
<span class="c1"># Width of signal </span>
|
||
<span class="n">Width</span> <span class="o">=</span> <span class="mf">0.1</span>
|
||
<span class="n">t</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="n">t0</span><span class="p">,</span> <span class="n">tn</span><span class="p">,</span> <span class="n">n</span><span class="p">,</span> <span class="n">endpoint</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
<span class="n">SqrSignal</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">FourierSeriesSignal</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">SqrSignal</span> <span class="o">=</span> <span class="mf">1.0</span><span class="o">+</span><span class="n">signal</span><span class="o">.</span><span class="n">square</span><span class="p">(</span><span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">pi</span><span class="o">*</span><span class="mi">5</span><span class="o">*</span><span class="n">t</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">pi</span><span class="o">*</span><span class="n">Width</span><span class="o">/</span><span class="n">T</span><span class="p">)</span>
|
||
<span class="n">a0</span> <span class="o">=</span> <span class="n">Fmax</span><span class="o">*</span><span class="n">Width</span><span class="o">/</span><span class="n">T</span>
|
||
<span class="n">FourierSeriesSignal</span> <span class="o">=</span> <span class="n">a0</span>
|
||
<span class="n">Factor</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">*</span><span class="n">Fmax</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">pi</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="mi">500</span><span class="p">):</span>
|
||
<span class="n">FourierSeriesSignal</span> <span class="o">+=</span> <span class="n">Factor</span><span class="o">/</span><span class="p">(</span><span class="n">i</span><span class="p">)</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">sin</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">pi</span><span class="o">*</span><span class="n">i</span><span class="o">*</span><span class="n">Width</span><span class="o">/</span><span class="n">T</span><span class="p">)</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">i</span><span class="o">*</span><span class="n">t</span><span class="o">*</span><span class="mi">2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">pi</span><span class="o">/</span><span class="n">T</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">SqrSignal</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="n">FourierSeriesSignal</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylim</span><span class="p">(</span><span class="o">-</span><span class="mf">0.5</span><span class="p">,</span> <span class="mf">2.5</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<img alt="_images/chapter12_51_0.png" src="_images/chapter12_51_0.png" />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="two-dimensional-objects">
|
||
<h2><span class="section-number">16.4. </span>Two-dimensional Objects<a class="headerlink" href="#two-dimensional-objects" title="Permalink to this headline">¶</a></h2>
|
||
<p>We often use convolutions over more than one dimension at a time. If
|
||
we have a two-dimensional image <span class="math notranslate nohighlight">\(I\)</span> as input, we can have a <strong>filter</strong>
|
||
defined by a two-dimensional <strong>kernel</strong> <span class="math notranslate nohighlight">\(K\)</span>. This leads to an output <span class="math notranslate nohighlight">\(S\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
S_(i,j)=(I * K)(i,j) = \sum_m\sum_n I(m,n)K(i-m,j-n).
|
||
\]</div>
|
||
<p>Convolution is a commutatitave process, which means we can rewrite this equation as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
S_(i,j)=(I * K)(i,j) = \sum_m\sum_n I(i-m,j-n)K(m,n).
|
||
\]</div>
|
||
<p>Normally the latter is more straightforward to implement in a machine elarning library since there is less variation in the range of values of <span class="math notranslate nohighlight">\(m\)</span> and <span class="math notranslate nohighlight">\(n\)</span>.</p>
|
||
<p>Many deep learning libraries implement cross-correlation instead of convolution</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
S_(i,j)=(I * K)(i,j) = \sum_m\sum_n I(i+m,j-+)K(m,n).
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="more-on-dimensionalities">
|
||
<h2><span class="section-number">16.5. </span>More on Dimensionalities<a class="headerlink" href="#more-on-dimensionalities" title="Permalink to this headline">¶</a></h2>
|
||
<p>In fields like signal processing (and imaging as well), one designs
|
||
so-called filters. These filters are defined by the convolutions and
|
||
are often hand-crafted. One may specify filters for smoothing, edge
|
||
detection, frequency reshaping, and similar operations. However with
|
||
neural networks the idea is to automatically learn the filters and use
|
||
many of them in conjunction with non-linear operations (activation
|
||
functions).</p>
|
||
<p>As an example consider a neural network operating on sound sequence
|
||
data. Assume that we an input vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> of length <span class="math notranslate nohighlight">\(d=10^6\)</span>. We
|
||
construct then a neural network with onle hidden layer only with
|
||
<span class="math notranslate nohighlight">\(10^4\)</span> nodes. This means that we will have a weight matrix with
|
||
<span class="math notranslate nohighlight">\(10^4\times 10^6=10^{10}\)</span> weights to be determined, together with <span class="math notranslate nohighlight">\(10^4\)</span> biases.</p>
|
||
<p>Assume furthermore that we have an output layer which is meant to train whether the sound sequence represents a human voice (true) or something else (false).
|
||
It means that we have only one output node. But since this output node connects to <span class="math notranslate nohighlight">\(10^4\)</span> nodes in the hidden layer, there are in total <span class="math notranslate nohighlight">\(10^4\)</span> weights to be determined for the output layer, plus one bias. In total we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{NumberParameters}=10^{10}+10^4+10^4+1 \approx 10^{10},
|
||
\]</div>
|
||
<p>that is ten billion parameters to determine.</p>
|
||
</div>
|
||
<div class="section" id="further-dimensionality-remarks">
|
||
<h2><span class="section-number">16.6. </span>Further Dimensionality Remarks<a class="headerlink" href="#further-dimensionality-remarks" title="Permalink to this headline">¶</a></h2>
|
||
<p>In today’s architecture one can train such neural networks, however
|
||
this is a huge number of parameters for the task at hand. In general,
|
||
it is a very wasteful and inefficient use of dense matrices as
|
||
parameters. Just as importantly, such trained network parameters are
|
||
very specific for the type of input data on which they were trained
|
||
and the network is not likely to generalize easily to variations in
|
||
the input.</p>
|
||
<p>The main principles that justify convolutions is locality of
|
||
information and repetion of patterns within the signal. Sound samples
|
||
of the input in adjacent spots are much more likely to affect each
|
||
other than those that are very far away. Similarly, sounds are
|
||
repeated in multiple times in the signal. While slightly simplistic,
|
||
reasoning about such a sound example demonstrates this. The same
|
||
principles then apply to images and other similar data.</p>
|
||
</div>
|
||
<div class="section" id="cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras">
|
||
<h2><span class="section-number">16.7. </span>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras<a class="headerlink" href="#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" title="Permalink to this headline">¶</a></h2>
|
||
<p>As discussed above, CNNs are neural networks built from the assumption that the inputs
|
||
to the network are 2D images. This is important because the number of features or pixels in images
|
||
grows very fast with the image size, and an enormous number of weights and biases are needed in order to build an accurate network.</p>
|
||
<p>As before, we still have our input, a hidden layer and an output. What’s novel about convolutional networks
|
||
are the <strong>convolutional</strong> and <strong>pooling</strong> layers stacked in pairs between the input and the hidden layer.
|
||
In addition, the data is no longer represented as a 2D feature matrix, instead each input is a number of 2D
|
||
matrices, typically 1 for each color dimension (Red, Green, Blue).</p>
|
||
<p>It means that to represent the entire
|
||
dataset of images, we require a 4D matrix or <strong>tensor</strong>. This tensor has the dimensions:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(n_{inputs},\, n_{pixels, width},\, n_{pixels, height},\, depth) .
|
||
\]</div>
|
||
<div class="section" id="the-mnist-dataset-again">
|
||
<h3><span class="section-number">16.7.1. </span>The MNIST dataset again<a class="headerlink" href="#the-mnist-dataset-again" title="Permalink to this headline">¶</a></h3>
|
||
<p>The MNIST dataset consists of grayscale images with a pixel size of
|
||
<span class="math notranslate nohighlight">\(28\times 28\)</span>, meaning we require <span class="math notranslate nohighlight">\(28 \times 28 = 724\)</span> weights to each
|
||
neuron in the first hidden layer.</p>
|
||
<p>If we were to analyze images of size <span class="math notranslate nohighlight">\(128\times 128\)</span> we would require
|
||
<span class="math notranslate nohighlight">\(128 \times 128 = 16384\)</span> weights to each neuron. Even worse if we were
|
||
dealing with color images, as most images are, we have an image matrix
|
||
of size <span class="math notranslate nohighlight">\(128\times 128\)</span> for each color dimension (Red, Green, Blue),
|
||
meaning 3 times the number of weights <span class="math notranslate nohighlight">\(= 49152\)</span> are required for every
|
||
single neuron in the first hidden layer.</p>
|
||
<p>Images typically have strong local correlations, meaning that a small
|
||
part of the image varies little from its neighboring regions. If for
|
||
example we have an image of a blue car, we can roughly assume that a
|
||
small blue part of the image is surrounded by other blue regions.</p>
|
||
<p>Therefore, instead of connecting every single pixel to a neuron in the
|
||
first hidden layer, as we have previously done with deep neural
|
||
networks, we can instead connect each neuron to a small part of the
|
||
image (in all 3 RGB depth dimensions). The size of each small area is
|
||
fixed, and known as a <a class="reference external" href="https://en.wikipedia.org/wiki/Receptive_field">receptive</a>.</p>
|
||
<p>The layers of a convolutional neural network arrange neurons in 3D: width, height and depth.<br />
|
||
The input image is typically a square matrix of depth 3.</p>
|
||
<p>A <strong>convolution</strong> is performed on the image which outputs
|
||
a 3D volume of neurons. The weights to the input are arranged in a number of 2D matrices, known as <strong>filters</strong>.</p>
|
||
<p>Each filter slides along the input image, taking the dot product
|
||
between each small part of the image and the filter, in all depth
|
||
dimensions. This is then passed through a non-linear function,
|
||
typically the <strong>Rectified Linear (ReLu)</strong> function, which serves as the
|
||
activation of the neurons in the first convolutional layer. This is
|
||
further passed through a <strong>pooling layer</strong>, which reduces the size of the
|
||
convolutional layer, e.g. by taking the maximum or average across some
|
||
small regions, and this serves as input to the next convolutional
|
||
layer.</p>
|
||
</div>
|
||
<div class="section" id="systematic-reduction">
|
||
<h3><span class="section-number">16.7.2. </span>Systematic reduction<a class="headerlink" href="#systematic-reduction" title="Permalink to this headline">¶</a></h3>
|
||
<p>By systematically reducing the size of the input volume, through
|
||
convolution and pooling, the network should create representations of
|
||
small parts of the input, and then from them assemble representations
|
||
of larger areas. The final pooling layer is flattened to serve as
|
||
input to a hidden layer, such that each neuron in the final pooling
|
||
layer is connected to every single neuron in the hidden layer. This
|
||
then serves as input to the output layer, e.g. a softmax output for
|
||
classification.</p>
|
||
</div>
|
||
<div class="section" id="prerequisites-collect-and-pre-process-data">
|
||
<h3><span class="section-number">16.7.3. </span>Prerequisites: Collect and pre-process data<a class="headerlink" href="#prerequisites-collect-and-pre-process-data" title="Permalink to this headline">¶</a></h3>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># import necessary packages</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">datasets</span>
|
||
|
||
|
||
<span class="c1"># ensure the same random numbers appear every time</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||
|
||
<span class="c1"># display images in notebook</span>
|
||
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s1">'figure.figsize'</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="mi">12</span><span class="p">,</span><span class="mi">12</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># download MNIST dataset</span>
|
||
<span class="n">digits</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">load_digits</span><span class="p">()</span>
|
||
|
||
<span class="c1"># define inputs and labels</span>
|
||
<span class="n">inputs</span> <span class="o">=</span> <span class="n">digits</span><span class="o">.</span><span class="n">images</span>
|
||
<span class="n">labels</span> <span class="o">=</span> <span class="n">digits</span><span class="o">.</span><span class="n">target</span>
|
||
|
||
<span class="c1"># RGB images have a depth of 3</span>
|
||
<span class="c1"># our images are grayscale so they should have a depth of 1</span>
|
||
<span class="n">inputs</span> <span class="o">=</span> <span class="n">inputs</span><span class="p">[:,:,:,</span><span class="n">np</span><span class="o">.</span><span class="n">newaxis</span><span class="p">]</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"inputs = (n_inputs, pixel_width, pixel_height, depth) = "</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">inputs</span><span class="o">.</span><span class="n">shape</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"labels = (n_inputs) = "</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">labels</span><span class="o">.</span><span class="n">shape</span><span class="p">))</span>
|
||
|
||
|
||
<span class="c1"># choose some random images to display</span>
|
||
<span class="n">n_inputs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
|
||
<span class="n">indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">arange</span><span class="p">(</span><span class="n">n_inputs</span><span class="p">)</span>
|
||
<span class="n">random_indices</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="n">indices</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">image</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">digits</span><span class="o">.</span><span class="n">images</span><span class="p">[</span><span class="n">random_indices</span><span class="p">]):</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s1">'off'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">cmap</span><span class="o">=</span><span class="n">plt</span><span class="o">.</span><span class="n">cm</span><span class="o">.</span><span class="n">gray_r</span><span class="p">,</span> <span class="n">interpolation</span><span class="o">=</span><span class="s1">'nearest'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="s2">"Label: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">digits</span><span class="o">.</span><span class="n">target</span><span class="p">[</span><span class="n">random_indices</span><span class="p">[</span><span class="n">i</span><span class="p">]])</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>inputs = (n_inputs, pixel_width, pixel_height, depth) = (1797, 8, 8, 1)
|
||
labels = (n_inputs) = (1797,)
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/chapter12_67_1.png" src="_images/chapter12_67_1.png" />
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">datasets</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">models</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.layers</span> <span class="kn">import</span> <span class="n">Input</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.models</span> <span class="kn">import</span> <span class="n">Sequential</span> <span class="c1">#This allows appending layers to existing models</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.layers</span> <span class="kn">import</span> <span class="n">Dense</span> <span class="c1">#This allows defining the characteristics of a particular layer</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">optimizers</span> <span class="c1">#This allows using whichever optimiser we want (sgd,adam,RMSprop)</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">regularizers</span> <span class="c1">#This allows using whichever regularizer we want (l1,l2,l1_l2)</span>
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras.utils</span> <span class="kn">import</span> <span class="n">to_categorical</span> <span class="c1">#This allows using categorical cross entropy as the cost function</span>
|
||
<span class="c1">#from tensorflow.keras import Conv2D</span>
|
||
<span class="c1">#from tensorflow.keras import MaxPooling2D</span>
|
||
<span class="c1">#from tensorflow.keras import Flatten</span>
|
||
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
|
||
<span class="c1"># representation of labels</span>
|
||
<span class="n">labels</span> <span class="o">=</span> <span class="n">to_categorical</span><span class="p">(</span><span class="n">labels</span><span class="p">)</span>
|
||
|
||
<span class="c1"># split into train and test data</span>
|
||
<span class="c1"># one-liner from scikit-learn library</span>
|
||
<span class="n">train_size</span> <span class="o">=</span> <span class="mf">0.8</span>
|
||
<span class="n">test_size</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">train_size</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">,</span> <span class="n">Y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">train_size</span><span class="o">=</span><span class="n">train_size</span><span class="p">,</span>
|
||
<span class="n">test_size</span><span class="o">=</span><span class="n">test_size</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">create_convolutional_neural_network_keras</span><span class="p">(</span><span class="n">input_shape</span><span class="p">,</span> <span class="n">receptive_field</span><span class="p">,</span>
|
||
<span class="n">n_filters</span><span class="p">,</span> <span class="n">n_neurons_connected</span><span class="p">,</span> <span class="n">n_categories</span><span class="p">,</span>
|
||
<span class="n">eta</span><span class="p">,</span> <span class="n">lmbd</span><span class="p">):</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">Sequential</span><span class="p">()</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2D</span><span class="p">(</span><span class="n">n_filters</span><span class="p">,</span> <span class="p">(</span><span class="n">receptive_field</span><span class="p">,</span> <span class="n">receptive_field</span><span class="p">),</span> <span class="n">input_shape</span><span class="o">=</span><span class="n">input_shape</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="s1">'same'</span><span class="p">,</span>
|
||
<span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">,</span> <span class="n">kernel_regularizer</span><span class="o">=</span><span class="n">regularizers</span><span class="o">.</span><span class="n">l2</span><span class="p">(</span><span class="n">lmbd</span><span class="p">)))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">MaxPooling2D</span><span class="p">(</span><span class="n">pool_size</span><span class="o">=</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Flatten</span><span class="p">())</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">n_neurons_connected</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">,</span> <span class="n">kernel_regularizer</span><span class="o">=</span><span class="n">regularizers</span><span class="o">.</span><span class="n">l2</span><span class="p">(</span><span class="n">lmbd</span><span class="p">)))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="n">n_categories</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'softmax'</span><span class="p">,</span> <span class="n">kernel_regularizer</span><span class="o">=</span><span class="n">regularizers</span><span class="o">.</span><span class="n">l2</span><span class="p">(</span><span class="n">lmbd</span><span class="p">)))</span>
|
||
|
||
<span class="n">sgd</span> <span class="o">=</span> <span class="n">optimizers</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">lr</span><span class="o">=</span><span class="n">eta</span><span class="p">)</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">loss</span><span class="o">=</span><span class="s1">'categorical_crossentropy'</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">=</span><span class="n">sgd</span><span class="p">,</span> <span class="n">metrics</span><span class="o">=</span><span class="p">[</span><span class="s1">'accuracy'</span><span class="p">])</span>
|
||
|
||
<span class="k">return</span> <span class="n">model</span>
|
||
|
||
<span class="n">epochs</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">batch_size</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">input_shape</span> <span class="o">=</span> <span class="n">X_train</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">:</span><span class="mi">4</span><span class="p">]</span>
|
||
<span class="n">receptive_field</span> <span class="o">=</span> <span class="mi">3</span>
|
||
<span class="n">n_filters</span> <span class="o">=</span> <span class="mi">10</span>
|
||
<span class="n">n_neurons_connected</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">n_categories</span> <span class="o">=</span> <span class="mi">10</span>
|
||
|
||
<span class="n">eta_vals</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">logspace</span><span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">7</span><span class="p">)</span>
|
||
<span class="n">lmbd_vals</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">logspace</span><span class="p">(</span><span class="o">-</span><span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">7</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">CNN_keras</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">eta_vals</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">lmbd_vals</span><span class="p">)),</span> <span class="n">dtype</span><span class="o">=</span><span class="nb">object</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">eta</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">eta_vals</span><span class="p">):</span>
|
||
<span class="k">for</span> <span class="n">j</span><span class="p">,</span> <span class="n">lmbd</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">lmbd_vals</span><span class="p">):</span>
|
||
<span class="n">CNN</span> <span class="o">=</span> <span class="n">create_convolutional_neural_network_keras</span><span class="p">(</span><span class="n">input_shape</span><span class="p">,</span> <span class="n">receptive_field</span><span class="p">,</span>
|
||
<span class="n">n_filters</span><span class="p">,</span> <span class="n">n_neurons_connected</span><span class="p">,</span> <span class="n">n_categories</span><span class="p">,</span>
|
||
<span class="n">eta</span><span class="p">,</span> <span class="n">lmbd</span><span class="p">)</span>
|
||
<span class="n">CNN</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="n">epochs</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">scores</span> <span class="o">=</span> <span class="n">CNN</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">Y_test</span><span class="p">)</span>
|
||
|
||
<span class="n">CNN_keras</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">]</span> <span class="o">=</span> <span class="n">CNN</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Learning rate = "</span><span class="p">,</span> <span class="n">eta</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Lambda = "</span><span class="p">,</span> <span class="n">lmbd</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Test accuracy: </span><span class="si">%.3f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">scores</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Metal device set to: Apple M1
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/keras/optimizer_v2/gradient_descent.py:102: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
|
||
super(SGD, self).__init__(name, **kwargs)
|
||
2023-11-21 06:09:14.627153: W tensorflow/core/platform/profile_utils/cpu_utils.cc:128] Failed to get CPU frequency: 0 Hz
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 1/12 [=>............................] - ETA: 1s - loss: 2.5762 - accuracy: 0.2188
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>
|
||
7/12 [================>.............] - ETA: 0s - loss: 2.7312 - accuracy: 0.2277
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>
|
||
12/12 [==============================] - ETA: 0s - loss: 2.7089 - accuracy: 0.2389
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>
|
||
12/12 [==============================] - 1s 35ms/step - loss: 2.7089 - accuracy: 0.2389
|
||
</pre></div>
|
||
</div>
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Learning rate = 1e-05
|
||
Lambda = 1e-05
|
||
Test accuracy: 0.239
|
||
</pre></div>
|
||
</div>
|
||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
|
||
<span class="ne">KeyboardInterrupt</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||
<span class="nn">Input In [6],</span> in <span class="ni"><cell line: 3></span><span class="nt">()</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">4</span> <span class="k">for</span> <span class="n">j</span><span class="p">,</span> <span class="n">lmbd</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">lmbd_vals</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">5</span> <span class="n">CNN</span> <span class="o">=</span> <span class="n">create_convolutional_neural_network_keras</span><span class="p">(</span><span class="n">input_shape</span><span class="p">,</span> <span class="n">receptive_field</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">6</span> <span class="n">n_filters</span><span class="p">,</span> <span class="n">n_neurons_connected</span><span class="p">,</span> <span class="n">n_categories</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">7</span> <span class="n">eta</span><span class="p">,</span> <span class="n">lmbd</span><span class="p">)</span>
|
||
<span class="ne">----> </span><span class="mi">8</span> <span class="n">CNN</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">,</span> <span class="n">epochs</span><span class="o">=</span><span class="n">epochs</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">9</span> <span class="n">scores</span> <span class="o">=</span> <span class="n">CNN</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">Y_test</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">11</span> <span class="n">CNN_keras</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">]</span> <span class="o">=</span> <span class="n">CNN</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/keras/utils/traceback_utils.py:64,</span> in <span class="ni">filter_traceback.<locals>.error_handler</span><span class="nt">(*args, **kwargs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">62</span> <span class="n">filtered_tb</span> <span class="o">=</span> <span class="kc">None</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">63</span> <span class="k">try</span><span class="p">:</span>
|
||
<span class="ne">---> </span><span class="mi">64</span> <span class="k">return</span> <span class="n">fn</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">65</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span> <span class="c1"># pylint: disable=broad-except</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">66</span> <span class="n">filtered_tb</span> <span class="o">=</span> <span class="n">_process_traceback_frames</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">__traceback__</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/keras/engine/training.py:1384,</span> in <span class="ni">Model.fit</span><span class="nt">(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1377</span> <span class="k">with</span> <span class="n">tf</span><span class="o">.</span><span class="n">profiler</span><span class="o">.</span><span class="n">experimental</span><span class="o">.</span><span class="n">Trace</span><span class="p">(</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1378</span> <span class="s1">'train'</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1379</span> <span class="n">epoch_num</span><span class="o">=</span><span class="n">epoch</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1380</span> <span class="n">step_num</span><span class="o">=</span><span class="n">step</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1381</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1382</span> <span class="n">_r</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1383</span> <span class="n">callbacks</span><span class="o">.</span><span class="n">on_train_batch_begin</span><span class="p">(</span><span class="n">step</span><span class="p">)</span>
|
||
<span class="ne">-> </span><span class="mi">1384</span> <span class="n">tmp_logs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_function</span><span class="p">(</span><span class="n">iterator</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1385</span> <span class="k">if</span> <span class="n">data_handler</span><span class="o">.</span><span class="n">should_sync</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1386</span> <span class="n">context</span><span class="o">.</span><span class="n">async_wait</span><span class="p">()</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py:150,</span> in <span class="ni">filter_traceback.<locals>.error_handler</span><span class="nt">(*args, **kwargs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">148</span> <span class="n">filtered_tb</span> <span class="o">=</span> <span class="kc">None</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">149</span> <span class="k">try</span><span class="p">:</span>
|
||
<span class="ne">--> </span><span class="mi">150</span> <span class="k">return</span> <span class="n">fn</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">151</span> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">152</span> <span class="n">filtered_tb</span> <span class="o">=</span> <span class="n">_process_traceback_frames</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">__traceback__</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py:915,</span> in <span class="ni">Function.__call__</span><span class="nt">(self, *args, **kwds)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">912</span> <span class="n">compiler</span> <span class="o">=</span> <span class="s2">"xla"</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_jit_compile</span> <span class="k">else</span> <span class="s2">"nonXla"</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">914</span> <span class="k">with</span> <span class="n">OptionalXlaContext</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_jit_compile</span><span class="p">):</span>
|
||
<span class="ne">--> </span><span class="mi">915</span> <span class="n">result</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_call</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwds</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">917</span> <span class="n">new_tracing_count</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">experimental_get_tracing_count</span><span class="p">()</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">918</span> <span class="n">without_tracing</span> <span class="o">=</span> <span class="p">(</span><span class="n">tracing_count</span> <span class="o">==</span> <span class="n">new_tracing_count</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/eager/def_function.py:947,</span> in <span class="ni">Function._call</span><span class="nt">(self, *args, **kwds)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">944</span> <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">945</span> <span class="c1"># In this case we have created variables on the first call, so we run the</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">946</span> <span class="c1"># defunned version which is guaranteed to never create variables.</span>
|
||
<span class="ne">--> </span><span class="mi">947</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_stateless_fn</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwds</span><span class="p">)</span> <span class="c1"># pylint: disable=not-callable</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">948</span> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_stateful_fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">949</span> <span class="c1"># Release the lock early so that multiple threads can perform the call</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">950</span> <span class="c1"># in parallel.</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">951</span> <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/eager/function.py:2956,</span> in <span class="ni">Function.__call__</span><span class="nt">(self, *args, **kwargs)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">2953</span> <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">2954</span> <span class="p">(</span><span class="n">graph_function</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">2955</span> <span class="n">filtered_flat_args</span><span class="p">)</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_maybe_define_function</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)</span>
|
||
<span class="ne">-> </span><span class="mi">2956</span> <span class="k">return</span> <span class="n">graph_function</span><span class="o">.</span><span class="n">_call_flat</span><span class="p">(</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">2957</span> <span class="n">filtered_flat_args</span><span class="p">,</span> <span class="n">captured_inputs</span><span class="o">=</span><span class="n">graph_function</span><span class="o">.</span><span class="n">captured_inputs</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/eager/function.py:1853,</span> in <span class="ni">ConcreteFunction._call_flat</span><span class="nt">(self, args, captured_inputs, cancellation_manager)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1849</span> <span class="n">possible_gradient_type</span> <span class="o">=</span> <span class="n">gradients_util</span><span class="o">.</span><span class="n">PossibleTapeGradientTypes</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1850</span> <span class="k">if</span> <span class="p">(</span><span class="n">possible_gradient_type</span> <span class="o">==</span> <span class="n">gradients_util</span><span class="o">.</span><span class="n">POSSIBLE_GRADIENT_TYPES_NONE</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1851</span> <span class="ow">and</span> <span class="n">executing_eagerly</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1852</span> <span class="c1"># No tape is watching; skip to running the function.</span>
|
||
<span class="ne">-> </span><span class="mi">1853</span> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_call_outputs</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_inference_function</span><span class="o">.</span><span class="n">call</span><span class="p">(</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1854</span> <span class="n">ctx</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">cancellation_manager</span><span class="o">=</span><span class="n">cancellation_manager</span><span class="p">))</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1855</span> <span class="n">forward_backward</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_select_forward_and_backward_functions</span><span class="p">(</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1856</span> <span class="n">args</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1857</span> <span class="n">possible_gradient_type</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1858</span> <span class="n">executing_eagerly</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">1859</span> <span class="n">forward_function</span><span class="p">,</span> <span class="n">args_with_tangents</span> <span class="o">=</span> <span class="n">forward_backward</span><span class="o">.</span><span class="n">forward</span><span class="p">()</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/eager/function.py:499,</span> in <span class="ni">_EagerDefinedFunction.call</span><span class="nt">(self, ctx, args, cancellation_manager)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">497</span> <span class="k">with</span> <span class="n">_InterpolateFunctionError</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">498</span> <span class="k">if</span> <span class="n">cancellation_manager</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||
<span class="ne">--> </span><span class="mi">499</span> <span class="n">outputs</span> <span class="o">=</span> <span class="n">execute</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">500</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">signature</span><span class="o">.</span><span class="n">name</span><span class="p">),</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">501</span> <span class="n">num_outputs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_outputs</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">502</span> <span class="n">inputs</span><span class="o">=</span><span class="n">args</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">503</span> <span class="n">attrs</span><span class="o">=</span><span class="n">attrs</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">504</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">505</span> <span class="k">else</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">506</span> <span class="n">outputs</span> <span class="o">=</span> <span class="n">execute</span><span class="o">.</span><span class="n">execute_with_cancellation</span><span class="p">(</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">507</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">signature</span><span class="o">.</span><span class="n">name</span><span class="p">),</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">508</span> <span class="n">num_outputs</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_num_outputs</span><span class="p">,</span>
|
||
<span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">511</span> <span class="n">ctx</span><span class="o">=</span><span class="n">ctx</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">512</span> <span class="n">cancellation_manager</span><span class="o">=</span><span class="n">cancellation_manager</span><span class="p">)</span>
|
||
|
||
<span class="nn">File ~/miniforge3/envs/myenv/lib/python3.9/site-packages/tensorflow/python/eager/execute.py:54,</span> in <span class="ni">quick_execute</span><span class="nt">(op_name, num_outputs, inputs, attrs, ctx, name)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">52</span> <span class="k">try</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">53</span> <span class="n">ctx</span><span class="o">.</span><span class="n">ensure_initialized</span><span class="p">()</span>
|
||
<span class="ne">---> </span><span class="mi">54</span> <span class="n">tensors</span> <span class="o">=</span> <span class="n">pywrap_tfe</span><span class="o">.</span><span class="n">TFE_Py_Execute</span><span class="p">(</span><span class="n">ctx</span><span class="o">.</span><span class="n">_handle</span><span class="p">,</span> <span class="n">device_name</span><span class="p">,</span> <span class="n">op_name</span><span class="p">,</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">55</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">attrs</span><span class="p">,</span> <span class="n">num_outputs</span><span class="p">)</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">56</span> <span class="k">except</span> <span class="n">core</span><span class="o">.</span><span class="n">_NotOkStatusException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||
<span class="g g-Whitespace"> </span><span class="mi">57</span> <span class="k">if</span> <span class="n">name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||
|
||
<span class="ne">KeyboardInterrupt</span>:
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># visual representation of grid search</span>
|
||
<span class="c1"># uses seaborn heatmap, could probably do this in matplotlib</span>
|
||
<span class="kn">import</span> <span class="nn">seaborn</span> <span class="k">as</span> <span class="nn">sns</span>
|
||
|
||
<span class="n">sns</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
||
|
||
<span class="n">train_accuracy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">eta_vals</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">lmbd_vals</span><span class="p">)))</span>
|
||
<span class="n">test_accuracy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">eta_vals</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">lmbd_vals</span><span class="p">)))</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">eta_vals</span><span class="p">)):</span>
|
||
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">lmbd_vals</span><span class="p">)):</span>
|
||
<span class="n">CNN</span> <span class="o">=</span> <span class="n">CNN_keras</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">]</span>
|
||
|
||
<span class="n">train_accuracy</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">]</span> <span class="o">=</span> <span class="n">CNN</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
|
||
<span class="n">test_accuracy</span><span class="p">[</span><span class="n">i</span><span class="p">][</span><span class="n">j</span><span class="p">]</span> <span class="o">=</span> <span class="n">CNN</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">Y_test</span><span class="p">)[</span><span class="mi">1</span><span class="p">]</span>
|
||
|
||
|
||
<span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="n">figsize</span> <span class="o">=</span> <span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||
<span class="n">sns</span><span class="o">.</span><span class="n">heatmap</span><span class="p">(</span><span class="n">train_accuracy</span><span class="p">,</span> <span class="n">annot</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="n">ax</span><span class="p">,</span> <span class="n">cmap</span><span class="o">=</span><span class="s2">"viridis"</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Training Accuracy"</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"$\eta$"</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"$\lambda$"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
|
||
<span class="n">fig</span><span class="p">,</span> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">subplots</span><span class="p">(</span><span class="n">figsize</span> <span class="o">=</span> <span class="p">(</span><span class="mi">10</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||
<span class="n">sns</span><span class="o">.</span><span class="n">heatmap</span><span class="p">(</span><span class="n">test_accuracy</span><span class="p">,</span> <span class="n">annot</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">ax</span><span class="o">=</span><span class="n">ax</span><span class="p">,</span> <span class="n">cmap</span><span class="o">=</span><span class="s2">"viridis"</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Test Accuracy"</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"$\eta$"</span><span class="p">)</span>
|
||
<span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"$\lambda$"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="the-cifar01-data-set">
|
||
<h2><span class="section-number">16.8. </span>The CIFAR01 data set<a class="headerlink" href="#the-cifar01-data-set" title="Permalink to this headline">¶</a></h2>
|
||
<p>The CIFAR10 dataset contains 60,000 color images in 10 classes, with
|
||
6,000 images in each class. The dataset is divided into 50,000
|
||
training images and 10,000 testing images. The classes are mutually
|
||
exclusive and there is no overlap between them.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tensorflow</span> <span class="k">as</span> <span class="nn">tf</span>
|
||
|
||
<span class="kn">from</span> <span class="nn">tensorflow.keras</span> <span class="kn">import</span> <span class="n">datasets</span><span class="p">,</span> <span class="n">layers</span><span class="p">,</span> <span class="n">models</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="c1"># We import the data set</span>
|
||
<span class="p">(</span><span class="n">train_images</span><span class="p">,</span> <span class="n">train_labels</span><span class="p">),</span> <span class="p">(</span><span class="n">test_images</span><span class="p">,</span> <span class="n">test_labels</span><span class="p">)</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">cifar10</span><span class="o">.</span><span class="n">load_data</span><span class="p">()</span>
|
||
|
||
<span class="c1"># Normalize pixel values to be between 0 and 1 by dividing by 255. </span>
|
||
<span class="n">train_images</span><span class="p">,</span> <span class="n">test_images</span> <span class="o">=</span> <span class="n">train_images</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span> <span class="n">test_images</span> <span class="o">/</span> <span class="mf">255.0</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>To verify that the dataset looks correct, let’s plot the first 25 images from the training set and display the class name below each image.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
|
||
'dog', 'frog', 'horse', 'ship', 'truck']
|
||
|
||
plt.figure(figsize=(10,10))
|
||
for i in range(25):
|
||
plt.subplot(5,5,i+1)
|
||
plt.xticks([])
|
||
plt.yticks([])
|
||
plt.grid(False)
|
||
plt.imshow(train_images[i], cmap=plt.cm.binary)
|
||
# The CIFAR labels happen to be arrays,
|
||
# which is why you need the extra index
|
||
plt.xlabel(class_names[train_labels[i][0]])
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The six lines of code below define the convolutional base using a common pattern: a stack of Conv2D and MaxPooling2D layers.</p>
|
||
<p>As input, a CNN takes tensors of shape (image_height, image_width, color_channels), ignoring the batch size. If you are new to these dimensions, color_channels refers to (R,G,B). In this example, you will configure our CNN to process inputs of shape (32, 32, 3), which is the format of CIFAR images. You can do this by passing the argument input_shape to our first layer.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">Sequential</span><span class="p">()</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2D</span><span class="p">(</span><span class="mi">32</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">,</span> <span class="n">input_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">3</span><span class="p">)))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">MaxPooling2D</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2D</span><span class="p">(</span><span class="mi">64</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">MaxPooling2D</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Conv2D</span><span class="p">(</span><span class="mi">64</span><span class="p">,</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">),</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">))</span>
|
||
|
||
<span class="c1"># Let's display the architecture of our model so far.</span>
|
||
|
||
<span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>You can see that the output of every Conv2D and MaxPooling2D layer is a 3D tensor of shape (height, width, channels). The width and height dimensions tend to shrink as you go deeper in the network. The number of output channels for each Conv2D layer is controlled by the first argument (e.g., 32 or 64). Typically, as the width and height shrink, you can afford (computationally) to add more output channels in each Conv2D layer.</p>
|
||
<p>To complete our model, you will feed the last output tensor from the
|
||
convolutional base (of shape (4, 4, 64)) into one or more Dense layers
|
||
to perform classification. Dense layers take vectors as input (which
|
||
are 1D), while the current output is a 3D tensor. First, you will
|
||
flatten (or unroll) the 3D output to 1D, then add one or more Dense
|
||
layers on top. CIFAR has 10 output classes, so you use a final Dense
|
||
layer with 10 outputs and a softmax activation.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Flatten</span><span class="p">())</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="mi">64</span><span class="p">,</span> <span class="n">activation</span><span class="o">=</span><span class="s1">'relu'</span><span class="p">))</span>
|
||
<span class="n">model</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">layers</span><span class="o">.</span><span class="n">Dense</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
|
||
<span class="n">Here</span><span class="s1">'s the complete architecture of our model.</span>
|
||
|
||
<span class="n">model</span><span class="o">.</span><span class="n">summary</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>As you can see, our (4, 4, 64) outputs were flattened into vectors of shape (1024) before going through two Dense layers.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span>model.compile(optimizer='adam',
|
||
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
|
||
metrics=['accuracy'])
|
||
|
||
history = model.fit(train_images, train_labels, epochs=10,
|
||
validation_data=(test_images, test_labels))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">history</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="s1">'accuracy'</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="s1">'accuracy'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">history</span><span class="o">.</span><span class="n">history</span><span class="p">[</span><span class="s1">'val_accuracy'</span><span class="p">],</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'val_accuracy'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">'Epoch'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">'Accuracy'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylim</span><span class="p">([</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="s1">'lower right'</span><span class="p">)</span>
|
||
|
||
<span class="n">test_loss</span><span class="p">,</span> <span class="n">test_acc</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span><span class="n">test_images</span><span class="p">,</span> <span class="n">test_labels</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">test_acc</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
kernelName: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Previous / next buttons -->
|
||
<div class='prev-next-area'>
|
||
<a class='left-prev' id="prev-link" href="chapter11.html" title="previous page">
|
||
<i class="fas fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title"><span class="section-number">15. </span>Solving Differential Equations with Deep Learning</p>
|
||
</div>
|
||
</a>
|
||
<a class='right-next' id="next-link" href="chapter13.html" title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title"><span class="section-number">17. </span>Recurrent neural networks: Overarching view</p>
|
||
</div>
|
||
<i class="fas fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
<footer class="footer">
|
||
<p>
|
||
|
||
By Morten Hjorth-Jensen<br/>
|
||
|
||
© Copyright 2021.<br/>
|
||
</p>
|
||
</footer>
|
||
</main>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
|
||
|
||
</body>
|
||
</html> |