Morten Hjorth-Jensen
1393 lines
73 KiB
HTML
1393 lines
73 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>16. Convolutional Neural Networks — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'chapter12';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="17. Recurrent neural networks: Overarching view" href="chapter13.html" />
|
||
<link rel="prev" title="15. Solving Differential Equations with Deep Learning" href="chapter11.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Applied Data Analysis and Machine Learning - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Course setting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Gradient descent methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek39.html">Weekly Exercises 6</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/chapter12.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Convolutional Neural Networks</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#neural-networks-vs-cnns">16.1. Neural Networks vs CNNs</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#layers-used-to-build-cnns">16.2. Layers used to build CNNs</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematics-of-cnns">16.3. Mathematics of CNNs</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#convolution-examples-polynomial-multiplication">16.3.1. Convolution Examples: Polynomial multiplication</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms">16.3.2. Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#two-dimensional-objects">16.4. Two-dimensional Objects</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-on-dimensionalities">16.5. More on Dimensionalities</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#further-dimensionality-remarks">16.6. Further Dimensionality Remarks</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras">16.7. CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#the-mnist-dataset-again">16.7.1. The MNIST dataset again</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#systematic-reduction">16.7.2. Systematic reduction</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites-collect-and-pre-process-data">16.7.3. Prerequisites: Collect and pre-process data</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-cifar01-data-set">16.8. The CIFAR01 data set</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html chapter12.do.txt --><section class="tex2jax_ignore mathjax_ignore" id="convolutional-neural-networks">
|
||
<h1><span class="section-number">16. </span>Convolutional Neural Networks<a class="headerlink" href="#convolutional-neural-networks" title="Link to this heading">#</a></h1>
|
||
<p>Convolutional neural networks (CNNs) were developed during the last
|
||
decade of the previous century, with a focus on character recognition
|
||
tasks. Nowadays, CNNs are a central element in the spectacular success
|
||
of deep learning methods. The success in for example image
|
||
classifications have made them a central tool for most machine
|
||
learning practitioners.</p>
|
||
<p>CNNs are very similar to ordinary Neural Networks.
|
||
They are made up of neurons that have learnable weights and
|
||
biases. Each neuron receives some inputs, performs a dot product and
|
||
optionally follows it with a non-linearity. The whole network still
|
||
expresses a single differentiable score function: from the raw image
|
||
pixels on one end to class scores at the other. And they still have a
|
||
loss function (for example Softmax) on the last (fully-connected) layer
|
||
and all the tips/tricks we developed for learning regular Neural
|
||
Networks still apply (back propagation, gradient descent etc etc).</p>
|
||
<p><strong>CNN architectures make the explicit assumption that
|
||
the inputs are images, which allows us to encode certain properties
|
||
into the architecture. These then make the forward function more
|
||
efficient to implement and vastly reduce the amount of parameters in
|
||
the network.</strong></p>
|
||
<p>Here we provide only a superficial overview, for the more interested, we recommend highly the course
|
||
<a class="reference external" href="https://www.uio.no/studier/emner/matnat/ifi/IN5400/index-eng.html">IN5400 – Machine Learning for Image Analysis</a>
|
||
and the slides of <a class="reference external" href="http://cs231n.github.io/convolutional-networks/">CS231</a>.</p>
|
||
<p>Another good read is the article here <a class="reference external" href="https://arxiv.org/pdf/1603.07285.pdf">https://arxiv.org/pdf/1603.07285.pdf</a>.</p>
|
||
<section id="neural-networks-vs-cnns">
|
||
<h2><span class="section-number">16.1. </span>Neural Networks vs CNNs<a class="headerlink" href="#neural-networks-vs-cnns" title="Link to this heading">#</a></h2>
|
||
<p>Neural networks are defined as <strong>affine transformations</strong>, that is
|
||
a vector is received as input and is multiplied with a matrix of so-called weights (our unknown paramters) to produce an
|
||
output (to which a bias vector is usually added before passing the result
|
||
through a nonlinear activation function). This is applicable to any type of input, be it an
|
||
image, a sound clip or an unordered collection of features: whatever their
|
||
dimensionality, their representation can always be flattened into a vector
|
||
before the transformation.</p>
|
||
<p>However, when we consider images, sound clips and many other similar kinds of data, these data have an intrinsic
|
||
structure. More formally, they share these important properties:</p>
|
||
<ul class="simple">
|
||
<li><p>They are stored as multi-dimensional arrays (think of the pixels of a figure) .</p></li>
|
||
<li><p>They feature one or more axes for which ordering matters (e.g., width and height axes for an image, time axis for a sound clip).</p></li>
|
||
<li><p>One axis, called the channel axis, is used to access different views of the data (e.g., the red, green and blue channels of a color image, or the left and right channels of a stereo audio track).</p></li>
|
||
</ul>
|
||
<p>These properties are not exploited when an affine transformation is applied; in
|
||
fact, all the axes are treated in the same way and the topological information
|
||
is not taken into account. Still, taking advantage of the implicit structure of
|
||
the data may prove very handy in solving some tasks, like computer vision and
|
||
speech recognition, and in these cases it would be best to preserve it. This is
|
||
where discrete convolutions come into play.</p>
|
||
<p>A discrete convolution is a linear transformation that preserves this notion of
|
||
ordering. It is sparse (only a few input units contribute to a given output
|
||
unit) and reuses parameters (the same weights are applied to multiple locations
|
||
in the input).</p>
|
||
<p>As an example, consider
|
||
an image of size <span class="math notranslate nohighlight">\(32\times 32\times 3\)</span> (32 wide, 32 high, 3 color channels), so a
|
||
single fully-connected neuron in a first hidden layer of a regular
|
||
Neural Network would have <span class="math notranslate nohighlight">\(32\times 32\times 3 = 3072\)</span> weights. This amount still
|
||
seems manageable, but clearly this fully-connected structure does not
|
||
scale to larger images. For example, an image of more respectable
|
||
size, say <span class="math notranslate nohighlight">\(200\times 200\times 3\)</span>, would lead to neurons that have
|
||
<span class="math notranslate nohighlight">\(200\times 200\times 3 = 120,000\)</span> weights.</p>
|
||
<p>We could have
|
||
several such neurons, and the parameters would add up quickly! Clearly,
|
||
this full connectivity is wasteful and the huge number of parameters
|
||
would quickly lead to possible overfitting.</p>
|
||
<!-- dom:FIGURE: [figslides/nn.jpeg, width=500 frac=0.6] A regular 3-layer Neural Network. -->
|
||
<!-- begin figure -->
|
||
<p><img src="figslides/nn.jpeg" width="500"><p style="font-size: 0.9em"><i>Figure 1: A regular 3-layer Neural Network.</i></p></p>
|
||
<!-- end figure -->
|
||
<p>Convolutional Neural Networks take advantage of the fact that the
|
||
input consists of images and they constrain the architecture in a more
|
||
sensible way.</p>
|
||
<p>In particular, unlike a regular Neural Network, the
|
||
layers of a CNN have neurons arranged in 3 dimensions: width,
|
||
height, depth. (Note that the word depth here refers to the third
|
||
dimension of an activation volume, not to the depth of a full Neural
|
||
Network, which can refer to the total number of layers in a network.)</p>
|
||
<p>To understand it better, the above example of an image
|
||
with an input volume of
|
||
activations has dimensions <span class="math notranslate nohighlight">\(32\times 32\times 3\)</span> (width, height,
|
||
depth respectively).</p>
|
||
<p>The neurons in a layer will
|
||
only be connected to a small region of the layer before it, instead of
|
||
all of the neurons in a fully-connected manner. Moreover, the final
|
||
output layer could for this specific image have dimensions <span class="math notranslate nohighlight">\(1\times 1 \times 10\)</span>,
|
||
because by the
|
||
end of the CNN architecture we will reduce the full image into a
|
||
single vector of class scores, arranged along the depth
|
||
dimension.</p>
|
||
<!-- dom:FIGURE: [figslides/cnn.jpeg, width=500 frac=0.6] A CNN arranges its neurons in three dimensions (width, height, depth), as visualized in one of the layers. Every layer of a CNN transforms the 3D input volume to a 3D output volume of neuron activations. In this example, the red input layer holds the image, so its width and height would be the dimensions of the image, and the depth would be 3 (Red, Green, Blue channels). -->
|
||
<!-- begin figure -->
|
||
<p><img src="figslides/cnn.jpeg" width="500"><p style="font-size: 0.9em"><i>Figure 1: A CNN arranges its neurons in three dimensions (width, height, depth), as visualized in one of the layers. Every layer of a CNN transforms the 3D input volume to a 3D output volume of neuron activations. In this example, the red input layer holds the image, so its width and height would be the dimensions of the image, and the depth would be 3 (Red, Green, Blue channels).</i></p></p>
|
||
<!-- end figure --></section>
|
||
<section id="layers-used-to-build-cnns">
|
||
<h2><span class="section-number">16.2. </span>Layers used to build CNNs<a class="headerlink" href="#layers-used-to-build-cnns" title="Link to this heading">#</a></h2>
|
||
<p>A simple CNN is a sequence of layers, and every layer of a CNN
|
||
transforms one volume of activations to another through a
|
||
differentiable function. We use three main types of layers to build
|
||
CNN architectures: Convolutional Layer, Pooling Layer, and
|
||
Fully-Connected Layer (exactly as seen in regular Neural Networks). We
|
||
will stack these layers to form a full CNN architecture.</p>
|
||
<p>A simple CNN for image classification could have the architecture:</p>
|
||
<ul class="simple">
|
||
<li><p><strong>INPUT</strong> (<span class="math notranslate nohighlight">\(32\times 32 \times 3\)</span>) will hold the raw pixel values of the image, in this case an image of width 32, height 32, and with three color channels R,G,B.</p></li>
|
||
<li><p><strong>CONV</strong> (convolutional )layer will compute the output of neurons that are connected to local regions in the input, each computing a dot product between their weights and a small region they are connected to in the input volume. This may result in volume such as <span class="math notranslate nohighlight">\([32\times 32\times 12]\)</span> if we decided to use 12 filters.</p></li>
|
||
<li><p><strong>RELU</strong> layer will apply an elementwise activation function, such as the <span class="math notranslate nohighlight">\(max(0,x)\)</span> thresholding at zero. This leaves the size of the volume unchanged (<span class="math notranslate nohighlight">\([32\times 32\times 12]\)</span>).</p></li>
|
||
<li><p><strong>POOL</strong> (pooling) layer will perform a downsampling operation along the spatial dimensions (width, height), resulting in volume such as <span class="math notranslate nohighlight">\([16\times 16\times 12]\)</span>.</p></li>
|
||
<li><p><strong>FC</strong> (i.e. fully-connected) layer will compute the class scores, resulting in volume of size <span class="math notranslate nohighlight">\([1\times 1\times 10]\)</span>, where each of the 10 numbers correspond to a class score, such as among the 10 categories of the MNIST images we considered above . As with ordinary Neural Networks and as the name implies, each neuron in this layer will be connected to all the numbers in the previous volume.</p></li>
|
||
</ul>
|
||
<p>CNNs transform the original image layer by layer from the original
|
||
pixel values to the final class scores.</p>
|
||
<p>Observe that some layers contain
|
||
parameters and other don’t. In particular, the CNN layers perform
|
||
transformations that are a function of not only the activations in the
|
||
input volume, but also of the parameters (the weights and biases of
|
||
the neurons). On the other hand, the RELU/POOL layers will implement a
|
||
fixed function. The parameters in the CONV/FC layers will be trained
|
||
with gradient descent so that the class scores that the CNN computes
|
||
are consistent with the labels in the training set for each image.</p>
|
||
<p>In summary:</p>
|
||
<ul class="simple">
|
||
<li><p>A CNN architecture is in the simplest case a list of Layers that transform the image volume into an output volume (e.g. holding the class scores)</p></li>
|
||
<li><p>There are a few distinct types of Layers (e.g. CONV/FC/RELU/POOL are by far the most popular)</p></li>
|
||
<li><p>Each Layer accepts an input 3D volume and transforms it to an output 3D volume through a differentiable function</p></li>
|
||
<li><p>Each Layer may or may not have parameters (e.g. CONV/FC do, RELU/POOL don’t)</p></li>
|
||
<li><p>Each Layer may or may not have additional hyperparameters (e.g. CONV/FC/POOL do, RELU doesn’t)</p></li>
|
||
</ul>
|
||
<p>A dense neural network is representd by an affine operation (like matrix-matrix multiplication) where all parameters are included.</p>
|
||
<p>The key idea in CNNs for say imaging is that in images neighbor pixels tend to be related! So we connect
|
||
only neighboring neurons in the input instead of connecting all with the first hidden layer.</p>
|
||
<p>We say we perform a filtering (convolution is the mathematical operation).</p>
|
||
</section>
|
||
<section id="mathematics-of-cnns">
|
||
<h2><span class="section-number">16.3. </span>Mathematics of CNNs<a class="headerlink" href="#mathematics-of-cnns" title="Link to this heading">#</a></h2>
|
||
<p>The mathematics of CNNs is based on the mathematical operation of
|
||
<strong>convolution</strong>. In mathematics (in particular in functional analysis),
|
||
convolution is represented by matheematical operation (integration,
|
||
summation etc) on two function in order to produce a third function
|
||
that expresses how the shape of one gets modified by the other.
|
||
Convolution has a plethora of applications in a variety of disciplines, spanning from statistics to signal processing, computer vision, solutions of differential equations,linear algebra, engineering, and yes, machine learning.</p>
|
||
<p>Mathematically, convolution is defined as follows (one-dimensional example):
|
||
Let us define a continuous function <span class="math notranslate nohighlight">\(y(t)\)</span> given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y(t) = \int x(a) w(t-a) da,
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(x(a)\)</span> represents a so-called input and <span class="math notranslate nohighlight">\(w(t-a)\)</span> is normally called the weight function or kernel.</p>
|
||
<p>The above integral is written in a more compact form as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y(t) = \left(x * w\right)(t).
|
||
\]</div>
|
||
<p>The discretized version reads</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y(t) = \sum_{a=-\infty}^{a=\infty}x(a)w(t-a).
|
||
\]</div>
|
||
<p>Computing the inverse of the above convolution operations is known as deconvolution.</p>
|
||
<p>How can we use this? And what does it mean? Let us study some familiar examples first.</p>
|
||
<section id="convolution-examples-polynomial-multiplication">
|
||
<h3><span class="section-number">16.3.1. </span>Convolution Examples: Polynomial multiplication<a class="headerlink" href="#convolution-examples-polynomial-multiplication" title="Link to this heading">#</a></h3>
|
||
<p>We have already met such an example in project 1 when we tried to set
|
||
up the design matrix for a two-dimensional function. This was an
|
||
example of polynomial multiplication. Let us recast such a problem in terms of the convolution operation.
|
||
Let us look a the following polynomials to second and third order, respectively:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
p(t) = \alpha_0+\alpha_1 t+\alpha_2 t^2,
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
s(t) = \beta_0+\beta_1 t+\beta_2 t^2+\beta_3 t^3.
|
||
\]</div>
|
||
<p>The polynomial multiplication gives us a new polynomial of degree <span class="math notranslate nohighlight">\(5\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
z(t) = \delta_0+\delta_1 t+\delta_2 t^2+\delta_3 t^3+\delta_4 t^4+\delta_5 t^5.
|
||
\]</div>
|
||
<p>Computing polynomial products can be implemented efficiently if we rewrite the more brute force multiplications using convolution.
|
||
We note first that the new coefficients are given as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{split}
|
||
\delta_0=&\alpha_0\beta_0\\
|
||
\delta_1=&\alpha_1\beta_0+\alpha_1\beta_0\\
|
||
\delta_2=&\alpha_0\beta_2+\alpha_1\beta_1+\alpha_2\beta_0\\
|
||
\delta_3=&\alpha_1\beta_2+\alpha_2\beta_1+\alpha_0\beta_3\\
|
||
\delta_4=&\alpha_2\beta_2+\alpha_1\beta_3\\
|
||
\delta_5=&\alpha_2\beta_3.\\
|
||
\end{split}
|
||
\end{split}\]</div>
|
||
<p>We note that <span class="math notranslate nohighlight">\(\alpha_i=0\)</span> except for <span class="math notranslate nohighlight">\(i\in \left\{0,1,2\right\}\)</span> and <span class="math notranslate nohighlight">\(\beta_i=0\)</span> except for <span class="math notranslate nohighlight">\(i\in\left\{0,1,2,3\right\}\)</span>.</p>
|
||
<p>We can then rewrite the coefficients <span class="math notranslate nohighlight">\(\delta_j\)</span> using a discrete convolution as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\delta_j = \sum_{i=-\infty}^{i=\infty}\alpha_i\beta_{j-i}=(\alpha * \beta)_j,
|
||
\]</div>
|
||
<p>or as a double sum with restriction <span class="math notranslate nohighlight">\(l=i+j\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\delta_l = \sum_{ij}\alpha_i\beta_{j}.
|
||
\]</div>
|
||
<p>Do you see a potential drawback with these equations?</p>
|
||
<p>Since we only have a finite number of <span class="math notranslate nohighlight">\(\alpha\)</span> and <span class="math notranslate nohighlight">\(\beta\)</span> values
|
||
which are non-zero, we can rewrite the above convolution expressions
|
||
as a matrix-vector multiplication</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\delta}=\begin{bmatrix}\alpha_0 & 0 & 0 & 0 \\
|
||
\alpha_1 & \alpha_0 & 0 & 0 \\
|
||
\alpha_2 & \alpha_1 & \alpha_0 & 0 \\
|
||
0 & \alpha_2 & \alpha_1 & \alpha_0 \\
|
||
0 & 0 & \alpha_2 & \alpha_1 \\
|
||
0 & 0 & 0 & \alpha_2
|
||
\end{bmatrix}\begin{bmatrix} \beta_0 \\ \beta_1 \\ \beta_2 \\ \beta_3\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>The process is commutative and we can easily see that we can rewrite the multiplication in terms of a matrix holding <span class="math notranslate nohighlight">\(\beta\)</span> and a vector holding <span class="math notranslate nohighlight">\(\alpha\)</span>.
|
||
In this case we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\delta}=\begin{bmatrix}\beta_0 & 0 & 0 \\
|
||
\beta_1 & \beta_0 & 0 \\
|
||
\beta_2 & \beta_1 & \beta_0 \\
|
||
\beta_3 & \beta_2 & \beta_1 \\
|
||
0 & \beta_3 & \beta_2 \\
|
||
0 & 0 & \beta_3
|
||
\end{bmatrix}\begin{bmatrix} \alpha_0 \\ \alpha_1 \\ \alpha_2\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>Note that the use of these matrices is for mathematical purposes only and not implementation purposes.
|
||
When implementing the above equation we do not encode (and allocate memory) the matrices explicitely.
|
||
We rather code the convolutions in the minimal memory footprint that they require.</p>
|
||
<p>Does the number of floating point operations change here when we use the commutative property?</p>
|
||
</section>
|
||
<section id="convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms">
|
||
<h3><span class="section-number">16.3.2. </span>Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)<a class="headerlink" href="#convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms" title="Link to this heading">#</a></h3>
|
||
<p>For problems with so-called harmonic oscillations, given by for example the following differential equation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
m\frac{d^2x}{dt^2}+\eta\frac{dx}{dt}+x(t)=F(t),
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(F(t)\)</span> is an applied external force acting on the system (often called a driving force), one can use the theory of Fourier transformations to find the solutions of this type of equations.</p>
|
||
<p>If one has several driving forces, <span class="math notranslate nohighlight">\(F(t)=\sum_n F_n(t)\)</span>, one can find
|
||
the particular solution to each <span class="math notranslate nohighlight">\(F_n\)</span>, <span class="math notranslate nohighlight">\(x_{pn}(t)\)</span>, and the particular
|
||
solution for the entire driving force is then given by a series like</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
x_p(t)=\sum_nx_{pn}(t).
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>This is known as the principle of superposition. It only applies when
|
||
the homogenous equation is linear. If there were an anharmonic term
|
||
such as <span class="math notranslate nohighlight">\(x^3\)</span> in the homogenous equation, then when one summed various
|
||
solutions, <span class="math notranslate nohighlight">\(x=(\sum_n x_n)^2\)</span>, one would get cross
|
||
terms. Superposition is especially useful when <span class="math notranslate nohighlight">\(F(t)\)</span> can be written
|
||
as a sum of sinusoidal terms, because the solutions for each
|
||
sinusoidal (sine or cosine) term is analytic.</p>
|
||
<p>Driving forces are often periodic, even when they are not
|
||
sinusoidal. Periodicity implies that for some time <span class="math notranslate nohighlight">\(\tau\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{eqnarray}
|
||
F(t+\tau)=F(t).
|
||
\end{eqnarray}
|
||
\]</div>
|
||
<p>One example of a non-sinusoidal periodic force is a square wave. Many
|
||
components in electric circuits are non-linear, e.g. diodes, which
|
||
makes many wave forms non-sinusoidal even when the circuits are being
|
||
driven by purely sinusoidal sources.</p>
|
||
<p>The code here shows a typical example of such a square wave generated using the functionality included in the <strong>scipy</strong> Python package. We have used a period of <span class="math notranslate nohighlight">\(\tau=0.2\)</span>.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>%matplotlib inline
|
||
|
||
import numpy as np
|
||
import math
|
||
from scipy import signal
|
||
import matplotlib.pyplot as plt
|
||
|
||
# number of points
|
||
n = 500
|
||
# start and final times
|
||
t0 = 0.0
|
||
tn = 1.0
|
||
# Period
|
||
t = np.linspace(t0, tn, n, endpoint=False)
|
||
SqrSignal = np.zeros(n)
|
||
SqrSignal = 1.0+signal.square(2*np.pi*5*t)
|
||
plt.plot(t, SqrSignal)
|
||
plt.ylim(-0.5, 2.5)
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>For the sinusoidal example the
|
||
period is <span class="math notranslate nohighlight">\(\tau=2\pi/\omega\)</span>. However, higher harmonics can also
|
||
satisfy the periodicity requirement. In general, any force that
|
||
satisfies the periodicity requirement can be expressed as a sum over
|
||
harmonics,</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto2"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
F(t)=\frac{f_0}{2}+\sum_{n>0} f_n\cos(2n\pi t/\tau)+g_n\sin(2n\pi t/\tau).
|
||
\label{_auto2} \tag{2}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>We can write down the answer for
|
||
<span class="math notranslate nohighlight">\(x_{pn}(t)\)</span>, by substituting <span class="math notranslate nohighlight">\(f_n/m\)</span> or <span class="math notranslate nohighlight">\(g_n/m\)</span> for <span class="math notranslate nohighlight">\(F_0/m\)</span>. By
|
||
writing each factor <span class="math notranslate nohighlight">\(2n\pi t/\tau\)</span> as <span class="math notranslate nohighlight">\(n\omega t\)</span>, with <span class="math notranslate nohighlight">\(\omega\equiv
|
||
2\pi/\tau\)</span>,</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:fourierdef1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\label{eq:fourierdef1} \tag{3}
|
||
F(t)=\frac{f_0}{2}+\sum_{n>0}f_n\cos(n\omega t)+g_n\sin(n\omega t).
|
||
\end{equation}
|
||
\]</div>
|
||
<p>The solutions for <span class="math notranslate nohighlight">\(x(t)\)</span> then come from replacing <span class="math notranslate nohighlight">\(\omega\)</span> with
|
||
<span class="math notranslate nohighlight">\(n\omega\)</span> for each term in the particular solution,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{eqnarray}
|
||
x_p(t)&=&\frac{f_0}{2k}+\sum_{n>0} \alpha_n\cos(n\omega t-\delta_n)+\beta_n\sin(n\omega t-\delta_n),\\
|
||
\nonumber
|
||
\alpha_n&=&\frac{f_n/m}{\sqrt{((n\omega)^2-\omega_0^2)+4\beta^2n^2\omega^2}},\\
|
||
\nonumber
|
||
\beta_n&=&\frac{g_n/m}{\sqrt{((n\omega)^2-\omega_0^2)+4\beta^2n^2\omega^2}},\\
|
||
\nonumber
|
||
\delta_n&=&\tan^{-1}\left(\frac{2\beta n\omega}{\omega_0^2-n^2\omega^2}\right).
|
||
\end{eqnarray}
|
||
\end{split}\]</div>
|
||
<p>Because the forces have been applied for a long time, any non-zero
|
||
damping eliminates the homogenous parts of the solution, so one need
|
||
only consider the particular solution for each <span class="math notranslate nohighlight">\(n\)</span>.</p>
|
||
<p>The problem is considered solved if one can find expressions for the
|
||
coefficients <span class="math notranslate nohighlight">\(f_n\)</span> and <span class="math notranslate nohighlight">\(g_n\)</span>, even though the solutions are expressed
|
||
as an infinite sum. The coefficients can be extracted from the
|
||
function <span class="math notranslate nohighlight">\(F(t)\)</span> by</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="eq:fourierdef2"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{eqnarray}
|
||
\label{eq:fourierdef2} \tag{4}
|
||
f_n&=&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~F(t)\cos(2n\pi t/\tau),\\
|
||
\nonumber
|
||
g_n&=&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~F(t)\sin(2n\pi t/\tau).
|
||
\end{eqnarray}
|
||
\end{split}\]</div>
|
||
<p>To check the consistency of these expressions and to verify
|
||
Eq. (<a class="reference internal" href="#eq:fourierdef2"><span class="xref myst">4</span></a>), one can insert the expansion of <span class="math notranslate nohighlight">\(F(t)\)</span> in
|
||
Eq. (<a class="reference internal" href="#eq:fourierdef1"><span class="xref myst">3</span></a>) into the expression for the coefficients in
|
||
Eq. (<a class="reference internal" href="#eq:fourierdef2"><span class="xref myst">4</span></a>) and see whether</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{eqnarray}
|
||
f_n&=?&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~\left\{
|
||
\frac{f_0}{2}+\sum_{m>0}f_m\cos(m\omega t)+g_m\sin(m\omega t)
|
||
\right\}\cos(n\omega t).
|
||
\end{eqnarray}
|
||
\]</div>
|
||
<p>Immediately, one can throw away all the terms with <span class="math notranslate nohighlight">\(g_m\)</span> because they
|
||
convolute an even and an odd function. The term with <span class="math notranslate nohighlight">\(f_0/2\)</span>
|
||
disappears because <span class="math notranslate nohighlight">\(\cos(n\omega t)\)</span> is equally positive and negative
|
||
over the interval and will integrate to zero. For all the terms
|
||
<span class="math notranslate nohighlight">\(f_m\cos(m\omega t)\)</span> appearing in the sum, one can use angle addition
|
||
formulas to see that <span class="math notranslate nohighlight">\(\cos(m\omega t)\cos(n\omega
|
||
t)=(1/2)(\cos[(m+n)\omega t]+\cos[(m-n)\omega t]\)</span>. This will integrate
|
||
to zero unless <span class="math notranslate nohighlight">\(m=n\)</span>. In that case the <span class="math notranslate nohighlight">\(m=n\)</span> term gives</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto3"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\int_{-\tau/2}^{\tau/2}dt~\cos^2(m\omega t)=\frac{\tau}{2},
|
||
\label{_auto3} \tag{5}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{eqnarray}
|
||
f_n&=?&\frac{2}{\tau}\int_{-\tau/2}^{\tau/2} dt~f_n/2\\
|
||
\nonumber
|
||
&=&f_n~\checkmark.
|
||
\end{eqnarray}
|
||
\end{split}\]</div>
|
||
<p>The same method can be used to check for the consistency of <span class="math notranslate nohighlight">\(g_n\)</span>.</p>
|
||
<p>The code here uses the Fourier series applied to a
|
||
square wave signal. The code here
|
||
visualizes the various approximations given by Fourier series compared
|
||
with a square wave with period <span class="math notranslate nohighlight">\(T=0.2\)</span> (dimensionless time), width <span class="math notranslate nohighlight">\(0.1\)</span> and max value of the force <span class="math notranslate nohighlight">\(F=2\)</span>. We
|
||
see that when we increase the number of components in the Fourier
|
||
series, the Fourier series approximation gets closer and closer to the
|
||
square wave signal.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import numpy as np
|
||
import math
|
||
from scipy import signal
|
||
import matplotlib.pyplot as plt
|
||
|
||
# number of points
|
||
n = 500
|
||
# start and final times
|
||
t0 = 0.0
|
||
tn = 1.0
|
||
# Period
|
||
T =0.2
|
||
# Max value of square signal
|
||
Fmax= 2.0
|
||
# Width of signal
|
||
Width = 0.1
|
||
t = np.linspace(t0, tn, n, endpoint=False)
|
||
SqrSignal = np.zeros(n)
|
||
FourierSeriesSignal = np.zeros(n)
|
||
SqrSignal = 1.0+signal.square(2*np.pi*5*t+np.pi*Width/T)
|
||
a0 = Fmax*Width/T
|
||
FourierSeriesSignal = a0
|
||
Factor = 2.0*Fmax/np.pi
|
||
for i in range(1,500):
|
||
FourierSeriesSignal += Factor/(i)*np.sin(np.pi*i*Width/T)*np.cos(i*t*2*np.pi/T)
|
||
plt.plot(t, SqrSignal)
|
||
plt.plot(t, FourierSeriesSignal)
|
||
plt.ylim(-0.5, 2.5)
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="two-dimensional-objects">
|
||
<h2><span class="section-number">16.4. </span>Two-dimensional Objects<a class="headerlink" href="#two-dimensional-objects" title="Link to this heading">#</a></h2>
|
||
<p>We often use convolutions over more than one dimension at a time. If
|
||
we have a two-dimensional image <span class="math notranslate nohighlight">\(I\)</span> as input, we can have a <strong>filter</strong>
|
||
defined by a two-dimensional <strong>kernel</strong> <span class="math notranslate nohighlight">\(K\)</span>. This leads to an output <span class="math notranslate nohighlight">\(S\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
S_(i,j)=(I * K)(i,j) = \sum_m\sum_n I(m,n)K(i-m,j-n).
|
||
\]</div>
|
||
<p>Convolution is a commutatitave process, which means we can rewrite this equation as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
S_(i,j)=(I * K)(i,j) = \sum_m\sum_n I(i-m,j-n)K(m,n).
|
||
\]</div>
|
||
<p>Normally the latter is more straightforward to implement in a machine elarning library since there is less variation in the range of values of <span class="math notranslate nohighlight">\(m\)</span> and <span class="math notranslate nohighlight">\(n\)</span>.</p>
|
||
<p>Many deep learning libraries implement cross-correlation instead of convolution</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
S_(i,j)=(I * K)(i,j) = \sum_m\sum_n I(i+m,j-+)K(m,n).
|
||
\]</div>
|
||
</section>
|
||
<section id="more-on-dimensionalities">
|
||
<h2><span class="section-number">16.5. </span>More on Dimensionalities<a class="headerlink" href="#more-on-dimensionalities" title="Link to this heading">#</a></h2>
|
||
<p>In fields like signal processing (and imaging as well), one designs
|
||
so-called filters. These filters are defined by the convolutions and
|
||
are often hand-crafted. One may specify filters for smoothing, edge
|
||
detection, frequency reshaping, and similar operations. However with
|
||
neural networks the idea is to automatically learn the filters and use
|
||
many of them in conjunction with non-linear operations (activation
|
||
functions).</p>
|
||
<p>As an example consider a neural network operating on sound sequence
|
||
data. Assume that we an input vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> of length <span class="math notranslate nohighlight">\(d=10^6\)</span>. We
|
||
construct then a neural network with onle hidden layer only with
|
||
<span class="math notranslate nohighlight">\(10^4\)</span> nodes. This means that we will have a weight matrix with
|
||
<span class="math notranslate nohighlight">\(10^4\times 10^6=10^{10}\)</span> weights to be determined, together with <span class="math notranslate nohighlight">\(10^4\)</span> biases.</p>
|
||
<p>Assume furthermore that we have an output layer which is meant to train whether the sound sequence represents a human voice (true) or something else (false).
|
||
It means that we have only one output node. But since this output node connects to <span class="math notranslate nohighlight">\(10^4\)</span> nodes in the hidden layer, there are in total <span class="math notranslate nohighlight">\(10^4\)</span> weights to be determined for the output layer, plus one bias. In total we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{NumberParameters}=10^{10}+10^4+10^4+1 \approx 10^{10},
|
||
\]</div>
|
||
<p>that is ten billion parameters to determine.</p>
|
||
</section>
|
||
<section id="further-dimensionality-remarks">
|
||
<h2><span class="section-number">16.6. </span>Further Dimensionality Remarks<a class="headerlink" href="#further-dimensionality-remarks" title="Link to this heading">#</a></h2>
|
||
<p>In today’s architecture one can train such neural networks, however
|
||
this is a huge number of parameters for the task at hand. In general,
|
||
it is a very wasteful and inefficient use of dense matrices as
|
||
parameters. Just as importantly, such trained network parameters are
|
||
very specific for the type of input data on which they were trained
|
||
and the network is not likely to generalize easily to variations in
|
||
the input.</p>
|
||
<p>The main principles that justify convolutions is locality of
|
||
information and repetion of patterns within the signal. Sound samples
|
||
of the input in adjacent spots are much more likely to affect each
|
||
other than those that are very far away. Similarly, sounds are
|
||
repeated in multiple times in the signal. While slightly simplistic,
|
||
reasoning about such a sound example demonstrates this. The same
|
||
principles then apply to images and other similar data.</p>
|
||
</section>
|
||
<section id="cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras">
|
||
<h2><span class="section-number">16.7. </span>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras<a class="headerlink" href="#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" title="Link to this heading">#</a></h2>
|
||
<p>As discussed above, CNNs are neural networks built from the assumption that the inputs
|
||
to the network are 2D images. This is important because the number of features or pixels in images
|
||
grows very fast with the image size, and an enormous number of weights and biases are needed in order to build an accurate network.</p>
|
||
<p>As before, we still have our input, a hidden layer and an output. What’s novel about convolutional networks
|
||
are the <strong>convolutional</strong> and <strong>pooling</strong> layers stacked in pairs between the input and the hidden layer.
|
||
In addition, the data is no longer represented as a 2D feature matrix, instead each input is a number of 2D
|
||
matrices, typically 1 for each color dimension (Red, Green, Blue).</p>
|
||
<p>It means that to represent the entire
|
||
dataset of images, we require a 4D matrix or <strong>tensor</strong>. This tensor has the dimensions:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(n_{inputs},\, n_{pixels, width},\, n_{pixels, height},\, depth) .
|
||
\]</div>
|
||
<section id="the-mnist-dataset-again">
|
||
<h3><span class="section-number">16.7.1. </span>The MNIST dataset again<a class="headerlink" href="#the-mnist-dataset-again" title="Link to this heading">#</a></h3>
|
||
<p>The MNIST dataset consists of grayscale images with a pixel size of
|
||
<span class="math notranslate nohighlight">\(28\times 28\)</span>, meaning we require <span class="math notranslate nohighlight">\(28 \times 28 = 724\)</span> weights to each
|
||
neuron in the first hidden layer.</p>
|
||
<p>If we were to analyze images of size <span class="math notranslate nohighlight">\(128\times 128\)</span> we would require
|
||
<span class="math notranslate nohighlight">\(128 \times 128 = 16384\)</span> weights to each neuron. Even worse if we were
|
||
dealing with color images, as most images are, we have an image matrix
|
||
of size <span class="math notranslate nohighlight">\(128\times 128\)</span> for each color dimension (Red, Green, Blue),
|
||
meaning 3 times the number of weights <span class="math notranslate nohighlight">\(= 49152\)</span> are required for every
|
||
single neuron in the first hidden layer.</p>
|
||
<p>Images typically have strong local correlations, meaning that a small
|
||
part of the image varies little from its neighboring regions. If for
|
||
example we have an image of a blue car, we can roughly assume that a
|
||
small blue part of the image is surrounded by other blue regions.</p>
|
||
<p>Therefore, instead of connecting every single pixel to a neuron in the
|
||
first hidden layer, as we have previously done with deep neural
|
||
networks, we can instead connect each neuron to a small part of the
|
||
image (in all 3 RGB depth dimensions). The size of each small area is
|
||
fixed, and known as a <a class="reference external" href="https://en.wikipedia.org/wiki/Receptive_field">receptive</a>.</p>
|
||
<p>The layers of a convolutional neural network arrange neurons in 3D: width, height and depth.<br />
|
||
The input image is typically a square matrix of depth 3.</p>
|
||
<p>A <strong>convolution</strong> is performed on the image which outputs
|
||
a 3D volume of neurons. The weights to the input are arranged in a number of 2D matrices, known as <strong>filters</strong>.</p>
|
||
<p>Each filter slides along the input image, taking the dot product
|
||
between each small part of the image and the filter, in all depth
|
||
dimensions. This is then passed through a non-linear function,
|
||
typically the <strong>Rectified Linear (ReLu)</strong> function, which serves as the
|
||
activation of the neurons in the first convolutional layer. This is
|
||
further passed through a <strong>pooling layer</strong>, which reduces the size of the
|
||
convolutional layer, e.g. by taking the maximum or average across some
|
||
small regions, and this serves as input to the next convolutional
|
||
layer.</p>
|
||
</section>
|
||
<section id="systematic-reduction">
|
||
<h3><span class="section-number">16.7.2. </span>Systematic reduction<a class="headerlink" href="#systematic-reduction" title="Link to this heading">#</a></h3>
|
||
<p>By systematically reducing the size of the input volume, through
|
||
convolution and pooling, the network should create representations of
|
||
small parts of the input, and then from them assemble representations
|
||
of larger areas. The final pooling layer is flattened to serve as
|
||
input to a hidden layer, such that each neuron in the final pooling
|
||
layer is connected to every single neuron in the hidden layer. This
|
||
then serves as input to the output layer, e.g. a softmax output for
|
||
classification.</p>
|
||
</section>
|
||
<section id="prerequisites-collect-and-pre-process-data">
|
||
<h3><span class="section-number">16.7.3. </span>Prerequisites: Collect and pre-process data<a class="headerlink" href="#prerequisites-collect-and-pre-process-data" title="Link to this heading">#</a></h3>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span># import necessary packages
|
||
import numpy as np
|
||
import matplotlib.pyplot as plt
|
||
from sklearn import datasets
|
||
|
||
|
||
# ensure the same random numbers appear every time
|
||
np.random.seed(0)
|
||
|
||
# display images in notebook
|
||
%matplotlib inline
|
||
plt.rcParams['figure.figsize'] = (12,12)
|
||
|
||
|
||
# download MNIST dataset
|
||
digits = datasets.load_digits()
|
||
|
||
# define inputs and labels
|
||
inputs = digits.images
|
||
labels = digits.target
|
||
|
||
# RGB images have a depth of 3
|
||
# our images are grayscale so they should have a depth of 1
|
||
inputs = inputs[:,:,:,np.newaxis]
|
||
|
||
print("inputs = (n_inputs, pixel_width, pixel_height, depth) = " + str(inputs.shape))
|
||
print("labels = (n_inputs) = " + str(labels.shape))
|
||
|
||
|
||
# choose some random images to display
|
||
n_inputs = len(inputs)
|
||
indices = np.arange(n_inputs)
|
||
random_indices = np.random.choice(indices, size=5)
|
||
|
||
for i, image in enumerate(digits.images[random_indices]):
|
||
plt.subplot(1, 5, i+1)
|
||
plt.axis('off')
|
||
plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
|
||
plt.title("Label: %d" % digits.target[random_indices[i]])
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from tensorflow.keras import datasets, layers, models
|
||
from tensorflow.keras.layers import Input
|
||
from tensorflow.keras.models import Sequential #This allows appending layers to existing models
|
||
from tensorflow.keras.layers import Dense #This allows defining the characteristics of a particular layer
|
||
from tensorflow.keras import optimizers #This allows using whichever optimiser we want (sgd,adam,RMSprop)
|
||
from tensorflow.keras import regularizers #This allows using whichever regularizer we want (l1,l2,l1_l2)
|
||
from tensorflow.keras.utils import to_categorical #This allows using categorical cross entropy as the cost function
|
||
#from tensorflow.keras import Conv2D
|
||
#from tensorflow.keras import MaxPooling2D
|
||
#from tensorflow.keras import Flatten
|
||
|
||
from sklearn.model_selection import train_test_split
|
||
|
||
# representation of labels
|
||
labels = to_categorical(labels)
|
||
|
||
# split into train and test data
|
||
# one-liner from scikit-learn library
|
||
train_size = 0.8
|
||
test_size = 1 - train_size
|
||
X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
|
||
test_size=test_size)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>def create_convolutional_neural_network_keras(input_shape, receptive_field,
|
||
n_filters, n_neurons_connected, n_categories,
|
||
eta, lmbd):
|
||
model = Sequential()
|
||
model.add(layers.Conv2D(n_filters, (receptive_field, receptive_field), input_shape=input_shape, padding='same',
|
||
activation='relu', kernel_regularizer=regularizers.l2(lmbd)))
|
||
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
|
||
model.add(layers.Flatten())
|
||
model.add(layers.Dense(n_neurons_connected, activation='relu', kernel_regularizer=regularizers.l2(lmbd)))
|
||
model.add(layers.Dense(n_categories, activation='softmax', kernel_regularizer=regularizers.l2(lmbd)))
|
||
|
||
sgd = optimizers.SGD(lr=eta)
|
||
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
|
||
|
||
return model
|
||
|
||
epochs = 100
|
||
batch_size = 100
|
||
input_shape = X_train.shape[1:4]
|
||
receptive_field = 3
|
||
n_filters = 10
|
||
n_neurons_connected = 50
|
||
n_categories = 10
|
||
|
||
eta_vals = np.logspace(-5, 1, 7)
|
||
lmbd_vals = np.logspace(-5, 1, 7)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>CNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
|
||
|
||
for i, eta in enumerate(eta_vals):
|
||
for j, lmbd in enumerate(lmbd_vals):
|
||
CNN = create_convolutional_neural_network_keras(input_shape, receptive_field,
|
||
n_filters, n_neurons_connected, n_categories,
|
||
eta, lmbd)
|
||
CNN.fit(X_train, Y_train, epochs=epochs, batch_size=batch_size, verbose=0)
|
||
scores = CNN.evaluate(X_test, Y_test)
|
||
|
||
CNN_keras[i][j] = CNN
|
||
|
||
print("Learning rate = ", eta)
|
||
print("Lambda = ", lmbd)
|
||
print("Test accuracy: %.3f" % scores[1])
|
||
print()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span># visual representation of grid search
|
||
# uses seaborn heatmap, could probably do this in matplotlib
|
||
import seaborn as sns
|
||
|
||
sns.set()
|
||
|
||
train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
||
test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
|
||
|
||
for i in range(len(eta_vals)):
|
||
for j in range(len(lmbd_vals)):
|
||
CNN = CNN_keras[i][j]
|
||
|
||
train_accuracy[i][j] = CNN.evaluate(X_train, Y_train)[1]
|
||
test_accuracy[i][j] = CNN.evaluate(X_test, Y_test)[1]
|
||
|
||
|
||
fig, ax = plt.subplots(figsize = (10, 10))
|
||
sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
|
||
ax.set_title("Training Accuracy")
|
||
ax.set_ylabel("$\eta$")
|
||
ax.set_xlabel("$\lambda$")
|
||
plt.show()
|
||
|
||
fig, ax = plt.subplots(figsize = (10, 10))
|
||
sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
|
||
ax.set_title("Test Accuracy")
|
||
ax.set_ylabel("$\eta$")
|
||
ax.set_xlabel("$\lambda$")
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="the-cifar01-data-set">
|
||
<h2><span class="section-number">16.8. </span>The CIFAR01 data set<a class="headerlink" href="#the-cifar01-data-set" title="Link to this heading">#</a></h2>
|
||
<p>The CIFAR10 dataset contains 60,000 color images in 10 classes, with
|
||
6,000 images in each class. The dataset is divided into 50,000
|
||
training images and 10,000 testing images. The classes are mutually
|
||
exclusive and there is no overlap between them.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import tensorflow as tf
|
||
|
||
from tensorflow.keras import datasets, layers, models
|
||
import matplotlib.pyplot as plt
|
||
|
||
# We import the data set
|
||
(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()
|
||
|
||
# Normalize pixel values to be between 0 and 1 by dividing by 255.
|
||
train_images, test_images = train_images / 255.0, test_images / 255.0
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>To verify that the dataset looks correct, let’s plot the first 25 images from the training set and display the class name below each image.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer',
|
||
'dog', 'frog', 'horse', 'ship', 'truck']
|
||
|
||
plt.figure(figsize=(10,10))
|
||
for i in range(25):
|
||
plt.subplot(5,5,i+1)
|
||
plt.xticks([])
|
||
plt.yticks([])
|
||
plt.grid(False)
|
||
plt.imshow(train_images[i], cmap=plt.cm.binary)
|
||
# The CIFAR labels happen to be arrays,
|
||
# which is why you need the extra index
|
||
plt.xlabel(class_names[train_labels[i][0]])
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The six lines of code below define the convolutional base using a common pattern: a stack of Conv2D and MaxPooling2D layers.</p>
|
||
<p>As input, a CNN takes tensors of shape (image_height, image_width, color_channels), ignoring the batch size. If you are new to these dimensions, color_channels refers to (R,G,B). In this example, you will configure our CNN to process inputs of shape (32, 32, 3), which is the format of CIFAR images. You can do this by passing the argument input_shape to our first layer.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>model = models.Sequential()
|
||
model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))
|
||
model.add(layers.MaxPooling2D((2, 2)))
|
||
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
|
||
model.add(layers.MaxPooling2D((2, 2)))
|
||
model.add(layers.Conv2D(64, (3, 3), activation='relu'))
|
||
|
||
# Let's display the architecture of our model so far.
|
||
|
||
model.summary()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>You can see that the output of every Conv2D and MaxPooling2D layer is a 3D tensor of shape (height, width, channels). The width and height dimensions tend to shrink as you go deeper in the network. The number of output channels for each Conv2D layer is controlled by the first argument (e.g., 32 or 64). Typically, as the width and height shrink, you can afford (computationally) to add more output channels in each Conv2D layer.</p>
|
||
<p>To complete our model, you will feed the last output tensor from the
|
||
convolutional base (of shape (4, 4, 64)) into one or more Dense layers
|
||
to perform classification. Dense layers take vectors as input (which
|
||
are 1D), while the current output is a 3D tensor. First, you will
|
||
flatten (or unroll) the 3D output to 1D, then add one or more Dense
|
||
layers on top. CIFAR has 10 output classes, so you use a final Dense
|
||
layer with 10 outputs and a softmax activation.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>model.add(layers.Flatten())
|
||
model.add(layers.Dense(64, activation='relu'))
|
||
model.add(layers.Dense(10))
|
||
Here's the complete architecture of our model.
|
||
|
||
model.summary()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>As you can see, our (4, 4, 64) outputs were flattened into vectors of shape (1024) before going through two Dense layers.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>model.compile(optimizer='adam',
|
||
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
|
||
metrics=['accuracy'])
|
||
|
||
history = model.fit(train_images, train_labels, epochs=10,
|
||
validation_data=(test_images, test_labels))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>plt.plot(history.history['accuracy'], label='accuracy')
|
||
plt.plot(history.history['val_accuracy'], label = 'val_accuracy')
|
||
plt.xlabel('Epoch')
|
||
plt.ylabel('Accuracy')
|
||
plt.ylim([0.5, 1])
|
||
plt.legend(loc='lower right')
|
||
|
||
test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=2)
|
||
|
||
print(test_acc)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="chapter11.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title"><span class="section-number">15. </span>Solving Differential Equations with Deep Learning</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="chapter13.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title"><span class="section-number">17. </span>Recurrent neural networks: Overarching view</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#neural-networks-vs-cnns">16.1. Neural Networks vs CNNs</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#layers-used-to-build-cnns">16.2. Layers used to build CNNs</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematics-of-cnns">16.3. Mathematics of CNNs</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#convolution-examples-polynomial-multiplication">16.3.1. Convolution Examples: Polynomial multiplication</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms">16.3.2. Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#two-dimensional-objects">16.4. Two-dimensional Objects</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-on-dimensionalities">16.5. More on Dimensionalities</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#further-dimensionality-remarks">16.6. Further Dimensionality Remarks</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras">16.7. CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#the-mnist-dataset-again">16.7.1. The MNIST dataset again</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#systematic-reduction">16.7.2. Systematic reduction</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#prerequisites-collect-and-pre-process-data">16.7.3. Prerequisites: Collect and pre-process data</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-cifar01-data-set">16.8. The CIFAR01 data set</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |