3602 lines
303 KiB
HTML
3602 lines
303 KiB
HTML
<!--
|
||
HTML file automatically generated from DocOnce source
|
||
(https://github.com/doconce/doconce/)
|
||
doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week44-bs --no_mako
|
||
-->
|
||
<html>
|
||
<head>
|
||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<meta name="description" content="Week 44, Convolutional Neural Networks (CNN)">
|
||
<title>Week 44, Convolutional Neural Networks (CNN)</title>
|
||
<!-- Bootstrap style: bootstrap -->
|
||
<!-- doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week44-bs --no_mako -->
|
||
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
||
<!-- not necessary
|
||
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
||
-->
|
||
<style type="text/css">
|
||
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
||
.dropdown-menu {
|
||
height: auto;
|
||
max-height: 400px;
|
||
overflow-x: hidden;
|
||
}
|
||
/* Adds an invisible element before each target to offset for the navigation
|
||
bar */
|
||
.anchor::before {
|
||
content:"";
|
||
display:block;
|
||
height:50px; /* fixed header height for style bootstrap */
|
||
margin:-50px 0 0; /* negative fixed header height */
|
||
}
|
||
</style>
|
||
</head>
|
||
|
||
<!-- tocinfo
|
||
{'highest level': 2,
|
||
'sections': [('Plan for week 44', 2, None, 'plan-for-week-44'),
|
||
('Material for Lecture Thursday November 2',
|
||
2,
|
||
None,
|
||
'material-for-lecture-thursday-november-2'),
|
||
('Convolutional Neural Networks (recognizing images)',
|
||
2,
|
||
None,
|
||
'convolutional-neural-networks-recognizing-images'),
|
||
('What is the Difference', 2, None, 'what-is-the-difference'),
|
||
('Neural Networks vs CNNs', 2, None, 'neural-networks-vs-cnns'),
|
||
('Why CNNS for images, sound files, medical images from CT scans '
|
||
'etc?',
|
||
2,
|
||
None,
|
||
'why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc'),
|
||
('Regular NNs don’t scale well to full images',
|
||
2,
|
||
None,
|
||
'regular-nns-don-t-scale-well-to-full-images'),
|
||
('3D volumes of neurons', 2, None, '3d-volumes-of-neurons'),
|
||
('Layers used to build CNNs',
|
||
2,
|
||
None,
|
||
'layers-used-to-build-cnns'),
|
||
('Transforming images', 2, None, 'transforming-images'),
|
||
('CNNs in brief', 2, None, 'cnns-in-brief'),
|
||
('Key Idea', 2, None, 'key-idea'),
|
||
('Mathematics of CNNs', 2, None, 'mathematics-of-cnns'),
|
||
('Convolution Examples: Polynomial multiplication',
|
||
2,
|
||
None,
|
||
'convolution-examples-polynomial-multiplication'),
|
||
('Efficient Polynomial Multiplication',
|
||
2,
|
||
None,
|
||
'efficient-polynomial-multiplication'),
|
||
('A more efficient way of coding the above Convolution',
|
||
2,
|
||
None,
|
||
'a-more-efficient-way-of-coding-the-above-convolution'),
|
||
('Convolution Examples: Principle of Superposition and Periodic '
|
||
'Forces (Fourier Transforms)',
|
||
2,
|
||
None,
|
||
'convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms'),
|
||
('Simple Code Example', 2, None, 'simple-code-example'),
|
||
('Wrapping up Fourier transforms',
|
||
2,
|
||
None,
|
||
'wrapping-up-fourier-transforms'),
|
||
('Finding the Coefficients', 2, None, 'finding-the-coefficients'),
|
||
('Final words on Fourier Transforms',
|
||
2,
|
||
None,
|
||
'final-words-on-fourier-transforms'),
|
||
('Fourier transforms and convolution',
|
||
3,
|
||
None,
|
||
'fourier-transforms-and-convolution'),
|
||
('Two-dimensional Objects', 2, None, 'two-dimensional-objects'),
|
||
('More on Dimensionalities', 2, None, 'more-on-dimensionalities'),
|
||
('Further Dimensionality Remarks',
|
||
2,
|
||
None,
|
||
'further-dimensionality-remarks'),
|
||
('CNNs in more detail', 2, None, 'cnns-in-more-detail'),
|
||
('Pooling', 2, None, 'pooling'),
|
||
('No zero padding, unit strides',
|
||
2,
|
||
None,
|
||
'no-zero-padding-unit-strides'),
|
||
('Zero padding, unit strides',
|
||
2,
|
||
None,
|
||
'zero-padding-unit-strides'),
|
||
('Half (same) padding', 2, None, 'half-same-padding'),
|
||
('Full padding', 2, None, 'full-padding'),
|
||
('Pooling arithmetic', 2, None, 'pooling-arithmetic'),
|
||
('CNNs in more detail, building convolutional neural networks in '
|
||
'Tensorflow and Keras',
|
||
2,
|
||
None,
|
||
'cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras'),
|
||
('Setting it up', 2, None, 'setting-it-up'),
|
||
('The MNIST dataset again', 2, None, 'the-mnist-dataset-again'),
|
||
('Strong correlations', 2, None, 'strong-correlations'),
|
||
('Layers of a CNN', 2, None, 'layers-of-a-cnn'),
|
||
('Systematic reduction', 2, None, 'systematic-reduction'),
|
||
('Prerequisites: Collect and pre-process data',
|
||
2,
|
||
None,
|
||
'prerequisites-collect-and-pre-process-data'),
|
||
('Importing Keras and Tensorflow',
|
||
2,
|
||
None,
|
||
'importing-keras-and-tensorflow'),
|
||
('Running with Keras', 2, None, 'running-with-keras'),
|
||
('Final part', 2, None, 'final-part'),
|
||
('Final visualization', 2, None, 'final-visualization'),
|
||
('The CIFAR01 data set', 2, None, 'the-cifar01-data-set'),
|
||
('Verifying the data set', 2, None, 'verifying-the-data-set'),
|
||
('Set up the model', 2, None, 'set-up-the-model'),
|
||
('Add Dense layers on top', 2, None, 'add-dense-layers-on-top'),
|
||
('Compile and train the model',
|
||
2,
|
||
None,
|
||
'compile-and-train-the-model'),
|
||
('Finally, evaluate the model',
|
||
2,
|
||
None,
|
||
'finally-evaluate-the-model'),
|
||
('Building our own CNN code',
|
||
2,
|
||
None,
|
||
'building-our-own-cnn-code'),
|
||
('List of contents:', 3, None, 'list-of-contents'),
|
||
('Schedulers', 3, None, 'schedulers'),
|
||
('Usage of schedulers', 3, None, 'usage-of-schedulers'),
|
||
('Cost functions', 3, None, 'cost-functions'),
|
||
('Usage of cost functions', 3, None, 'usage-of-cost-functions'),
|
||
('Activation functions', 3, None, 'activation-functions'),
|
||
('Usage of activation functions',
|
||
3,
|
||
None,
|
||
'usage-of-activation-functions'),
|
||
('Convolution', 3, None, 'convolution'),
|
||
('Layers', 3, None, 'layers'),
|
||
('Convolution2DLayer: convolution in a hidden layer',
|
||
3,
|
||
None,
|
||
'convolution2dlayer-convolution-in-a-hidden-layer'),
|
||
('Backpropagation in the convolutional layer',
|
||
3,
|
||
None,
|
||
'backpropagation-in-the-convolutional-layer'),
|
||
('Demonstration', 3, None, 'demonstration'),
|
||
('Pooling Layer', 3, None, 'pooling-layer'),
|
||
('Flattening Layer', 3, None, 'flattening-layer'),
|
||
('Fully Connected Layers', 3, None, 'fully-connected-layers'),
|
||
('Optimized Convolution2DLayer',
|
||
3,
|
||
None,
|
||
'optimized-convolution2dlayer'),
|
||
('The Convolutional Neural Network (CNN)',
|
||
3,
|
||
None,
|
||
'the-convolutional-neural-network-cnn'),
|
||
('Usage of CNN code', 3, None, 'usage-of-cnn-code'),
|
||
('Additional Remarks', 3, None, 'additional-remarks'),
|
||
('Remarks on the speed', 3, None, 'remarks-on-the-speed'),
|
||
('Convolution using separable kernels',
|
||
3,
|
||
None,
|
||
'convolution-using-separable-kernels'),
|
||
('Convolution in the Fourier domain',
|
||
3,
|
||
None,
|
||
'convolution-in-the-fourier-domain')]}
|
||
end of tocinfo -->
|
||
|
||
<body>
|
||
|
||
|
||
|
||
<script type="text/x-mathjax-config">
|
||
MathJax.Hub.Config({
|
||
TeX: {
|
||
equationNumbers: { autoNumber: "none" },
|
||
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
||
}
|
||
});
|
||
</script>
|
||
<script type="text/javascript" async
|
||
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
||
</script>
|
||
|
||
|
||
<!-- Bootstrap navigation bar -->
|
||
<div class="navbar navbar-default navbar-fixed-top">
|
||
<div class="navbar-header">
|
||
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
<span class="icon-bar"></span>
|
||
</button>
|
||
<a class="navbar-brand" href="week44-bs.html">Week 44, Convolutional Neural Networks (CNN)</a>
|
||
</div>
|
||
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
||
<ul class="nav navbar-nav navbar-right">
|
||
<li class="dropdown">
|
||
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
||
<ul class="dropdown-menu">
|
||
<!-- navigation toc: --> <li><a href="._week44-bs001.html#plan-for-week-44" style="font-size: 80%;"><b>Plan for week 44</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs002.html#material-for-lecture-thursday-november-2" style="font-size: 80%;"><b>Material for Lecture Thursday November 2</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs003.html#convolutional-neural-networks-recognizing-images" style="font-size: 80%;"><b>Convolutional Neural Networks (recognizing images)</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs004.html#what-is-the-difference" style="font-size: 80%;"><b>What is the Difference</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs005.html#neural-networks-vs-cnns" style="font-size: 80%;"><b>Neural Networks vs CNNs</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs006.html#why-cnns-for-images-sound-files-medical-images-from-ct-scans-etc" style="font-size: 80%;"><b>Why CNNS for images, sound files, medical images from CT scans etc?</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs007.html#regular-nns-don-t-scale-well-to-full-images" style="font-size: 80%;"><b>Regular NNs don’t scale well to full images</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs008.html#3d-volumes-of-neurons" style="font-size: 80%;"><b>3D volumes of neurons</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs009.html#layers-used-to-build-cnns" style="font-size: 80%;"><b>Layers used to build CNNs</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs010.html#transforming-images" style="font-size: 80%;"><b>Transforming images</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs011.html#cnns-in-brief" style="font-size: 80%;"><b>CNNs in brief</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs012.html#key-idea" style="font-size: 80%;"><b>Key Idea</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs013.html#mathematics-of-cnns" style="font-size: 80%;"><b>Mathematics of CNNs</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs014.html#convolution-examples-polynomial-multiplication" style="font-size: 80%;"><b>Convolution Examples: Polynomial multiplication</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs015.html#efficient-polynomial-multiplication" style="font-size: 80%;"><b>Efficient Polynomial Multiplication</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs016.html#a-more-efficient-way-of-coding-the-above-convolution" style="font-size: 80%;"><b>A more efficient way of coding the above Convolution</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs017.html#convolution-examples-principle-of-superposition-and-periodic-forces-fourier-transforms" style="font-size: 80%;"><b>Convolution Examples: Principle of Superposition and Periodic Forces (Fourier Transforms)</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs018.html#simple-code-example" style="font-size: 80%;"><b>Simple Code Example</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs019.html#wrapping-up-fourier-transforms" style="font-size: 80%;"><b>Wrapping up Fourier transforms</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs020.html#finding-the-coefficients" style="font-size: 80%;"><b>Finding the Coefficients</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs021.html#final-words-on-fourier-transforms" style="font-size: 80%;"><b>Final words on Fourier Transforms</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs021.html#fourier-transforms-and-convolution" style="font-size: 80%;"> Fourier transforms and convolution</a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs021.html#two-dimensional-objects" style="font-size: 80%;"><b>Two-dimensional Objects</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs022.html#more-on-dimensionalities" style="font-size: 80%;"><b>More on Dimensionalities</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs023.html#further-dimensionality-remarks" style="font-size: 80%;"><b>Further Dimensionality Remarks</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs024.html#cnns-in-more-detail" style="font-size: 80%;"><b>CNNs in more detail</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs025.html#pooling" style="font-size: 80%;"><b>Pooling</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs026.html#no-zero-padding-unit-strides" style="font-size: 80%;"><b>No zero padding, unit strides</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs027.html#zero-padding-unit-strides" style="font-size: 80%;"><b>Zero padding, unit strides</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs028.html#half-same-padding" style="font-size: 80%;"><b>Half (same) padding</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs029.html#full-padding" style="font-size: 80%;"><b>Full padding</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs030.html#pooling-arithmetic" style="font-size: 80%;"><b>Pooling arithmetic</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs031.html#cnns-in-more-detail-building-convolutional-neural-networks-in-tensorflow-and-keras" style="font-size: 80%;"><b>CNNs in more detail, building convolutional neural networks in Tensorflow and Keras</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs032.html#setting-it-up" style="font-size: 80%;"><b>Setting it up</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs033.html#the-mnist-dataset-again" style="font-size: 80%;"><b>The MNIST dataset again</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs034.html#strong-correlations" style="font-size: 80%;"><b>Strong correlations</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs035.html#layers-of-a-cnn" style="font-size: 80%;"><b>Layers of a CNN</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs036.html#systematic-reduction" style="font-size: 80%;"><b>Systematic reduction</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs037.html#prerequisites-collect-and-pre-process-data" style="font-size: 80%;"><b>Prerequisites: Collect and pre-process data</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs038.html#importing-keras-and-tensorflow" style="font-size: 80%;"><b>Importing Keras and Tensorflow</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs039.html#running-with-keras" style="font-size: 80%;"><b>Running with Keras</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs040.html#final-part" style="font-size: 80%;"><b>Final part</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs041.html#final-visualization" style="font-size: 80%;"><b>Final visualization</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs042.html#the-cifar01-data-set" style="font-size: 80%;"><b>The CIFAR01 data set</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs043.html#verifying-the-data-set" style="font-size: 80%;"><b>Verifying the data set</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs044.html#set-up-the-model" style="font-size: 80%;"><b>Set up the model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs045.html#add-dense-layers-on-top" style="font-size: 80%;"><b>Add Dense layers on top</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs046.html#compile-and-train-the-model" style="font-size: 80%;"><b>Compile and train the model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="._week44-bs047.html#finally-evaluate-the-model" style="font-size: 80%;"><b>Finally, evaluate the model</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="#building-our-own-cnn-code" style="font-size: 80%;"><b>Building our own CNN code</b></a></li>
|
||
<!-- navigation toc: --> <li><a href="#list-of-contents" style="font-size: 80%;"> List of contents:</a></li>
|
||
<!-- navigation toc: --> <li><a href="#schedulers" style="font-size: 80%;"> Schedulers</a></li>
|
||
<!-- navigation toc: --> <li><a href="#usage-of-schedulers" style="font-size: 80%;"> Usage of schedulers</a></li>
|
||
<!-- navigation toc: --> <li><a href="#cost-functions" style="font-size: 80%;"> Cost functions</a></li>
|
||
<!-- navigation toc: --> <li><a href="#usage-of-cost-functions" style="font-size: 80%;"> Usage of cost functions</a></li>
|
||
<!-- navigation toc: --> <li><a href="#activation-functions" style="font-size: 80%;"> Activation functions</a></li>
|
||
<!-- navigation toc: --> <li><a href="#usage-of-activation-functions" style="font-size: 80%;"> Usage of activation functions</a></li>
|
||
<!-- navigation toc: --> <li><a href="#convolution" style="font-size: 80%;"> Convolution</a></li>
|
||
<!-- navigation toc: --> <li><a href="#layers" style="font-size: 80%;"> Layers</a></li>
|
||
<!-- navigation toc: --> <li><a href="#convolution2dlayer-convolution-in-a-hidden-layer" style="font-size: 80%;"> Convolution2DLayer: convolution in a hidden layer</a></li>
|
||
<!-- navigation toc: --> <li><a href="#backpropagation-in-the-convolutional-layer" style="font-size: 80%;"> Backpropagation in the convolutional layer</a></li>
|
||
<!-- navigation toc: --> <li><a href="#demonstration" style="font-size: 80%;"> Demonstration</a></li>
|
||
<!-- navigation toc: --> <li><a href="#pooling-layer" style="font-size: 80%;"> Pooling Layer</a></li>
|
||
<!-- navigation toc: --> <li><a href="#flattening-layer" style="font-size: 80%;"> Flattening Layer</a></li>
|
||
<!-- navigation toc: --> <li><a href="#fully-connected-layers" style="font-size: 80%;"> Fully Connected Layers</a></li>
|
||
<!-- navigation toc: --> <li><a href="#optimized-convolution2dlayer" style="font-size: 80%;"> Optimized Convolution2DLayer</a></li>
|
||
<!-- navigation toc: --> <li><a href="#the-convolutional-neural-network-cnn" style="font-size: 80%;"> The Convolutional Neural Network (CNN)</a></li>
|
||
<!-- navigation toc: --> <li><a href="#usage-of-cnn-code" style="font-size: 80%;"> Usage of CNN code</a></li>
|
||
<!-- navigation toc: --> <li><a href="#additional-remarks" style="font-size: 80%;"> Additional Remarks</a></li>
|
||
<!-- navigation toc: --> <li><a href="#remarks-on-the-speed" style="font-size: 80%;"> Remarks on the speed</a></li>
|
||
<!-- navigation toc: --> <li><a href="#convolution-using-separable-kernels" style="font-size: 80%;"> Convolution using separable kernels</a></li>
|
||
<!-- navigation toc: --> <li><a href="#convolution-in-the-fourier-domain" style="font-size: 80%;"> Convolution in the Fourier domain</a></li>
|
||
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div> <!-- end of navigation bar -->
|
||
<div class="container">
|
||
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
||
<a name="part0048"></a>
|
||
<!-- !split -->
|
||
<h2 id="building-our-own-cnn-code" class="anchor">Building our own CNN code </h2>
|
||
|
||
<p>Here we present a flexible and readable python code for a CNN
|
||
implemented with NumPy. We will present the code, showcase how to use
|
||
the codebase and fit a CNN that yields a 99% accuracy on the 28x28
|
||
MNIST dataset within reasonable time.
|
||
</p>
|
||
|
||
<p>The CNN is compatible with all schedulers, cost functions and
|
||
activation functions discussed in constructing our neural network
|
||
codes.
|
||
</p>
|
||
|
||
<p> The CNN code consists of different types of Layer classes, including
|
||
Convolution2DLayer, Pooling2DLayer, FlattenLayer, FullyConnectedLayer
|
||
and OutputLayer, which can be added to the CNN object using the
|
||
interface of the CNN class. This allows you to easily construct your
|
||
own CNN, as well as allowing you to get used to an interface similar
|
||
to that of TensorFlow which is used for real world applications.
|
||
</p>
|
||
|
||
<p>Another important feature of this code is that it throws errors if
|
||
unreasonable decisions are made (for example using a kernel that is
|
||
larger than the image, not using a FlattenLayer, etc), and provides
|
||
the user with an informative error message.
|
||
</p>
|
||
<h3 id="list-of-contents" class="anchor">List of contents: </h3>
|
||
<ol>
|
||
<li> Schedulers</li>
|
||
<li> Activation Functions</li>
|
||
<li> Cost Functions</li>
|
||
<li> Convolution</li>
|
||
<li> Layers</li>
|
||
<li> CNN</li>
|
||
<li> Some final remarks</li>
|
||
</ol>
|
||
<h3 id="schedulers" class="anchor">Schedulers </h3>
|
||
|
||
<p>The code below shows object oriented implementations of the Constant,
|
||
Momentum, Adagrad, AdagradMomentum, RMS prop and Adam schedulers. All
|
||
of the classes belong to the shared abstract Scheduler class, and
|
||
share the update_change() and reset() methods allowing for any of the
|
||
schedulers to be seamlessly used during the training stage, as will
|
||
later be shown in the fit() method of the neural
|
||
network. Update_change() only has one parameter, the gradient
|
||
(\( \delta^{l}_{j}a^{l-1}_k \)), and returns the change which will be
|
||
subtracted from the weights. The reset() function takes no parameters,
|
||
and resets the desired variables. For Constant and Momentum, reset
|
||
does nothing.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Scheduler</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Abstract class for Schedulers</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, eta):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">=</span> eta
|
||
|
||
<span style="color: #408080; font-style: italic"># should be overwritten</span>
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_change</span>(<span style="color: #008000">self</span>, gradient):
|
||
<span style="color: #008000; font-weight: bold">raise</span> <span style="color: #D2413A; font-weight: bold">NotImplementedError</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># overwritten if needed</span>
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">reset</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000; font-weight: bold">pass</span>
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Constant</span>(Scheduler):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, eta):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(eta)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_change</span>(<span style="color: #008000">self</span>, gradient):
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> gradient
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">reset</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000; font-weight: bold">pass</span>
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Momentum</span>(Scheduler):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, eta: <span style="color: #008000">float</span>, momentum: <span style="color: #008000">float</span>):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(eta)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>momentum <span style="color: #666666">=</span> momentum
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>change <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_change</span>(<span style="color: #008000">self</span>, gradient):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>change <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>momentum <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>change <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> gradient
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>change
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">reset</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000; font-weight: bold">pass</span>
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Adagrad</span>(Scheduler):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, eta):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(eta)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_change</span>(<span style="color: #008000">self</span>, gradient):
|
||
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span> <span style="color: #408080; font-style: italic"># avoid division ny zero</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((gradient<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], gradient<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]))
|
||
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">+=</span> gradient <span style="color: #666666">@</span> gradient<span style="color: #666666">.</span>T
|
||
|
||
G_t_inverse <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">/</span> (
|
||
delta <span style="color: #666666">+</span> np<span style="color: #666666">.</span>sqrt(np<span style="color: #666666">.</span>reshape(np<span style="color: #666666">.</span>diagonal(<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t), (<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], <span style="color: #666666">1</span>)))
|
||
)
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> gradient <span style="color: #666666">*</span> G_t_inverse
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">reset</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">AdagradMomentum</span>(Scheduler):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, eta, momentum):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(eta)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>momentum <span style="color: #666666">=</span> momentum
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>change <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_change</span>(<span style="color: #008000">self</span>, gradient):
|
||
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span> <span style="color: #408080; font-style: italic"># avoid division ny zero</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((gradient<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], gradient<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]))
|
||
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">+=</span> gradient <span style="color: #666666">@</span> gradient<span style="color: #666666">.</span>T
|
||
|
||
G_t_inverse <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">/</span> (
|
||
delta <span style="color: #666666">+</span> np<span style="color: #666666">.</span>sqrt(np<span style="color: #666666">.</span>reshape(np<span style="color: #666666">.</span>diagonal(<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t), (<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], <span style="color: #666666">1</span>)))
|
||
)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>change <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>change <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>momentum <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> gradient <span style="color: #666666">*</span> G_t_inverse
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>change
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">reset</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>G_t <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">RMS_prop</span>(Scheduler):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, eta, rho):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(eta)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>rho <span style="color: #666666">=</span> rho
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_change</span>(<span style="color: #008000">self</span>, gradient):
|
||
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span> <span style="color: #408080; font-style: italic"># avoid division ny zero</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">+</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho) <span style="color: #666666">*</span> gradient <span style="color: #666666">*</span> gradient
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> gradient <span style="color: #666666">/</span> (np<span style="color: #666666">.</span>sqrt(<span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">+</span> delta))
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">reset</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">=</span> <span style="color: #666666">0.0</span>
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Adam</span>(Scheduler):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, eta, rho, rho2):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(eta)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>rho <span style="color: #666666">=</span> rho
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>rho2 <span style="color: #666666">=</span> rho2
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>moment <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_epochs <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">update_change</span>(<span style="color: #008000">self</span>, gradient):
|
||
delta <span style="color: #666666">=</span> <span style="color: #666666">1e-8</span> <span style="color: #408080; font-style: italic"># avoid division ny zero</span>
|
||
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>moment <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>moment <span style="color: #666666">+</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho) <span style="color: #666666">*</span> gradient
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho2 <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">+</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho2) <span style="color: #666666">*</span> gradient <span style="color: #666666">*</span> gradient
|
||
|
||
moment_corrected <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>moment <span style="color: #666666">/</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho<span style="color: #666666">**</span><span style="color: #008000">self</span><span style="color: #666666">.</span>n_epochs)
|
||
second_corrected <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">/</span> (<span style="color: #666666">1</span> <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>rho2<span style="color: #666666">**</span><span style="color: #008000">self</span><span style="color: #666666">.</span>n_epochs)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>eta <span style="color: #666666">*</span> moment_corrected <span style="color: #666666">/</span> (np<span style="color: #666666">.</span>sqrt(second_corrected <span style="color: #666666">+</span> delta))
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">reset</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>n_epochs <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>moment <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>second <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="usage-of-schedulers" class="anchor">Usage of schedulers </h3>
|
||
|
||
<p>To initalize a scheduler, simply create the object and pass in the necessary parameters such as the learning rate and the momentum as shown below. As the Scheduler class is an abstract class it should not called directly, and will raise an error upon usage.</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">momentum_scheduler <span style="color: #666666">=</span> Momentum(eta<span style="color: #666666">=1e-3</span>, momentum<span style="color: #666666">=0.9</span>)
|
||
adam_scheduler <span style="color: #666666">=</span> Adam(eta<span style="color: #666666">=1e-3</span>, rho<span style="color: #666666">=0.9</span>, rho2<span style="color: #666666">=0.999</span>)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>Here is a small example for how a segment of code using schedulers could look. Switching out the schedulers is simple.</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ones((<span style="color: #666666">3</span>,<span style="color: #666666">3</span>))
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Before scheduler:</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BB6688; font-weight: bold">{</span>weights<span style="color: #BB6688; font-weight: bold">=}</span><span style="color: #BA2121">"</span>)
|
||
|
||
epochs <span style="color: #666666">=</span> <span style="color: #666666">10</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> e <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(epochs):
|
||
gradient <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(<span style="color: #666666">3</span>, <span style="color: #666666">3</span>)
|
||
change <span style="color: #666666">=</span> adam_scheduler<span style="color: #666666">.</span>update_change(gradient)
|
||
weights <span style="color: #666666">=</span> weights <span style="color: #666666">-</span> change
|
||
adam_scheduler<span style="color: #666666">.</span>reset()
|
||
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">After scheduler:</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BB6688; font-weight: bold">{</span>weights<span style="color: #BB6688; font-weight: bold">=}</span><span style="color: #BA2121">"</span>)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="cost-functions" class="anchor">Cost functions </h3>
|
||
|
||
<p>In this section we will quickly look at cost functions that can be
|
||
used when creating the neural network. Every cost function takes the
|
||
target vector as its parameter, and returns a function valued only at
|
||
X such that it may easily be differentiated.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostOLS</span>(target):
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Return OLS function valued only at X, so</span>
|
||
<span style="color: #BA2121; font-style: italic"> that it may be easily differentiated</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">func</span>(X):
|
||
<span style="color: #008000; font-weight: bold">return</span> (<span style="color: #666666">1.0</span> <span style="color: #666666">/</span> target<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]) <span style="color: #666666">*</span> np<span style="color: #666666">.</span>sum((target <span style="color: #666666">-</span> X) <span style="color: #666666">**</span> <span style="color: #666666">2</span>)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> func
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostLogReg</span>(target):
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Return Logistic Regression cost function</span>
|
||
<span style="color: #BA2121; font-style: italic"> valued only at X, so that it may be easily differentiated</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">func</span>(X):
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">-</span>(<span style="color: #666666">1.0</span> <span style="color: #666666">/</span> target<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]) <span style="color: #666666">*</span> np<span style="color: #666666">.</span>sum(
|
||
(target <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(X <span style="color: #666666">+</span> <span style="color: #666666">10e-10</span>)) <span style="color: #666666">+</span> ((<span style="color: #666666">1</span> <span style="color: #666666">-</span> target) <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(<span style="color: #666666">1</span> <span style="color: #666666">-</span> X <span style="color: #666666">+</span> <span style="color: #666666">10e-10</span>))
|
||
)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> func
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">CostCrossEntropy</span>(target):
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Return cross entropy cost function valued only at X, so</span>
|
||
<span style="color: #BA2121; font-style: italic"> that it may be easily differentiated</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">func</span>(X):
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">-</span>(<span style="color: #666666">1.0</span> <span style="color: #666666">/</span> target<span style="color: #666666">.</span>size) <span style="color: #666666">*</span> np<span style="color: #666666">.</span>sum(target <span style="color: #666666">*</span> np<span style="color: #666666">.</span>log(X <span style="color: #666666">+</span> <span style="color: #666666">10e-10</span>))
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> func
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="usage-of-cost-functions" class="anchor">Usage of cost functions </h3>
|
||
|
||
<p>Below we will provide a short example of how these cost function may
|
||
be used to obtain results if you wish to test them out on your own
|
||
using AutoGrad's automatic differentiation.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> grad
|
||
|
||
target <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">1</span>, <span style="color: #666666">2</span>, <span style="color: #666666">3</span>]])<span style="color: #666666">.</span>T
|
||
a <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">4</span>, <span style="color: #666666">5</span>, <span style="color: #666666">6</span>]])<span style="color: #666666">.</span>T
|
||
|
||
cost_func <span style="color: #666666">=</span> CostCrossEntropy
|
||
cost_func_derivative <span style="color: #666666">=</span> grad(cost_func(target))
|
||
|
||
valued_at_a <span style="color: #666666">=</span> cost_func_derivative(a)
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Derivative of cost function </span><span style="color: #BB6688; font-weight: bold">{</span>cost_func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> valued at a:</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BB6688; font-weight: bold">{</span>valued_at_a<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="activation-functions" class="anchor">Activation functions </h3>
|
||
|
||
<p>Finally, before we look at the layers that make up the neural network,
|
||
we will look at the activation functions which can be specified
|
||
between the hidden layers and as the output function. Each function
|
||
can be valued for any given vector or matrix X, and can be
|
||
differentiated via derivate().
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> elementwise_grad
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">identity</span>(X):
|
||
<span style="color: #008000; font-weight: bold">return</span> X
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">sigmoid</span>(X):
|
||
<span style="color: #008000; font-weight: bold">try</span>:
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #666666">1.0</span> <span style="color: #666666">/</span> (<span style="color: #666666">1</span> <span style="color: #666666">+</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>X))
|
||
<span style="color: #008000; font-weight: bold">except</span> <span style="color: #D2413A; font-weight: bold">FloatingPointError</span>:
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>where(X <span style="color: #666666">></span> np<span style="color: #666666">.</span>zeros(X<span style="color: #666666">.</span>shape), np<span style="color: #666666">.</span>ones(X<span style="color: #666666">.</span>shape), np<span style="color: #666666">.</span>zeros(X<span style="color: #666666">.</span>shape))
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">softmax</span>(X):
|
||
X <span style="color: #666666">=</span> X <span style="color: #666666">-</span> np<span style="color: #666666">.</span>max(X, axis<span style="color: #666666">=-1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
||
delta <span style="color: #666666">=</span> <span style="color: #666666">10e-10</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>exp(X) <span style="color: #666666">/</span> (np<span style="color: #666666">.</span>sum(np<span style="color: #666666">.</span>exp(X), axis<span style="color: #666666">=-1</span>, keepdims<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>) <span style="color: #666666">+</span> delta)
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">RELU</span>(X):
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>where(X <span style="color: #666666">></span> np<span style="color: #666666">.</span>zeros(X<span style="color: #666666">.</span>shape), X, np<span style="color: #666666">.</span>zeros(X<span style="color: #666666">.</span>shape))
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">LRELU</span>(X):
|
||
delta <span style="color: #666666">=</span> <span style="color: #666666">10e-4</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>where(X <span style="color: #666666">></span> np<span style="color: #666666">.</span>zeros(X<span style="color: #666666">.</span>shape), X, delta <span style="color: #666666">*</span> X)
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">derivate</span>(func):
|
||
<span style="color: #008000; font-weight: bold">if</span> func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"RELU"</span>:
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">func</span>(X):
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>where(X <span style="color: #666666">></span> <span style="color: #666666">0</span>, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> func
|
||
|
||
<span style="color: #008000; font-weight: bold">elif</span> func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"LRELU"</span>:
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">func</span>(X):
|
||
delta <span style="color: #666666">=</span> <span style="color: #666666">10e-4</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>where(X <span style="color: #666666">></span> <span style="color: #666666">0</span>, <span style="color: #666666">1</span>, delta)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> func
|
||
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
<span style="color: #008000; font-weight: bold">return</span> elementwise_grad(func)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="usage-of-activation-functions" class="anchor">Usage of activation functions </h3>
|
||
|
||
<p>Below we present a short demonstration of how to use an activation
|
||
function. The derivative of the activation function will be important
|
||
when calculating the output delta term during backpropagation. Note
|
||
that derivate() can also be used for cost functions for a more
|
||
generalized approach.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">z <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">4</span>, <span style="color: #666666">5</span>, <span style="color: #666666">6</span>]])<span style="color: #666666">.</span>T
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"Input to activation function:</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BB6688; font-weight: bold">{</span>z<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
||
|
||
act_func <span style="color: #666666">=</span> sigmoid
|
||
a <span style="color: #666666">=</span> act_func(z)
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">Output from </span><span style="color: #BB6688; font-weight: bold">{</span>act_func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> activation function:</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BB6688; font-weight: bold">{</span>a<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
||
|
||
act_func_derivative <span style="color: #666666">=</span> derivate(act_func)
|
||
valued_at_z <span style="color: #666666">=</span> act_func_derivative(a)
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BA2121">Derivative of </span><span style="color: #BB6688; font-weight: bold">{</span>act_func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> activation function valued at z:</span><span style="color: #BB6622; font-weight: bold">\n</span><span style="color: #BB6688; font-weight: bold">{</span>valued_at_z<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="convolution" class="anchor">Convolution </h3>
|
||
|
||
<p>In order to construct a convolutional neural network (CNN), it is
|
||
crucial to comprehend the fundamental principles of convolution and
|
||
how it aids in extracting information from images. Convolution, at its
|
||
core, is merely a mathematical operation between two functions that
|
||
yields another function. It is represented by an integral between two
|
||
functions, which is typically expressed as:
|
||
</p>
|
||
|
||
$$
|
||
(f \ast g)(t):=\int_{-\infty}^{\infty} f(\tau) g(t-\tau) d \tau.
|
||
$$
|
||
|
||
<p>Here, f and g are the two functions on which we want to perform an
|
||
operation. The outcome of the convolution operation is represented by
|
||
\( (f \ast g) \), and it is derived by sliding the function g over f and
|
||
computing the integral of their product at each position. If both
|
||
functions are continuous, convolution takes the form shown
|
||
above. However, if we discretize both f and g, the convolution
|
||
operation will take the form of a sum between the elements of f and g:
|
||
</p>
|
||
$$
|
||
(f \ast g)[n]=\sum_{m=0}^{n-1} f[m] g[n-m].
|
||
$$
|
||
|
||
<p>The key idea we utilize to extract the information contained in an
|
||
image is to slide an \( m \times n \) matrix <em>g</em> over an \( m \times n \)
|
||
matrix <em>f</em>. In our case, <em>f</em> represents the image, while <em>g</em>
|
||
represents the kernel, oftentimes called a filter. However, since our
|
||
convolution will be a two-dimensional variant, we need to extend our
|
||
mathematical formula with an additional summation:
|
||
</p>
|
||
|
||
$$
|
||
(f \ast g)[i, j]\sum_{m=0}^{M-1}\sum_{n=0}^{N-1} f[m,n] g[i-m, j-n].
|
||
$$
|
||
|
||
<p>It is imperative to note that the size of the kernel g is
|
||
significantly smaller than the size of the input image f, thereby
|
||
reducing the amount of computation necessary for feature
|
||
extraction. Furthermore, the kernel is usually a trainable parameter
|
||
in a convolutional neural network, allowing the network to learn
|
||
appropriate kernels for specific tasks.
|
||
</p>
|
||
|
||
<p>To give you an example of how 2D convolution works in practice,
|
||
suppose we have an image <em>f</em> of dimension \( 6 \times 6 \)
|
||
</p>
|
||
|
||
$$
|
||
f = \begin{bmatrix}
|
||
4 & 1 & 2 & 9 & 8 & 6 \\
|
||
9 & 5 & 9 & 5 & 8 & 5 \\
|
||
1 & 5 & 9 & 7 & 6 & 4 \\
|
||
2 & 9 & 8 & 3 & 7 & 1 \\
|
||
8 & 1 & 6 & 4 & 2 & 2 \\
|
||
1 & 0 & 5 & 7 & 8 & 2 \\
|
||
\end{bmatrix}
|
||
$$
|
||
|
||
<p>and a \( 3 \times 3 \) kernel <em>g</em> called a low-pass filter. Note that the
|
||
kernel is usually rotated by 180 degrees during convolution, however
|
||
this has no effect on this kernel.
|
||
</p>
|
||
|
||
$$
|
||
g = \frac{1}{9}
|
||
\begin{bmatrix}
|
||
1 & 1 & 1 \\
|
||
1 & 1 & 1 \\
|
||
1 & 1 & 1 \\
|
||
\end{bmatrix}
|
||
$$
|
||
|
||
<p>In order to filter the image, we have to extract a \( 3 \times 3 \)
|
||
element from the upper left corner of <em>f</em>, and perform element-wise
|
||
multiplication of the extracted image pixels with the elements of the
|
||
kernel <em>g</em>:
|
||
</p>
|
||
|
||
$$
|
||
\begin{bmatrix}
|
||
4 & 1 & 2 \\
|
||
9 & 5 & 9 \\
|
||
1 & 5 & 9 \\
|
||
\end{bmatrix}
|
||
\cdot
|
||
\begin{bmatrix}
|
||
\frac{1}{9} & \frac{1}{9} & \frac{1}{9} \\
|
||
\frac{1}{9} & \frac{1}{9} & \frac{1}{9} \\
|
||
\frac{1}{9} & \frac{1}{9} & \frac{1}{9} \\
|
||
\end{bmatrix}
|
||
=
|
||
\begin{bmatrix}
|
||
\frac{4}{9} & \frac{1}{9} & \frac{2}{9} \\
|
||
\frac{9}{9} & \frac{5}{9} & \frac{9}{9} \\
|
||
\frac{1}{9} & \frac{5}{9} & \frac{9}{9} \\
|
||
\end {bmatrix}
|
||
= \textbf{A}
|
||
$$
|
||
|
||
<p>Then, following the multiplication, we summarize all the elements of the resulting matrix A:</p>
|
||
|
||
$$
|
||
(f \ast g)[0, 0]= \sum_{i=0}^{2} \sum_{j=0}^{2} a_{i,j} = 5
|
||
$$
|
||
|
||
<p>Which corresponds to the first element of the filtered image \( (f \ast g) \).</p>
|
||
|
||
<p>Here we use a stride of 1, a parameter denoted <em>s</em> which describes how
|
||
many indexes we move the kernel <em>g</em> to the right before repeating the
|
||
calculations above for the next \( 3 \times 3 \) element of the image
|
||
<em>f</em>. It is usually presumed that *s*=1, however, larger values for <em>s</em>
|
||
can be used to reduce the dimentionality of the filtered image such
|
||
that the convolution operation is more computationally efficient. In
|
||
the context of a convolutional neural network, this will become very
|
||
useful.
|
||
</p>
|
||
|
||
<p>The full result of the convolution is:</p>
|
||
|
||
$$
|
||
(f \ast g) =
|
||
\begin{bmatrix}
|
||
5 & 5.78 & 7 & 6.44 \\
|
||
6.33 & 6.67 & 6.89 & 5.11 \\
|
||
5.44 & 5.78 & 5.78 & 4 \\
|
||
4.44 & 4.78 & 5.56 & 4 \\
|
||
\end{bmatrix}
|
||
$$
|
||
|
||
<p>The result is markedly smaller in shape than the original image. This occurs when using convolution without first padding the image with additional columns and rows, allowing us to keep the original image shape after sliding the kernel over the image.
|
||
How many rows and columns we wish to pad the image with depends strictly on the shape of the kernel, as we wish to pad the image with <em>r</em> additional rows and <em>c</em> additional columns.
|
||
</p>
|
||
|
||
$$
|
||
r =\lfloor \frac{kernel\ height}{2} \rfloor \cdot 2 \\
|
||
c =\lfloor \frac{kernel\ width}{2} \rfloor \cdot 2
|
||
$$
|
||
|
||
<p>Note the notation \( \lfloor \frac{kernel width}{2} \rfloor \) means that
|
||
we floor the result of the division, meaning we round down to a whole
|
||
number in case \( \frac{kernel width}{2} \) results in a floating point
|
||
number.
|
||
</p>
|
||
|
||
<p>Using those simple equations, we find out by how much we have to
|
||
extend the dimensions of the original image. Before proceeding,
|
||
however, we might ask what we shall fill the additional rows and
|
||
columns with? One of the most common approaches to padding is
|
||
zero-padding, which as the name suggest, involves filling the rows and
|
||
columns with zeros. This is the approach that we will be using for
|
||
this demonstration. If we apply this padding to out original \( 6 \times 6 \)
|
||
image, the result will be an \( 8 \times 8 \) image as the kernel has a width and
|
||
height of 3. Note that the original image is encapsuled by the
|
||
zero-padded rows and columns:
|
||
</p>
|
||
|
||
$$
|
||
\begin{bmatrix}
|
||
0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 \\
|
||
0 & 4 & 1 & 2 & 9 & 8 & 6 & 0 \\
|
||
0 & 9 & 5 & 9 & 5 & 8 & 5 & 0 \\
|
||
0 & 1 & 5 & 9 & 7 & 6 & 4 & 0 \\
|
||
0 & 2 & 9 & 8 & 3 & 7 & 1 & 0 \\
|
||
0 & 8 & 1 & 6 & 4 & 2 & 2 & 0 \\
|
||
0 & 1 & 0 & 5 & 7 & 8 & 2 & 0 \\
|
||
0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 \\
|
||
\]
|
||
\end{bmatrix}
|
||
$$
|
||
|
||
<p>Below we have provided code that demonstrates padding and convolution. As you will see when we run the code, the size of the image will remain unchanged when using padding. </p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">padding</span>(image, kernel):
|
||
<span style="color: #408080; font-style: italic"># calculate r and c</span>
|
||
r <span style="color: #666666">=</span> (kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
c <span style="color: #666666">=</span> (kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># padded image dimensions</span>
|
||
padded_height <span style="color: #666666">=</span> image<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">+</span> r
|
||
padded_width <span style="color: #666666">=</span> image<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">+</span> c
|
||
|
||
<span style="color: #408080; font-style: italic"># for more readable code</span>
|
||
k_half_height <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
k_half_width <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># zero matrix with padded dimensions</span>
|
||
padded_img <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((padded_height, padded_width))
|
||
|
||
<span style="color: #408080; font-style: italic"># place image into zero matrix</span>
|
||
padded_img[k_half_height : padded_height <span style="color: #666666">-</span> k_half_height,
|
||
k_half_width : padded_width <span style="color: #666666">-</span> k_half_width] <span style="color: #666666">=</span> image[:, :]
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> padded_img
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">convolve</span>(original_image, padded_image, kernel, stride<span style="color: #666666">=1</span>):
|
||
<span style="color: #408080; font-style: italic"># rotate kernel by 180 degrees</span>
|
||
kernel <span style="color: #666666">=</span> np<span style="color: #666666">.</span>rot90(np<span style="color: #666666">.</span>rot90(kernel))
|
||
|
||
<span style="color: #408080; font-style: italic"># note that kernel height // 2 is written as 'm'</span>
|
||
<span style="color: #408080; font-style: italic"># and kernel width // 2 as 'n' in the mathematical notation</span>
|
||
m <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
n <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
|
||
r <span style="color: #666666">=</span> (kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
c <span style="color: #666666">=</span> (kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize output array</span>
|
||
convolved_image <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(original_image<span style="color: #666666">.</span>shape)
|
||
image_height <span style="color: #666666">=</span> original_image<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]
|
||
image_width <span style="color: #666666">=</span> original_image<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>]
|
||
|
||
<span style="color: #408080; font-style: italic"># the convolution</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(m, image_height <span style="color: #666666">+</span> m, stride):
|
||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(n, image_width <span style="color: #666666">+</span> n, stride):
|
||
convolved_image[i<span style="color: #666666">-</span>m, j<span style="color: #666666">-</span>n] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(
|
||
padded_image[i : i <span style="color: #666666">+</span> m, j : j <span style="color: #666666">+</span> n]
|
||
<span style="color: #666666">*</span> kernel
|
||
)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> convolved_image
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">convolve</span>(image, kernel, stride<span style="color: #666666">=1</span>):
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">2</span>):
|
||
kernel <span style="color: #666666">=</span> np<span style="color: #666666">.</span>rot90(kernel)
|
||
|
||
k_half_height <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
k_half_width <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
|
||
conv_image <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(image<span style="color: #666666">.</span>shape)
|
||
pad_image <span style="color: #666666">=</span> padding(image, kernel)
|
||
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(k_half_height, conv_image<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">+</span> k_half_height, stride):
|
||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(k_half_width, conv_image<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">+</span> k_half_width, stride):
|
||
conv_image[i <span style="color: #666666">-</span> k_half_height, j <span style="color: #666666">-</span> k_half_width] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(
|
||
pad_image[
|
||
i <span style="color: #666666">-</span> k_half_height : i <span style="color: #666666">+</span> k_half_height <span style="color: #666666">+</span> <span style="color: #666666">1</span>, j <span style="color: #666666">-</span> k_half_width : j <span style="color: #666666">+</span> k_half_width <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
||
]
|
||
<span style="color: #666666">*</span> kernel
|
||
)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> conv_image
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>Fun fact: When filtering images, you will see that convolution involves rotating the kernel by 180 degrees.
|
||
However, this is not the case when applying convolution in a CNN, where the same operation not rotated by 180 degrees is called
|
||
cross-correlation.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">original_image <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">4</span>, <span style="color: #666666">1</span>, <span style="color: #666666">2</span>, <span style="color: #666666">9</span>, <span style="color: #666666">8</span>, <span style="color: #666666">6</span>],
|
||
[<span style="color: #666666">9</span>, <span style="color: #666666">5</span>, <span style="color: #666666">9</span>, <span style="color: #666666">5</span>, <span style="color: #666666">8</span>, <span style="color: #666666">5</span>],
|
||
[<span style="color: #666666">1</span>, <span style="color: #666666">5</span>, <span style="color: #666666">9</span>, <span style="color: #666666">7</span>, <span style="color: #666666">6</span>, <span style="color: #666666">4</span>],
|
||
[<span style="color: #666666">2</span>, <span style="color: #666666">9</span>, <span style="color: #666666">8</span>, <span style="color: #666666">3</span>, <span style="color: #666666">7</span>, <span style="color: #666666">1</span>],
|
||
[<span style="color: #666666">8</span>, <span style="color: #666666">1</span>, <span style="color: #666666">6</span>, <span style="color: #666666">4</span>, <span style="color: #666666">2</span>, <span style="color: #666666">2</span>],
|
||
[<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">5</span>, <span style="color: #666666">7</span>, <span style="color: #666666">8</span>, <span style="color: #666666">2</span>]])
|
||
|
||
kernel <span style="color: #666666">=</span> (<span style="color: #666666">1/9</span>)<span style="color: #666666">*</span>np<span style="color: #666666">.</span>ones((<span style="color: #666666">3</span>,<span style="color: #666666">3</span>))
|
||
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"</span><span style="color: #BB6688; font-weight: bold">{</span>original_image<span style="color: #666666">.</span>shape<span style="color: #BB6688; font-weight: bold">=}</span><span style="color: #BA2121">"</span>)
|
||
|
||
<span style="color: #408080; font-style: italic"># note that convolve() performs padding</span>
|
||
convolved_image <span style="color: #666666">=</span> convolve(original_image, kernel, stride<span style="color: #666666">=1</span>)
|
||
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f"</span><span style="color: #BB6688; font-weight: bold">{</span>convolved_image<span style="color: #666666">.</span>shape<span style="color: #BB6688; font-weight: bold">=}</span><span style="color: #BA2121">"</span>)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>As you can see, the resulting image is of the same size as the
|
||
original image. To round of our demonstration of convolution, we will
|
||
present the results of convolution using commonly used kernels. In a
|
||
CNN, the values of the kernels are randomly initialized, and then
|
||
learned during training. These kernels will extract information
|
||
regarding the picture, such as for example the edge detection filter
|
||
demonstrated below extracts the edges present in the picture. Of
|
||
course, there is no guarantee that the CNN will learn an edge
|
||
detection filter, but this should provide some intuiton as to how the
|
||
CNN is able to use kernels to make better predictions than a regular
|
||
feed forward neural network.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #408080; font-style: italic"># Now an example using a real image and first a gaussian low-pass filter and then a sobel filter</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">imageio.v3</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">imageio</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">generate_gauss_mask</span>(sigma, K<span style="color: #666666">=1</span>):
|
||
side <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ceil(<span style="color: #666666">1</span> <span style="color: #666666">+</span> <span style="color: #666666">8</span> <span style="color: #666666">*</span> sigma)
|
||
y, x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mgrid[<span style="color: #666666">-</span>side <span style="color: #666666">//</span> <span style="color: #666666">2</span> <span style="color: #666666">+</span> <span style="color: #666666">1</span> : (side <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1</span>, <span style="color: #666666">-</span>side <span style="color: #666666">//</span> <span style="color: #666666">2</span> <span style="color: #666666">+</span> <span style="color: #666666">1</span> : (side <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1</span>]
|
||
ker_coef <span style="color: #666666">=</span> K <span style="color: #666666">/</span> (<span style="color: #666666">2</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>pi <span style="color: #666666">*</span> sigma<span style="color: #666666">**2</span>)
|
||
g <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>((x<span style="color: #666666">**2</span> <span style="color: #666666">+</span> y<span style="color: #666666">**2</span>) <span style="color: #666666">/</span> (<span style="color: #666666">2.0</span> <span style="color: #666666">*</span> sigma<span style="color: #666666">**2</span>)))
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> g, ker_coef
|
||
|
||
|
||
img_path <span style="color: #666666">=</span> <span style="color: #BA2121">"data/IMG-2167.JPG"</span>
|
||
image_of_cute_dog <span style="color: #666666">=</span> imageio<span style="color: #666666">.</span>imread(img_path, mode<span style="color: #666666">=</span><span style="color: #BA2121">'L'</span>)
|
||
|
||
plt<span style="color: #666666">.</span>imshow(image_of_cute_dog, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"gray"</span>, vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=255</span>, aspect<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Original image"</span>)
|
||
plt<span style="color: #666666">.</span>show()
|
||
|
||
gauss, kernel <span style="color: #666666">=</span> generate_gauss_mask(sigma<span style="color: #666666">=6</span>)
|
||
gauss_kernel <span style="color: #666666">=</span> gauss<span style="color: #666666">*</span>kernel
|
||
|
||
filtered_image <span style="color: #666666">=</span> convolve(image_of_cute_dog, gauss_kernel)
|
||
plt<span style="color: #666666">.</span>imshow(filtered_image, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"gray"</span>, vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=255</span>, aspect<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Result of convolution with gauss kernel (blurring filter)"</span>)
|
||
plt<span style="color: #666666">.</span>show()
|
||
|
||
sobel_kernel <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array([[<span style="color: #666666">1</span>, <span style="color: #666666">2</span>, <span style="color: #666666">1</span>],
|
||
[<span style="color: #666666">0</span>, <span style="color: #666666">0</span>, <span style="color: #666666">0</span>],
|
||
[<span style="color: #666666">-1</span>, <span style="color: #666666">-2</span>, <span style="color: #666666">-1</span>]])
|
||
|
||
filtered_image <span style="color: #666666">=</span> convolve(image_of_cute_dog, sobel_kernel)
|
||
|
||
plt<span style="color: #666666">.</span>imshow(filtered_image, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"gray"</span>, vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=255</span>, aspect<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Result of convolution with sobel kernel (edge detection filter)"</span>)
|
||
plt<span style="color: #666666">.</span>show()
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="layers" class="anchor">Layers </h3>
|
||
|
||
<p>The code below initialises global variables for readability and
|
||
describes the abstract class Layers. This is not important in order to
|
||
understand the CNN, but is benefitial for organizing the code neatly.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">math</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">copy</span> <span style="color: #008000; font-weight: bold">import</span> deepcopy, copy
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> grad
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">typing</span> <span style="color: #008000; font-weight: bold">import</span> Callable
|
||
|
||
<span style="color: #408080; font-style: italic"># global variables for index readability</span>
|
||
input_index <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
node_index <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
bias_index <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
input_channel_index <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
feature_maps_index <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
height_index <span style="color: #666666">=</span> <span style="color: #666666">2</span>
|
||
width_index <span style="color: #666666">=</span> <span style="color: #666666">3</span>
|
||
kernel_feature_maps_index <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
kernel_input_channels_index <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Layer</span>:
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, seed):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>seed <span style="color: #666666">=</span> seed
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000; font-weight: bold">raise</span> <span style="color: #D2413A; font-weight: bold">NotImplementedError</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000; font-weight: bold">raise</span> <span style="color: #D2413A; font-weight: bold">NotImplementedError</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_weights</span>(<span style="color: #008000">self</span>, previous_nodes):
|
||
<span style="color: #008000; font-weight: bold">raise</span> <span style="color: #D2413A; font-weight: bold">NotImplementedError</span>
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="convolution2dlayer-convolution-in-a-hidden-layer" class="anchor">Convolution2DLayer: convolution in a hidden layer </h3>
|
||
|
||
<p>After establishing the foundational understanding of applying
|
||
convolution to spatial data, let us delve into the intricate workings
|
||
of a convolutional layer in a Convolutional Neural Network (CNN). The
|
||
primary function of convolution, as previously discussed, is to
|
||
extract pertinent information from images while simultaneously
|
||
decreasing the scale of our data. To initiate the image processing, we
|
||
shall begin by partitioning the images into color channels (unless the
|
||
image is grayscale), comprising three primary colors: red, green, and
|
||
blue. We will subsequently utilize trainable kernels to construct a
|
||
higher-dimensional encoding of each channel called feature
|
||
maps. Successive layers will receive these feature maps as inputs,
|
||
generating further encodings, albeit with reduced dimensions. The term
|
||
trainable kernels denotes the initialization of pre-defined
|
||
kernel-shaped weights, which we will then train via backpropagation,
|
||
similar to how weights are trained in a Feedforward Neural Network.
|
||
</p>
|
||
|
||
<p>To ensure seamless integration between our implementation of the
|
||
convolutional layer and popular machine learning frameworks like
|
||
Tensorflow (Keras) and PyTorch, we have adopted a design pattern that
|
||
mirrors the construction of models using these APIs. This involves
|
||
implementing our convolutional layer as a Python class or object,
|
||
which allows for a more modular and flexible approach to building
|
||
neural networks. By structuring our code in this way, users can easily
|
||
incorporate our implementation into their existing machine learning
|
||
pipelines without having to make significant changes to their
|
||
codebase. Additionally, this design pattern promotes code reusability
|
||
and makes it easier to maintain and update our convolutional layer
|
||
implementation over time.
|
||
</p>
|
||
|
||
<p>Note that the Convolution2DLayer takes in an activation function as a parameter, as it also performs non-linearity.</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Convolution2DLayer</span>(Layer):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||
<span style="color: #008000">self</span>,
|
||
input_channels,
|
||
feature_maps,
|
||
kernel_height,
|
||
kernel_width,
|
||
v_stride,
|
||
h_stride,
|
||
pad,
|
||
act_func: Callable,
|
||
seed<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>,
|
||
reset_weights_independently<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,
|
||
):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(seed)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels <span style="color: #666666">=</span> input_channels
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>feature_maps <span style="color: #666666">=</span> feature_maps
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height <span style="color: #666666">=</span> kernel_height
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width <span style="color: #666666">=</span> kernel_width
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">=</span> v_stride
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">=</span> h_stride
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pad <span style="color: #666666">=</span> pad
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func <span style="color: #666666">=</span> act_func
|
||
|
||
<span style="color: #408080; font-style: italic"># such that the layer can be used on its own</span>
|
||
<span style="color: #408080; font-style: italic"># outside of the CNN module</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> reset_weights_independently <span style="color: #666666">==</span> <span style="color: #008000; font-weight: bold">True</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_reset_weights_independently()
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>, X_batch):
|
||
<span style="color: #408080; font-style: italic"># note that the shape of X_batch = [inputs, input_maps, img_height, img_width]</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># pad the input batch</span>
|
||
X_batch_padded <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_padding(X_batch)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate height_index and width_index after stride</span>
|
||
strided_height <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>ceil(X_batch<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride))
|
||
strided_width <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>ceil(X_batch<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride))
|
||
|
||
<span style="color: #408080; font-style: italic"># create output array</span>
|
||
output <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ndarray(
|
||
(
|
||
X_batch<span style="color: #666666">.</span>shape[input_index],
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>feature_maps,
|
||
strided_height,
|
||
strided_width,
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># save input and output for backpropagation</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward <span style="color: #666666">=</span> X_batch
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>output_shape <span style="color: #666666">=</span> output<span style="color: #666666">.</span>shape
|
||
|
||
<span style="color: #408080; font-style: italic"># checking for errors, no need to look here :)</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_check_for_errors()
|
||
|
||
<span style="color: #408080; font-style: italic"># convolve input with kernel</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> img <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(X_batch<span style="color: #666666">.</span>shape[input_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> chin <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels):
|
||
<span style="color: #008000; font-weight: bold">for</span> fmap <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>feature_maps):
|
||
out_h <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> h <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">0</span>, X_batch<span style="color: #666666">.</span>shape[height_index], <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride):
|
||
out_w <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">0</span>, X_batch<span style="color: #666666">.</span>shape[width_index], <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride):
|
||
output[img, fmap, out_h, out_w] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(
|
||
X_batch_padded[
|
||
img,
|
||
chin,
|
||
h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
]
|
||
<span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel[chin, fmap, :, :]
|
||
)
|
||
out_w <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||
out_h <span style="color: #666666">+=</span> <span style="color: #666666">1</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># Pay attention to the fact that we're not rotating the kernel by 180 degrees when filtering the image in</span>
|
||
<span style="color: #408080; font-style: italic"># the convolutional layer, as convolution in terms of Machine Learning is a procedure known as cross-correlation</span>
|
||
<span style="color: #408080; font-style: italic"># in image processing and signal processing</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># return a</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func(output <span style="color: #666666">/</span> (<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height))
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>, delta_term_next):
|
||
<span style="color: #408080; font-style: italic"># intiate matrices</span>
|
||
delta_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape))
|
||
gradient_kernel <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel<span style="color: #666666">.</span>shape))
|
||
|
||
<span style="color: #408080; font-style: italic"># pad input for convolution</span>
|
||
X_batch_padded <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_padding(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward)
|
||
|
||
<span style="color: #408080; font-style: italic"># Since an activation function is used at the output of the convolution layer, its derivative</span>
|
||
<span style="color: #408080; font-style: italic"># has to be accounted for in the backpropagation -> as if ReLU was a layer on its own.</span>
|
||
act_derivative <span style="color: #666666">=</span> derivate(<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func)
|
||
delta_term_next <span style="color: #666666">=</span> act_derivative(delta_term_next)
|
||
|
||
<span style="color: #408080; font-style: italic"># fill in 0's for values removed by vertical stride in feedforward</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">></span> <span style="color: #666666">1</span>:
|
||
v_ind <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(delta_term_next<span style="color: #666666">.</span>shape[height_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">-</span> <span style="color: #666666">1</span>):
|
||
delta_term_next <span style="color: #666666">=</span> np<span style="color: #666666">.</span>insert(
|
||
delta_term_next, v_ind, <span style="color: #666666">0</span>, axis<span style="color: #666666">=</span>height_index
|
||
)
|
||
v_ind <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride
|
||
|
||
<span style="color: #408080; font-style: italic"># fill in 0's for values removed by horizontal stride in feedforward</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">></span> <span style="color: #666666">1</span>:
|
||
h_ind <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(delta_term_next<span style="color: #666666">.</span>shape[width_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">-</span> <span style="color: #666666">1</span>):
|
||
delta_term_next <span style="color: #666666">=</span> np<span style="color: #666666">.</span>insert(
|
||
delta_term_next, h_ind, <span style="color: #666666">0</span>, axis<span style="color: #666666">=</span>width_index
|
||
)
|
||
h_ind <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride
|
||
|
||
<span style="color: #408080; font-style: italic"># crops out 0-rows and 0-columns</span>
|
||
delta_term_next <span style="color: #666666">=</span> delta_term_next[
|
||
:,
|
||
:,
|
||
: <span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[height_index],
|
||
: <span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[width_index],
|
||
]
|
||
|
||
<span style="color: #408080; font-style: italic"># the gradient received from the next layer also needs to be padded</span>
|
||
delta_term_next <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_padding(delta_term_next)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate delta term by convolving next delta term with kernel</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> img <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> chin <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels):
|
||
<span style="color: #008000; font-weight: bold">for</span> fmap <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>feature_maps):
|
||
<span style="color: #008000; font-weight: bold">for</span> h <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[height_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[width_index]):
|
||
delta_term[img, chin, h, w] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(
|
||
delta_term_next[
|
||
img,
|
||
fmap,
|
||
h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
]
|
||
<span style="color: #666666">*</span> np<span style="color: #666666">.</span>rot90(np<span style="color: #666666">.</span>rot90(<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel[chin, fmap, :, :]))
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate gradient for kernel for weight update</span>
|
||
<span style="color: #408080; font-style: italic"># also via convolution</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> chin <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels):
|
||
<span style="color: #008000; font-weight: bold">for</span> fmap <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>feature_maps):
|
||
<span style="color: #008000; font-weight: bold">for</span> k_x <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height):
|
||
<span style="color: #008000; font-weight: bold">for</span> k_y <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width):
|
||
gradient_kernel[chin, fmap, k_x, k_y] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(
|
||
X_batch_padded[
|
||
img,
|
||
chin,
|
||
h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
]
|
||
<span style="color: #666666">*</span> delta_term_next[
|
||
img,
|
||
fmap,
|
||
h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
]
|
||
)
|
||
<span style="color: #408080; font-style: italic"># all kernels are updated with weight gradient of kernel</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel <span style="color: #666666">-=</span> gradient_kernel
|
||
|
||
<span style="color: #408080; font-style: italic"># return delta term</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> delta_term
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_padding</span>(<span style="color: #008000">self</span>, X_batch, batch_type<span style="color: #666666">=</span><span style="color: #BA2121">"image"</span>):
|
||
|
||
<span style="color: #408080; font-style: italic"># same padding for images</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pad <span style="color: #666666">==</span> <span style="color: #BA2121">"same"</span> <span style="color: #AA22FF; font-weight: bold">and</span> batch_type <span style="color: #666666">==</span> <span style="color: #BA2121">"image"</span>:
|
||
padded_height <span style="color: #666666">=</span> X_batch<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">+</span> (<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
padded_width <span style="color: #666666">=</span> X_batch<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">+</span> (<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
half_kernel_height <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
half_kernel_width <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize padded array</span>
|
||
X_batch_padded <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ndarray(
|
||
(
|
||
X_batch<span style="color: #666666">.</span>shape[input_index],
|
||
X_batch<span style="color: #666666">.</span>shape[feature_maps_index],
|
||
padded_height,
|
||
padded_width,
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># zero pad all images in X_batch</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> img <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(X_batch<span style="color: #666666">.</span>shape[input_index]):
|
||
padded_img <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(
|
||
(X_batch<span style="color: #666666">.</span>shape[feature_maps_index], padded_height, padded_width)
|
||
)
|
||
padded_img[
|
||
:,
|
||
half_kernel_height : padded_height <span style="color: #666666">-</span> half_kernel_height,
|
||
half_kernel_width : padded_width <span style="color: #666666">-</span> half_kernel_width,
|
||
] <span style="color: #666666">=</span> X_batch[img, :, :, :]
|
||
X_batch_padded[img, :, :, :] <span style="color: #666666">=</span> padded_img[:, :, :]
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> X_batch_padded
|
||
|
||
<span style="color: #408080; font-style: italic"># same padding for gradients</span>
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pad <span style="color: #666666">==</span> <span style="color: #BA2121">"same"</span> <span style="color: #AA22FF; font-weight: bold">and</span> batch_type <span style="color: #666666">==</span> <span style="color: #BA2121">"grad"</span>:
|
||
padded_height <span style="color: #666666">=</span> X_batch<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">+</span> (<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
padded_width <span style="color: #666666">=</span> X_batch<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">+</span> (<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width <span style="color: #666666">//</span> <span style="color: #666666">2</span>) <span style="color: #666666">*</span> <span style="color: #666666">2</span>
|
||
half_kernel_height <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
half_kernel_width <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize padded array</span>
|
||
delta_term_padded <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(
|
||
(
|
||
X_batch<span style="color: #666666">.</span>shape[input_index],
|
||
X_batch<span style="color: #666666">.</span>shape[feature_maps_index],
|
||
padded_height,
|
||
padded_width,
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># zero pad delta term</span>
|
||
delta_term_padded[
|
||
:, :, : X_batch<span style="color: #666666">.</span>shape[height_index], : X_batch<span style="color: #666666">.</span>shape[width_index]
|
||
] <span style="color: #666666">=</span> X_batch[:, :, :, :]
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> delta_term_padded
|
||
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
<span style="color: #008000; font-weight: bold">return</span> X_batch
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_weights_independently</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># sets seed to remove randomness inbetween runs</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>seed <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #008000">self</span><span style="color: #666666">.</span>seed)
|
||
|
||
<span style="color: #408080; font-style: italic"># initializes kernel matrix</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ndarray(
|
||
(
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels,
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>feature_maps,
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># randomly initializes weights</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> chin <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel<span style="color: #666666">.</span>shape[kernel_input_channels_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> fmap <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel<span style="color: #666666">.</span>shape[kernel_feature_maps_index]):
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel[chin, fmap, :, :] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height, <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width
|
||
)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_weights</span>(<span style="color: #008000">self</span>, previous_nodes):
|
||
<span style="color: #408080; font-style: italic"># sets weights</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_reset_weights_independently()
|
||
|
||
<span style="color: #408080; font-style: italic"># returns shape of output used for subsequent layer's weight initiation</span>
|
||
strided_height <span style="color: #666666">=</span> <span style="color: #008000">int</span>(
|
||
np<span style="color: #666666">.</span>ceil(previous_nodes<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride)
|
||
)
|
||
strided_width <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>ceil(previous_nodes<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride))
|
||
next_nodes <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ones(
|
||
(
|
||
previous_nodes<span style="color: #666666">.</span>shape[input_index],
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>feature_maps,
|
||
strided_height,
|
||
strided_width,
|
||
)
|
||
)
|
||
<span style="color: #008000; font-weight: bold">return</span> next_nodes <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_check_for_errors</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_channel_index] <span style="color: #666666">!=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels:
|
||
<span style="color: #008000; font-weight: bold">raise</span> <span style="color: #D2413A; font-weight: bold">AssertionError</span>(
|
||
<span style="color: #BA2121">f"ERROR: Number of input channels in data (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_channel_index]<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">) is not equal to input channels in Convolution2DLayerOPT (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">)! Please change the number of input channels of the Convolution2DLayer such that they are equal"</span>
|
||
)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="backpropagation-in-the-convolutional-layer" class="anchor">Backpropagation in the convolutional layer </h3>
|
||
|
||
<p>As you may have noticed, we have not yet explained how the
|
||
backpropagation algorithm works in a convolutional layer. However,
|
||
having covered all other major details about convolutional layers, we
|
||
are now prepared to do so. It should come as no surprise that the
|
||
calculation of delta terms at each convolutional layer takes the form
|
||
of convolution. After the gradient has been propagated backwards
|
||
through the flattening layer, where it was reshaped into an
|
||
appropriate form, calculating the update value for the kernel is
|
||
simply a matter of convolving the output gradient with the input of
|
||
the layer for which we are updating the weights. For more detail, this
|
||
article serves as an excellent resource, see
|
||
<a href="https://pavisj.medium.com/convolutions-and-backpropagations-46026a8f5d2c" target="_self"><tt>https://pavisj.medium.com/convolutions-and-backpropagations-46026a8f5d2c</tt></a>
|
||
</p>
|
||
<h3 id="demonstration" class="anchor">Demonstration </h3>
|
||
|
||
<p>We can use the convolutional layer above to perform a simple convolution on an image of the now familiar cute dog.</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">imageio.v3</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">imageio</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">plot_convolution_result</span>(X, layer):
|
||
plt<span style="color: #666666">.</span>imshow(X[<span style="color: #666666">0</span>, <span style="color: #666666">0</span>, :, :], vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=255</span>, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"gray"</span>)
|
||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Original image"</span>)
|
||
plt<span style="color: #666666">.</span>colorbar()
|
||
plt<span style="color: #666666">.</span>show()
|
||
conv_result <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_feedforward(X)
|
||
plt<span style="color: #666666">.</span>title(<span style="color: #BA2121">"Result of convolutional layer"</span>)
|
||
plt<span style="color: #666666">.</span>imshow(conv_result[<span style="color: #666666">0</span>, <span style="color: #666666">0</span>, :, :], vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=255</span>, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"gray"</span>)
|
||
plt<span style="color: #666666">.</span>colorbar()
|
||
plt<span style="color: #666666">.</span>show()
|
||
|
||
<span style="color: #408080; font-style: italic"># create layer</span>
|
||
layer <span style="color: #666666">=</span> Convolution2DLayer(
|
||
input_channels<span style="color: #666666">=3</span>,
|
||
feature_maps<span style="color: #666666">=1</span>,
|
||
kernel_height<span style="color: #666666">=4</span>,
|
||
kernel_width<span style="color: #666666">=4</span>,
|
||
v_stride<span style="color: #666666">=2</span>,
|
||
h_stride<span style="color: #666666">=2</span>,
|
||
pad<span style="color: #666666">=</span><span style="color: #BA2121">"same"</span>,
|
||
act_func<span style="color: #666666">=</span>identity,
|
||
seed<span style="color: #666666">=2023</span>,
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># read in image path, make data correct format</span>
|
||
img_path <span style="color: #666666">=</span> img_path <span style="color: #666666">=</span> <span style="color: #BA2121">"data/IMG-2167.JPG"</span>
|
||
image_of_cute_dog <span style="color: #666666">=</span> imageio<span style="color: #666666">.</span>imread(img_path)
|
||
image_shape <span style="color: #666666">=</span> image_of_cute_dog<span style="color: #666666">.</span>shape
|
||
image_of_cute_dog <span style="color: #666666">=</span> image_of_cute_dog<span style="color: #666666">.</span>reshape(<span style="color: #666666">1</span>, image_shape[<span style="color: #666666">0</span>], image_shape[<span style="color: #666666">1</span>], image_shape[<span style="color: #666666">2</span>])
|
||
image_of_cute_dog <span style="color: #666666">=</span> image_of_cute_dog<span style="color: #666666">.</span>transpose(<span style="color: #666666">0</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1</span>, <span style="color: #666666">2</span>)
|
||
|
||
<span style="color: #408080; font-style: italic"># plot the result of the convolution</span>
|
||
plot_convolution_result(image_of_cute_dog, layer)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>We cobserve that the result has half the pixels on each axis due to
|
||
the fact that we've used a horizontal and vertical stride of 2. The
|
||
result of this convolution is not very insightfull, as the kernel has
|
||
completely random values for the first feedforward pass. However, as
|
||
we perform multiple forward and backward passes, the results of the
|
||
convolution should provide identifying features of the image it uses
|
||
for classification.
|
||
</p>
|
||
|
||
<p>Note that image data usually comes in many different shapes and sizes,
|
||
but for our CNN we require the input data be formatted as \[Number of
|
||
inputs, input channels, input height, input width\]. Occasionally, the
|
||
data you come accross use will be formatted like this, but on many
|
||
occasions reshaping and transposing the dimensions is sadly necessary.
|
||
</p>
|
||
<h3 id="pooling-layer" class="anchor">Pooling Layer </h3>
|
||
|
||
<p>The pooling layer is another widely used type of layer in
|
||
convolutional neural networks that enables data downsampling to a more
|
||
manageable size. Despite recent technological advancements that allow
|
||
for convolution without excessive size reduction of the data, the
|
||
pooling layer still remains a fundamental component of convolutional
|
||
neural networks. It can be used before, after, or in between
|
||
convolutional layers, although finding the optimal placement of layers
|
||
and network depth requires experimentation to achieve the best
|
||
performance for a given problem. The code we provide allows you to
|
||
perform two types of pooling known as max pooling and average pooling.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Pooling2DLayer</span>(Layer):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||
<span style="color: #008000">self</span>,
|
||
kernel_height,
|
||
kernel_width,
|
||
v_stride,
|
||
h_stride,
|
||
pooling<span style="color: #666666">=</span><span style="color: #BA2121">"max"</span>,
|
||
seed<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>,
|
||
):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(seed)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height <span style="color: #666666">=</span> kernel_height
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width <span style="color: #666666">=</span> kernel_width
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">=</span> v_stride
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">=</span> h_stride
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pooling <span style="color: #666666">=</span> pooling
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>, X_batch):
|
||
<span style="color: #408080; font-style: italic"># Saving the input for use in the backwardpass</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward <span style="color: #666666">=</span> X_batch
|
||
|
||
<span style="color: #408080; font-style: italic"># check if user is silly</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_check_for_errors()
|
||
|
||
<span style="color: #408080; font-style: italic"># Computing the size of the feature maps based on kernel size and the stride parameter</span>
|
||
strided_height <span style="color: #666666">=</span> (
|
||
X_batch<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height
|
||
) <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> X_batch<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">==</span> X_batch<span style="color: #666666">.</span>shape[width_index]:
|
||
strided_width <span style="color: #666666">=</span> strided_height
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
strided_width <span style="color: #666666">=</span> (
|
||
X_batch<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width
|
||
) <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize output array</span>
|
||
output <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ndarray(
|
||
(
|
||
X_batch<span style="color: #666666">.</span>shape[input_index],
|
||
X_batch<span style="color: #666666">.</span>shape[feature_maps_index],
|
||
strided_height,
|
||
strided_width,
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># select pooling action, either max or average pooling</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pooling <span style="color: #666666">==</span> <span style="color: #BA2121">"max"</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pooling_action <span style="color: #666666">=</span> np<span style="color: #666666">.</span>max
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pooling <span style="color: #666666">==</span> <span style="color: #BA2121">"average"</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pooling_action <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean
|
||
|
||
<span style="color: #408080; font-style: italic"># pool based on kernel size and stride</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> img <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(output<span style="color: #666666">.</span>shape[input_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> fmap <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(output<span style="color: #666666">.</span>shape[feature_maps_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> h <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(strided_height):
|
||
<span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(strided_width):
|
||
output[img, fmap, h, w] <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pooling_action(
|
||
X_batch[
|
||
img,
|
||
fmap,
|
||
(h <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride) : (h <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride)
|
||
<span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
(w <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride) : (w <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride)
|
||
<span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
]
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># output for feedforward in next layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> output
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>, delta_term_next):
|
||
<span style="color: #408080; font-style: italic"># initiate delta term array</span>
|
||
delta_term <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape))
|
||
|
||
<span style="color: #008000; font-weight: bold">for</span> img <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(delta_term_next<span style="color: #666666">.</span>shape[input_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> fmap <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(delta_term_next<span style="color: #666666">.</span>shape[feature_maps_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> h <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">0</span>, delta_term_next<span style="color: #666666">.</span>shape[height_index], <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride):
|
||
<span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(
|
||
<span style="color: #666666">0</span>, delta_term_next<span style="color: #666666">.</span>shape[width_index], <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride
|
||
):
|
||
<span style="color: #408080; font-style: italic"># max pooling</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pooling <span style="color: #666666">==</span> <span style="color: #BA2121">"max"</span>:
|
||
<span style="color: #408080; font-style: italic"># get window</span>
|
||
window <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward[
|
||
img,
|
||
fmap,
|
||
h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
]
|
||
|
||
<span style="color: #408080; font-style: italic"># find max values indices in window</span>
|
||
max_h, max_w <span style="color: #666666">=</span> np<span style="color: #666666">.</span>unravel_index(
|
||
window<span style="color: #666666">.</span>argmax(), window<span style="color: #666666">.</span>shape
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># set values in new, upsampled delta term</span>
|
||
delta_term[
|
||
img,
|
||
fmap,
|
||
(h <span style="color: #666666">+</span> max_h),
|
||
(w <span style="color: #666666">+</span> max_w),
|
||
] <span style="color: #666666">+=</span> delta_term_next[img, fmap, h, w]
|
||
|
||
<span style="color: #408080; font-style: italic"># average pooling</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pooling <span style="color: #666666">==</span> <span style="color: #BA2121">"average"</span>:
|
||
delta_term[
|
||
img,
|
||
fmap,
|
||
h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
] <span style="color: #666666">=</span> (
|
||
delta_term_next[img, fmap, h, w]
|
||
<span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height
|
||
<span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width
|
||
)
|
||
<span style="color: #408080; font-style: italic"># returns input to backpropagation in previous layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> delta_term
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_weights</span>(<span style="color: #008000">self</span>, previous_nodes):
|
||
<span style="color: #408080; font-style: italic"># calculate strided height, strided width</span>
|
||
strided_height <span style="color: #666666">=</span> (
|
||
previous_nodes<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height
|
||
) <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> previous_nodes<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">==</span> previous_nodes<span style="color: #666666">.</span>shape[width_index]:
|
||
strided_width <span style="color: #666666">=</span> strided_height
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
strided_width <span style="color: #666666">=</span> (
|
||
previous_nodes<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">-</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width
|
||
) <span style="color: #666666">//</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># initiate output array</span>
|
||
output <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ones(
|
||
(
|
||
previous_nodes<span style="color: #666666">.</span>shape[input_index],
|
||
previous_nodes<span style="color: #666666">.</span>shape[feature_maps_index],
|
||
strided_height,
|
||
strided_width,
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># returns output with shape used for reset weights in next layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> output
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_check_for_errors</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># check if input is smaller than kernel size -> error</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> (
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">>=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width
|
||
), <span style="color: #BA2121">f"ERROR: Pooling kernel width_index (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">) larger than data width_index (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>input<span style="color: #666666">.</span>shape[<span style="color: #666666">2</span>]<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">), please lower the kernel width_index of the Pooling2DLayer"</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> (
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">>=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height
|
||
), <span style="color: #BA2121">f"ERROR: Pooling kernel height_index (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">) larger than data height_index (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>input<span style="color: #666666">.</span>shape[<span style="color: #666666">3</span>]<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">), please lower the kernel height_index of the Pooling2DLayer"</span>
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="flattening-layer" class="anchor">Flattening Layer </h3>
|
||
|
||
<p>Before we can begin building our first CNN model, we need to introduce
|
||
the flattening layer. As its name suggests, the flattening layer
|
||
transforms the data into a one-dimensional vector that can be fed into
|
||
the feedforward layers of our network. This layer plays a crucial role
|
||
in preparing the data for further processing in the
|
||
network. Additionally, the flattening layer is responsible for
|
||
reshaping the gradient to the proper shape during
|
||
backpropagation. This ensures that the kernels are correctly updated,
|
||
allowing for effective learning in the network.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">FlattenLayer</span>(Layer):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(<span style="color: #008000">self</span>, act_func<span style="color: #666666">=</span>LRELU, seed<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(seed)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func <span style="color: #666666">=</span> act_func
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>, X_batch):
|
||
<span style="color: #408080; font-style: italic"># save input for backpropagation</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward_shape <span style="color: #666666">=</span> X_batch<span style="color: #666666">.</span>shape
|
||
<span style="color: #408080; font-style: italic"># Remember, the data has the following shape: (I, FM, H, W, ) in the convolutional layers</span>
|
||
<span style="color: #408080; font-style: italic"># whilst the data has the shape (I, FM * H * W) in the fully connected layers</span>
|
||
<span style="color: #408080; font-style: italic"># I = Inputs, FM = Feature Maps, H = Height and W = Width.</span>
|
||
X_batch <span style="color: #666666">=</span> X_batch<span style="color: #666666">.</span>reshape(
|
||
X_batch<span style="color: #666666">.</span>shape[input_index],
|
||
X_batch<span style="color: #666666">.</span>shape[feature_maps_index]
|
||
<span style="color: #666666">*</span> X_batch<span style="color: #666666">.</span>shape[height_index]
|
||
<span style="color: #666666">*</span> X_batch<span style="color: #666666">.</span>shape[width_index],
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># add bias to a</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix <span style="color: #666666">=</span> X_batch
|
||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ones((X_batch<span style="color: #666666">.</span>shape[input_index], <span style="color: #666666">1</span>)) <span style="color: #666666">*</span> <span style="color: #666666">0.01</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>hstack([bias, X_batch])
|
||
|
||
<span style="color: #408080; font-style: italic"># return a, the input to feedforward in next layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>, weights_next, delta_term_next):
|
||
activation_derivative <span style="color: #666666">=</span> derivate(<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate delta term</span>
|
||
delta_term <span style="color: #666666">=</span> (
|
||
weights_next[bias_index:, :] <span style="color: #666666">@</span> delta_term_next<span style="color: #666666">.</span>T
|
||
)<span style="color: #666666">.</span>T <span style="color: #666666">*</span> activation_derivative(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix)
|
||
|
||
<span style="color: #408080; font-style: italic"># FlattenLayer does not update weights</span>
|
||
<span style="color: #408080; font-style: italic"># reshapes delta layer to convolutional layer data format [Input, Feature_Maps, Height, Width]</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> delta_term<span style="color: #666666">.</span>reshape(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward_shape)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_weights</span>(<span style="color: #008000">self</span>, previous_nodes):
|
||
<span style="color: #408080; font-style: italic"># note that the previous nodes to the FlattenLayer are from the convolutional layers</span>
|
||
previous_nodes <span style="color: #666666">=</span> previous_nodes<span style="color: #666666">.</span>reshape(
|
||
previous_nodes<span style="color: #666666">.</span>shape[input_index],
|
||
previous_nodes<span style="color: #666666">.</span>shape[feature_maps_index]
|
||
<span style="color: #666666">*</span> previous_nodes<span style="color: #666666">.</span>shape[height_index]
|
||
<span style="color: #666666">*</span> previous_nodes<span style="color: #666666">.</span>shape[width_index],
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># return shape used in reset_weights in next layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> previous_nodes<span style="color: #666666">.</span>shape[node_index]
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_prev_a</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="fully-connected-layers" class="anchor">Fully Connected Layers </h3>
|
||
|
||
<p>Finally, the result from the flatten layer will pass to a series of
|
||
fully connected layers, which function as a normal feed forward neural
|
||
network. The fully connected layers are split into two classes;
|
||
FullyConnectedLayer which acts as a hidden layer, and OutputLayer,
|
||
which acts as the single output layer at the end of the CNN. If one
|
||
wishes to use this codebase to construct a normal feed forward neural
|
||
network, it must start with a FlattenLayer due to techincal details
|
||
regarding weight intitialization. However many FullyConnectedLayers
|
||
can be added to the CNN, and in each layer the amount of nodes, which
|
||
activation function and scheduler to use can be specified. In
|
||
practice, the scheduler will be specified in the CNN object
|
||
initialization, and inherited if no other scheduler is specified.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">FullyConnectedLayer</span>(Layer):
|
||
<span style="color: #408080; font-style: italic"># FullyConnectedLayer per default uses LRELU and Adam scheduler</span>
|
||
<span style="color: #408080; font-style: italic"># with an eta of 0.0001, rho of 0.9 and rho2 of 0.999</span>
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||
<span style="color: #008000">self</span>,
|
||
nodes: <span style="color: #008000">int</span>,
|
||
act_func: Callable <span style="color: #666666">=</span> LRELU,
|
||
scheduler: Scheduler <span style="color: #666666">=</span> Adam(eta<span style="color: #666666">=1e-4</span>, rho<span style="color: #666666">=0.9</span>, rho2<span style="color: #666666">=0.999</span>),
|
||
seed: <span style="color: #008000">int</span> <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>,
|
||
):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(seed)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>nodes <span style="color: #666666">=</span> nodes
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func <span style="color: #666666">=</span> act_func
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_weight <span style="color: #666666">=</span> copy(scheduler)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_bias <span style="color: #666666">=</span> copy(scheduler)
|
||
|
||
<span style="color: #408080; font-style: italic"># initiate matrices for later</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>, X_batch):
|
||
<span style="color: #408080; font-style: italic"># calculate z</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix <span style="color: #666666">=</span> X_batch <span style="color: #666666">@</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate a, add bias</span>
|
||
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>ones((X_batch<span style="color: #666666">.</span>shape[input_index], <span style="color: #666666">1</span>)) <span style="color: #666666">*</span> <span style="color: #666666">0.01</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>hstack([bias, <span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix])
|
||
|
||
<span style="color: #408080; font-style: italic"># return a, the input for feedforward in next layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>, weights_next, delta_term_next, a_previous, lam):
|
||
<span style="color: #408080; font-style: italic"># take the derivative of the activation function</span>
|
||
activation_derivative <span style="color: #666666">=</span> derivate(<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate the delta term</span>
|
||
delta_term <span style="color: #666666">=</span> (
|
||
weights_next[bias_index:, :] <span style="color: #666666">@</span> delta_term_next<span style="color: #666666">.</span>T
|
||
)<span style="color: #666666">.</span>T <span style="color: #666666">*</span> activation_derivative(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix)
|
||
|
||
<span style="color: #408080; font-style: italic"># intitiate matrix to store gradient</span>
|
||
<span style="color: #408080; font-style: italic"># note that we exclude the bias term, which we will calculate later</span>
|
||
gradient_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(
|
||
(
|
||
a_previous<span style="color: #666666">.</span>shape[input_index],
|
||
a_previous<span style="color: #666666">.</span>shape[node_index] <span style="color: #666666">-</span> bias_index,
|
||
delta_term<span style="color: #666666">.</span>shape[node_index],
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate gradient = delta term * previous a</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(delta_term)):
|
||
gradient_weights[i, :, :] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>outer(
|
||
a_previous[i, bias_index:], delta_term[i, :]
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># sum the gradient, divide by input_index</span>
|
||
gradient_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(gradient_weights, axis<span style="color: #666666">=</span>input_index)
|
||
<span style="color: #408080; font-style: italic"># for the bias gradient we do not multiply by previous a</span>
|
||
gradient_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(delta_term, axis<span style="color: #666666">=</span>input_index)<span style="color: #666666">.</span>reshape(
|
||
<span style="color: #666666">1</span>, delta_term<span style="color: #666666">.</span>shape[node_index]
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># regularization term</span>
|
||
gradient_weights <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights[bias_index:, :] <span style="color: #666666">*</span> lam
|
||
|
||
<span style="color: #408080; font-style: italic"># send gradients into scheduler</span>
|
||
<span style="color: #408080; font-style: italic"># returns update matrix which will be used to update the weights and bias</span>
|
||
update_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>vstack(
|
||
[
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_bias<span style="color: #666666">.</span>update_change(gradient_bias),
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_weight<span style="color: #666666">.</span>update_change(gradient_weights),
|
||
]
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># update weights</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">-=</span> update_matrix
|
||
|
||
<span style="color: #408080; font-style: italic"># return weights and delta term, input for backpropagation in previous layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights, delta_term
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_weights</span>(<span style="color: #008000">self</span>, previous_nodes):
|
||
<span style="color: #408080; font-style: italic"># sets seed to remove randomness inbetween runs</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>seed <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #008000">self</span><span style="color: #666666">.</span>seed)
|
||
|
||
<span style="color: #408080; font-style: italic"># add bias, initiate random weights</span>
|
||
bias <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>randn(previous_nodes <span style="color: #666666">+</span> bias, <span style="color: #008000">self</span><span style="color: #666666">.</span>nodes)
|
||
|
||
<span style="color: #408080; font-style: italic"># returns number of nodes, used for reset_weights in next layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>nodes
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_scheduler</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># resets scheduler per epoch</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_weight<span style="color: #666666">.</span>reset()
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_bias<span style="color: #666666">.</span>reset()
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_prev_a</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># returns a matrix, used in backpropagation</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">OutputLayer</span>(FullyConnectedLayer):
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||
<span style="color: #008000">self</span>,
|
||
nodes: <span style="color: #008000">int</span>,
|
||
output_func: Callable <span style="color: #666666">=</span> LRELU,
|
||
cost_func: Callable <span style="color: #666666">=</span> CostCrossEntropy,
|
||
scheduler: Scheduler <span style="color: #666666">=</span> Adam(eta<span style="color: #666666">=1e-4</span>, rho<span style="color: #666666">=0.9</span>, rho2<span style="color: #666666">=0.999</span>),
|
||
seed: <span style="color: #008000">int</span> <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>,
|
||
):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(nodes, output_func, copy(scheduler), seed)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>cost_func <span style="color: #666666">=</span> cost_func
|
||
|
||
<span style="color: #408080; font-style: italic"># initiate matrices for later</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># decides if the output layer performs binary or multi-class classification</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_set_pred_format()
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>, X_batch: np<span style="color: #666666">.</span>ndarray):
|
||
<span style="color: #408080; font-style: italic"># calculate a, z</span>
|
||
<span style="color: #408080; font-style: italic"># note that bias is not added as this would create an extra output class</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix <span style="color: #666666">=</span> X_batch <span style="color: #666666">@</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix)
|
||
|
||
<span style="color: #408080; font-style: italic"># returns prediction</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>, target, a_previous, lam):
|
||
<span style="color: #408080; font-style: italic"># note that in the OutputLayer the activation function is the output function</span>
|
||
activation_derivative <span style="color: #666666">=</span> derivate(<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate output delta terms</span>
|
||
<span style="color: #408080; font-style: italic"># for multi-class or binary classification</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">==</span> <span style="color: #BA2121">"Multi-class"</span>:
|
||
delta_term <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix <span style="color: #666666">-</span> target
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
cost_func_derivative <span style="color: #666666">=</span> grad(<span style="color: #008000">self</span><span style="color: #666666">.</span>cost_func(target))
|
||
delta_term <span style="color: #666666">=</span> activation_derivative(<span style="color: #008000">self</span><span style="color: #666666">.</span>z_matrix) <span style="color: #666666">*</span> cost_func_derivative(
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>a_matrix
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># intiate matrix that stores gradient</span>
|
||
gradient_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(
|
||
(
|
||
a_previous<span style="color: #666666">.</span>shape[input_index],
|
||
a_previous<span style="color: #666666">.</span>shape[node_index] <span style="color: #666666">-</span> bias_index,
|
||
delta_term<span style="color: #666666">.</span>shape[node_index],
|
||
)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate gradient = delta term * previous a</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(delta_term)):
|
||
gradient_weights[i, :, :] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>outer(
|
||
a_previous[i, bias_index:], delta_term[i, :]
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># sum the gradient, divide by input_index</span>
|
||
gradient_weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(gradient_weights, axis<span style="color: #666666">=</span>input_index)
|
||
<span style="color: #408080; font-style: italic"># for the bias gradient we do not multiply by previous a</span>
|
||
gradient_bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean(delta_term, axis<span style="color: #666666">=</span>input_index)<span style="color: #666666">.</span>reshape(
|
||
<span style="color: #666666">1</span>, delta_term<span style="color: #666666">.</span>shape[node_index]
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># regularization term</span>
|
||
gradient_weights <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights[bias_index:, :] <span style="color: #666666">*</span> lam
|
||
|
||
<span style="color: #408080; font-style: italic"># send gradients into scheduler</span>
|
||
<span style="color: #408080; font-style: italic"># returns update matrix which will be used to update the weights and bias</span>
|
||
update_matrix <span style="color: #666666">=</span> np<span style="color: #666666">.</span>vstack(
|
||
[
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_bias<span style="color: #666666">.</span>update_change(gradient_bias),
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_weight<span style="color: #666666">.</span>update_change(gradient_weights),
|
||
]
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># update weights</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">-=</span> update_matrix
|
||
|
||
<span style="color: #408080; font-style: italic"># return weights and delta term, input for backpropagation in previous layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>weights, delta_term
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_weights</span>(<span style="color: #008000">self</span>, previous_nodes):
|
||
<span style="color: #408080; font-style: italic"># sets seed to remove randomness inbetween runs</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>seed <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #008000">self</span><span style="color: #666666">.</span>seed)
|
||
|
||
<span style="color: #408080; font-style: italic"># add bias, initiate random weights</span>
|
||
bias <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>weights <span style="color: #666666">=</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>rand(previous_nodes <span style="color: #666666">+</span> bias, <span style="color: #008000">self</span><span style="color: #666666">.</span>nodes)
|
||
|
||
<span style="color: #408080; font-style: italic"># returns number of nodes, used for reset_weights in next layer</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>nodes
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_reset_scheduler</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># resets scheduler per epoch</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_weight<span style="color: #666666">.</span>reset()
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler_bias<span style="color: #666666">.</span>reset()
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_set_pred_format</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># sets prediction format to either regression, binary or multi-class classification</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span> <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #008000; font-weight: bold">None</span> <span style="color: #AA22FF; font-weight: bold">or</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"identity"</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">=</span> <span style="color: #BA2121">"Regression"</span>
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"sigmoid"</span> <span style="color: #AA22FF; font-weight: bold">or</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func<span style="color: #666666">.</span><span style="color: #19177C">__name__</span> <span style="color: #666666">==</span> <span style="color: #BA2121">"tanh"</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">=</span> <span style="color: #BA2121">"Binary"</span>
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">=</span> <span style="color: #BA2121">"Multi-class"</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">get_pred_format</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># returns format of prediction</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="optimized-convolution2dlayer" class="anchor">Optimized Convolution2DLayer </h3>
|
||
|
||
<p>For our CNN, we have also implemented an optimized version of the
|
||
Convolution2DLayer, Convolution2DLayerOPT, which runs much faster. See
|
||
VII. Remarks for discussion. This layer will per default be used by
|
||
the CNN due to its computational advantages, but is much less
|
||
readable. We've documented it such that specially interested students
|
||
can understand the principles behind it, but it is not recommended to
|
||
read. In short, we reshape and transpose parts of the image such that
|
||
the convolutional operation can be swapped out for a simple matrix
|
||
multiplication.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">Convolution2DLayerOPT</span>(Convolution2DLayer):
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Am optimized version of the convolution layer above which</span>
|
||
<span style="color: #BA2121; font-style: italic"> utilizes an approach of extracting windows of size equivalent</span>
|
||
<span style="color: #BA2121; font-style: italic"> in size to the filter. The convoution is then performed on those</span>
|
||
<span style="color: #BA2121; font-style: italic"> windows instead of a full feature map.</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||
<span style="color: #008000">self</span>,
|
||
input_channels,
|
||
feature_maps,
|
||
kernel_height,
|
||
kernel_width,
|
||
v_stride,
|
||
h_stride,
|
||
pad,
|
||
act_func: Callable,
|
||
seed<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>,
|
||
reset_weights_independently<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,
|
||
):
|
||
<span style="color: #008000">super</span>()<span style="color: #666666">.</span><span style="color: #0000FF">__init__</span>(
|
||
input_channels,
|
||
feature_maps,
|
||
kernel_height,
|
||
kernel_width,
|
||
v_stride,
|
||
h_stride,
|
||
pad,
|
||
act_func,
|
||
seed,
|
||
)
|
||
<span style="color: #408080; font-style: italic"># true if layer is used outside of CNN</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> reset_weights_independently <span style="color: #666666">==</span> <span style="color: #008000; font-weight: bold">True</span>:
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_reset_weights_independently()
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>, X_batch):
|
||
<span style="color: #408080; font-style: italic"># The optimized _feedforward method is difficult to understand but computationally more efficient</span>
|
||
<span style="color: #408080; font-style: italic"># for a more "by the book" approach, please look at the _feedforward method of Convolution2DLayer</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># save the input for backpropagation</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward <span style="color: #666666">=</span> X_batch
|
||
|
||
<span style="color: #408080; font-style: italic"># check that there are the correct amount of input channels</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_check_for_errors()
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate new shape after stride</span>
|
||
strided_height <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>ceil(X_batch<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride))
|
||
strided_width <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>ceil(X_batch<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride))
|
||
|
||
<span style="color: #408080; font-style: italic"># get windows of the image for more computationally efficient convolution</span>
|
||
<span style="color: #408080; font-style: italic"># the idea is that we want to align the dimensions that we wish to matrix</span>
|
||
<span style="color: #408080; font-style: italic"># multiply, then use a simple matrix multiplication instead of convolution.</span>
|
||
<span style="color: #408080; font-style: italic"># then, we reshape the size back to its intended shape</span>
|
||
windows <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_extract_windows(X_batch)
|
||
windows <span style="color: #666666">=</span> windows<span style="color: #666666">.</span>transpose(<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">2</span>, <span style="color: #666666">3</span>, <span style="color: #666666">4</span>)<span style="color: #666666">.</span>reshape(
|
||
X_batch<span style="color: #666666">.</span>shape[input_index],
|
||
strided_height <span style="color: #666666">*</span> strided_width,
|
||
<span style="color: #666666">-1</span>,
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># reshape the kernel for more computationally efficient convolution</span>
|
||
kernel <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel
|
||
kernel <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>transpose(<span style="color: #666666">0</span>, <span style="color: #666666">2</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1</span>)<span style="color: #666666">.</span>reshape(
|
||
kernel<span style="color: #666666">.</span>shape[kernel_input_channels_index]
|
||
<span style="color: #666666">*</span> kernel<span style="color: #666666">.</span>shape[height_index]
|
||
<span style="color: #666666">*</span> kernel<span style="color: #666666">.</span>shape[width_index],
|
||
<span style="color: #666666">-1</span>,
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># use simple matrix calculation to obtain output</span>
|
||
output <span style="color: #666666">=</span> (
|
||
(windows <span style="color: #666666">@</span> kernel)
|
||
<span style="color: #666666">.</span>reshape(
|
||
X_batch<span style="color: #666666">.</span>shape[input_index],
|
||
strided_height,
|
||
strided_width,
|
||
<span style="color: #666666">-1</span>,
|
||
)
|
||
<span style="color: #666666">.</span>transpose(<span style="color: #666666">0</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1</span>, <span style="color: #666666">2</span>)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># The output is reshaped and rearranged to appropriate shape</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>act_func(
|
||
output <span style="color: #666666">/</span> (<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height <span style="color: #666666">*</span> X_batch<span style="color: #666666">.</span>shape[feature_maps_index])
|
||
)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>, delta_term_next):
|
||
<span style="color: #408080; font-style: italic"># The optimized _backpropagate method is difficult to understand but computationally more efficient</span>
|
||
<span style="color: #408080; font-style: italic"># for a more "by the book" approach, please look at the _backpropagate method of Convolution2DLayer</span>
|
||
act_derivative <span style="color: #666666">=</span> derivate(<span style="color: #008000">self</span><span style="color: #666666">.</span>act_func)
|
||
delta_term_next <span style="color: #666666">=</span> act_derivative(delta_term_next)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate strided dimensions</span>
|
||
strided_height <span style="color: #666666">=</span> <span style="color: #008000">int</span>(
|
||
np<span style="color: #666666">.</span>ceil(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride)
|
||
)
|
||
strided_width <span style="color: #666666">=</span> <span style="color: #008000">int</span>(
|
||
np<span style="color: #666666">.</span>ceil(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">/</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># copy kernel</span>
|
||
kernel <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel
|
||
|
||
<span style="color: #408080; font-style: italic"># get windows, reshape for matrix multiplication</span>
|
||
windows <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_extract_windows(<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward, <span style="color: #BA2121">"image"</span>)<span style="color: #666666">.</span>reshape(
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_index]
|
||
<span style="color: #666666">*</span> strided_height
|
||
<span style="color: #666666">*</span> strided_width,
|
||
<span style="color: #666666">-1</span>,
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize output gradient, reshape and transpose into correct shape</span>
|
||
<span style="color: #408080; font-style: italic"># for matrix multiplication</span>
|
||
output_grad_tr <span style="color: #666666">=</span> delta_term_next<span style="color: #666666">.</span>transpose(<span style="color: #666666">0</span>, <span style="color: #666666">2</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1</span>)<span style="color: #666666">.</span>reshape(
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_index]
|
||
<span style="color: #666666">*</span> strided_height
|
||
<span style="color: #666666">*</span> strided_width,
|
||
<span style="color: #666666">-1</span>,
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate gradient kernel via simple matrix multiplication and reshaping</span>
|
||
gradient_kernel <span style="color: #666666">=</span> (
|
||
(windows<span style="color: #666666">.</span>T <span style="color: #666666">@</span> output_grad_tr)
|
||
<span style="color: #666666">.</span>reshape(
|
||
kernel<span style="color: #666666">.</span>shape[kernel_input_channels_index],
|
||
kernel<span style="color: #666666">.</span>shape[height_index],
|
||
kernel<span style="color: #666666">.</span>shape[width_index],
|
||
kernel<span style="color: #666666">.</span>shape[kernel_feature_maps_index],
|
||
)
|
||
<span style="color: #666666">.</span>transpose(<span style="color: #666666">0</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1</span>, <span style="color: #666666">2</span>)
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># for computing the input gradient</span>
|
||
windows_out, upsampled_height, upsampled_width <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_extract_windows(
|
||
delta_term_next, <span style="color: #BA2121">"grad"</span>
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculate new window dimensions</span>
|
||
new_windows_first_dim <span style="color: #666666">=</span> (
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_index]
|
||
<span style="color: #666666">*</span> upsampled_height
|
||
<span style="color: #666666">*</span> upsampled_width
|
||
)
|
||
<span style="color: #408080; font-style: italic"># ceil allows for various asymmetric kernels</span>
|
||
new_windows_sec_dim <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>ceil(windows_out<span style="color: #666666">.</span>size <span style="color: #666666">/</span> new_windows_first_dim))
|
||
|
||
<span style="color: #408080; font-style: italic"># reshape for matrix multiplication</span>
|
||
windows_out <span style="color: #666666">=</span> windows_out<span style="color: #666666">.</span>transpose(<span style="color: #666666">1</span>, <span style="color: #666666">0</span>, <span style="color: #666666">2</span>, <span style="color: #666666">3</span>, <span style="color: #666666">4</span>)<span style="color: #666666">.</span>reshape(
|
||
new_windows_first_dim, new_windows_sec_dim
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># reshape for matrix multiplication</span>
|
||
kernel_reshaped <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>reshape(<span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels, <span style="color: #666666">-1</span>)
|
||
|
||
<span style="color: #408080; font-style: italic"># calculating input gradient for next convolutional layer</span>
|
||
input_grad <span style="color: #666666">=</span> (windows_out <span style="color: #666666">@</span> kernel_reshaped<span style="color: #666666">.</span>T)<span style="color: #666666">.</span>reshape(
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_index],
|
||
upsampled_height,
|
||
upsampled_width,
|
||
kernel<span style="color: #666666">.</span>shape[kernel_input_channels_index],
|
||
)
|
||
input_grad <span style="color: #666666">=</span> input_grad<span style="color: #666666">.</span>transpose(<span style="color: #666666">0</span>, <span style="color: #666666">3</span>, <span style="color: #666666">1</span>, <span style="color: #666666">2</span>)
|
||
|
||
<span style="color: #408080; font-style: italic"># Update the weights in the kernel</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>kernel <span style="color: #666666">-=</span> gradient_kernel
|
||
|
||
<span style="color: #408080; font-style: italic"># Output the gradient to propagate backwards</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> input_grad
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_extract_windows</span>(<span style="color: #008000">self</span>, X_batch, batch_type<span style="color: #666666">=</span><span style="color: #BA2121">"image"</span>):
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Receives as input the X_batch with shape (inputs, feature_maps, image_height, image_width)</span>
|
||
<span style="color: #BA2121; font-style: italic"> and extract windows of size kernel_height * kernel_width for every image and every feature_map.</span>
|
||
<span style="color: #BA2121; font-style: italic"> It then returns an np.ndarray of shape (image_height * image_width, inputs, feature_maps, kernel_height, kernel_width)</span>
|
||
<span style="color: #BA2121; font-style: italic"> which will be used either to filter the images in feedforward or to calculate the gradient.</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize list of windows</span>
|
||
windows <span style="color: #666666">=</span> []
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> batch_type <span style="color: #666666">==</span> <span style="color: #BA2121">"image"</span>:
|
||
<span style="color: #408080; font-style: italic"># pad the images</span>
|
||
X_batch_padded <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_padding(X_batch, batch_type<span style="color: #666666">=</span><span style="color: #BA2121">"image"</span>)
|
||
img_height, img_width <span style="color: #666666">=</span> X_batch_padded<span style="color: #666666">.</span>shape[<span style="color: #666666">2</span>:]
|
||
<span style="color: #408080; font-style: italic"># For each location in the image...</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> h <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(
|
||
<span style="color: #666666">0</span>,
|
||
X_batch<span style="color: #666666">.</span>shape[height_index],
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride,
|
||
):
|
||
<span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(
|
||
<span style="color: #666666">0</span>,
|
||
X_batch<span style="color: #666666">.</span>shape[width_index],
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride,
|
||
):
|
||
<span style="color: #408080; font-style: italic"># ...obtain an image patch of the original size (strided)</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># get window</span>
|
||
window <span style="color: #666666">=</span> X_batch_padded[
|
||
:,
|
||
:,
|
||
h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height,
|
||
w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width,
|
||
]
|
||
|
||
<span style="color: #408080; font-style: italic"># append to list of windows</span>
|
||
windows<span style="color: #666666">.</span>append(window)
|
||
|
||
<span style="color: #408080; font-style: italic"># return numpy array instead of list</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>stack(windows)
|
||
|
||
<span style="color: #408080; font-style: italic"># In order to be able to perform backprogagation by the method of window extraction,</span>
|
||
<span style="color: #408080; font-style: italic"># here is a modified approach to extracting the windows which allow for the necessary</span>
|
||
<span style="color: #408080; font-style: italic"># upsampling of the gradient in case the on of the stride parameters is larger than one.</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> batch_type <span style="color: #666666">==</span> <span style="color: #BA2121">"grad"</span>:
|
||
|
||
<span style="color: #408080; font-style: italic"># In the case of one of the stride parameters being odd, we have to take some</span>
|
||
<span style="color: #408080; font-style: italic"># extra care in calculating the upsampled size of X_batch. We solve this</span>
|
||
<span style="color: #408080; font-style: italic"># by simply flooring the result of dividing stride by 2.</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666"><</span> <span style="color: #666666">2</span> <span style="color: #AA22FF; font-weight: bold">or</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">%</span> <span style="color: #666666">2</span> <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
||
v_stride <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
v_stride <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>floor(<span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">/</span> <span style="color: #666666">2</span>))
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666"><</span> <span style="color: #666666">2</span> <span style="color: #AA22FF; font-weight: bold">or</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">%</span> <span style="color: #666666">2</span> <span style="color: #666666">==</span> <span style="color: #666666">0</span>:
|
||
h_stride <span style="color: #666666">=</span> <span style="color: #666666">0</span>
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
h_stride <span style="color: #666666">=</span> <span style="color: #008000">int</span>(np<span style="color: #666666">.</span>floor(<span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">/</span> <span style="color: #666666">2</span>))
|
||
|
||
upsampled_height <span style="color: #666666">=</span> (X_batch<span style="color: #666666">.</span>shape[height_index] <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride) <span style="color: #666666">-</span> v_stride
|
||
upsampled_width <span style="color: #666666">=</span> (X_batch<span style="color: #666666">.</span>shape[width_index] <span style="color: #666666">*</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride) <span style="color: #666666">-</span> h_stride
|
||
|
||
<span style="color: #408080; font-style: italic"># When upsampling, we need to insert rows and columns filled with zeros</span>
|
||
<span style="color: #408080; font-style: italic"># into each feature map. How many of those we have to insert is purely</span>
|
||
<span style="color: #408080; font-style: italic"># dependant on the value of stride parameter in the vertical and horizontal</span>
|
||
<span style="color: #408080; font-style: italic"># direction.</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">></span> <span style="color: #666666">1</span>:
|
||
v_ind <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(X_batch<span style="color: #666666">.</span>shape[height_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride <span style="color: #666666">-</span> <span style="color: #666666">1</span>):
|
||
X_batch <span style="color: #666666">=</span> np<span style="color: #666666">.</span>insert(X_batch, v_ind, <span style="color: #666666">0</span>, axis<span style="color: #666666">=</span>height_index)
|
||
v_ind <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">></span> <span style="color: #666666">1</span>:
|
||
h_ind <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(X_batch<span style="color: #666666">.</span>shape[width_index]):
|
||
<span style="color: #008000; font-weight: bold">for</span> k <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride <span style="color: #666666">-</span> <span style="color: #666666">1</span>):
|
||
X_batch <span style="color: #666666">=</span> np<span style="color: #666666">.</span>insert(X_batch, h_ind, <span style="color: #666666">0</span>, axis<span style="color: #666666">=</span>width_index)
|
||
h_ind <span style="color: #666666">+=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride
|
||
|
||
<span style="color: #408080; font-style: italic"># Since the insertion of zero-filled rows and columns isn't perfect, we have</span>
|
||
<span style="color: #408080; font-style: italic"># to assure that the resulting feature maps will have the expected upsampled height</span>
|
||
<span style="color: #408080; font-style: italic"># and width by cutting them og at desired dimensions.</span>
|
||
|
||
X_batch <span style="color: #666666">=</span> X_batch[:, :, :upsampled_height, :upsampled_width]
|
||
|
||
X_batch_padded <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_padding(X_batch, batch_type<span style="color: #666666">=</span><span style="color: #BA2121">"grad"</span>)
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize list of windows</span>
|
||
windows <span style="color: #666666">=</span> []
|
||
|
||
<span style="color: #408080; font-style: italic"># For each location in the image...</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> h <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(
|
||
<span style="color: #666666">0</span>,
|
||
X_batch<span style="color: #666666">.</span>shape[height_index],
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>v_stride,
|
||
):
|
||
<span style="color: #008000; font-weight: bold">for</span> w <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(
|
||
<span style="color: #666666">0</span>,
|
||
X_batch<span style="color: #666666">.</span>shape[width_index],
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>h_stride,
|
||
):
|
||
<span style="color: #408080; font-style: italic"># ...obtain an image patch of the original size (strided)</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># get window</span>
|
||
window <span style="color: #666666">=</span> X_batch_padded[
|
||
:, :, h : h <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_height, w : w <span style="color: #666666">+</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>kernel_width
|
||
]
|
||
|
||
<span style="color: #408080; font-style: italic"># append window to list</span>
|
||
windows<span style="color: #666666">.</span>append(window)
|
||
|
||
<span style="color: #408080; font-style: italic"># return numpy array, unsampled dimensions</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>stack(windows), upsampled_height, upsampled_width
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_check_for_errors</span>(<span style="color: #008000">self</span>):
|
||
<span style="color: #408080; font-style: italic"># compares input channels of data to input channels of Convolution2DLayer</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_channel_index] <span style="color: #666666">!=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels:
|
||
<span style="color: #008000; font-weight: bold">raise</span> <span style="color: #D2413A; font-weight: bold">AssertionError</span>(
|
||
<span style="color: #BA2121">f"ERROR: Number of input channels in data (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>X_batch_feedforward<span style="color: #666666">.</span>shape[input_channel_index]<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">) is not equal to input channels in Convolution2DLayerOPT (</span><span style="color: #BB6688; font-weight: bold">{</span><span style="color: #008000">self</span><span style="color: #666666">.</span>input_channels<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">)! Please change the number of input channels of the Convolution2DLayer such that they are equal"</span>
|
||
)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="the-convolutional-neural-network-cnn" class="anchor">The Convolutional Neural Network (CNN) </h3>
|
||
|
||
<p>Finally, we present the code for the CNN. The CNN class organizes all the layers, and allows for training on image data.</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">math</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">autograd.numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sys</span>
|
||
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">warnings</span>
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">autograd</span> <span style="color: #008000; font-weight: bold">import</span> grad, elementwise_grad
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">random</span> <span style="color: #008000; font-weight: bold">import</span> random, seed
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">copy</span> <span style="color: #008000; font-weight: bold">import</span> deepcopy
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">typing</span> <span style="color: #008000; font-weight: bold">import</span> Tuple, Callable
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.utils</span> <span style="color: #008000; font-weight: bold">import</span> resample
|
||
|
||
warnings<span style="color: #666666">.</span>simplefilter(<span style="color: #BA2121">"error"</span>)
|
||
|
||
|
||
<span style="color: #008000; font-weight: bold">class</span> <span style="color: #0000FF; font-weight: bold">CNN</span>:
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">__init__</span>(
|
||
<span style="color: #008000">self</span>,
|
||
cost_func: Callable <span style="color: #666666">=</span> CostCrossEntropy,
|
||
scheduler: Scheduler <span style="color: #666666">=</span> Adam(eta<span style="color: #666666">=1e-4</span>, rho<span style="color: #666666">=0.9</span>, rho2<span style="color: #666666">=0.999</span>),
|
||
seed: <span style="color: #008000">int</span> <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>,
|
||
):
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Instantiates CNN object</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I output_func (costFunctions) cost function for feed forward neural network part of CNN,</span>
|
||
<span style="color: #BA2121; font-style: italic"> such as "CostLogReg", "CostOLS" or "CostCrossEntropy"</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> II scheduler (Scheduler) optional parameter, default set to Adam. Can also be set to other</span>
|
||
<span style="color: #BA2121; font-style: italic"> schedulers such as AdaGrad, Momentum, RMS_prop and Constant. Note that schedulers have</span>
|
||
<span style="color: #BA2121; font-style: italic"> to be instantiated first with proper parameters (for example eta, rho and rho2 for Adam)</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> III seed (int) used for seeding all random operations</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>layers <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>cost_func <span style="color: #666666">=</span> cost_func
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler <span style="color: #666666">=</span> scheduler
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>schedulers_weight <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>schedulers_bias <span style="color: #666666">=</span> <span style="color: #008000">list</span>()
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>seed <span style="color: #666666">=</span> seed
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">add_FullyConnectedLayer</span>(
|
||
<span style="color: #008000">self</span>, nodes: <span style="color: #008000">int</span>, act_func<span style="color: #666666">=</span>LRELU, scheduler<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>
|
||
) <span style="color: #666666">-></span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Add a FullyConnectedLayer to the CNN, i.e. a hidden layer in the feed forward neural</span>
|
||
<span style="color: #BA2121; font-style: italic"> network part of the CNN. Often called a Dense layer in literature</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I nodes (int) number of nodes in FullyConnectedLayer</span>
|
||
<span style="color: #BA2121; font-style: italic"> II act_func (activationFunctions) activation function of FullyConnectedLayer,</span>
|
||
<span style="color: #BA2121; font-style: italic"> such as "sigmoid", "RELU", "LRELU", "softmax" or "identity"</span>
|
||
<span style="color: #BA2121; font-style: italic"> III scheduler (Scheduler) optional parameter, default set to Adam. Can also be set to other</span>
|
||
<span style="color: #BA2121; font-style: italic"> schedulers such as AdaGrad, Momentum, RMS_prop and Constant</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>layers, <span style="color: #BA2121">"FullyConnectedLayer should follow FlattenLayer in CNN"</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> scheduler <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
scheduler <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler
|
||
|
||
layer <span style="color: #666666">=</span> FullyConnectedLayer(nodes, act_func, scheduler, <span style="color: #008000">self</span><span style="color: #666666">.</span>seed)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>layers<span style="color: #666666">.</span>append(layer)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">add_OutputLayer</span>(<span style="color: #008000">self</span>, nodes: <span style="color: #008000">int</span>, output_func<span style="color: #666666">=</span>sigmoid, scheduler<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>) <span style="color: #666666">-></span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Add an OutputLayer to the CNN, i.e. a the final layer in the feed forward neural</span>
|
||
<span style="color: #BA2121; font-style: italic"> network part of the CNN</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I nodes (int) number of nodes in OutputLayer. Set nodes=1 for binary classification and</span>
|
||
<span style="color: #BA2121; font-style: italic"> nodes = number of classes for multi-class classification</span>
|
||
<span style="color: #BA2121; font-style: italic"> II output_func (activationFunctions) activation function for the output layer, such as</span>
|
||
<span style="color: #BA2121; font-style: italic"> "identity" for regression, "sigmoid" for binary classification and "softmax" for multi-class</span>
|
||
<span style="color: #BA2121; font-style: italic"> classification</span>
|
||
<span style="color: #BA2121; font-style: italic"> III scheduler (Scheduler) optional parameter, default set to Adam. Can also be set to other</span>
|
||
<span style="color: #BA2121; font-style: italic"> schedulers such as AdaGrad, Momentum, RMS_prop and Constant</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>layers, <span style="color: #BA2121">"OutputLayer should follow FullyConnectedLayer in CNN"</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> scheduler <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
scheduler <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>scheduler
|
||
|
||
output_layer <span style="color: #666666">=</span> OutputLayer(
|
||
nodes, output_func, <span style="color: #008000">self</span><span style="color: #666666">.</span>cost_func, scheduler, <span style="color: #008000">self</span><span style="color: #666666">.</span>seed
|
||
)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>layers<span style="color: #666666">.</span>append(output_layer)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">=</span> output_layer<span style="color: #666666">.</span>get_pred_format()
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">add_FlattenLayer</span>(<span style="color: #008000">self</span>, act_func<span style="color: #666666">=</span>LRELU) <span style="color: #666666">-></span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Add a FlattenLayer to the CNN, which flattens the image data such that it is formatted to</span>
|
||
<span style="color: #BA2121; font-style: italic"> be used in the feed forward neural network part of the CNN</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>layers<span style="color: #666666">.</span>append(FlattenLayer(act_func<span style="color: #666666">=</span>act_func, seed<span style="color: #666666">=</span><span style="color: #008000">self</span><span style="color: #666666">.</span>seed))
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">add_Convolution2DLayer</span>(
|
||
<span style="color: #008000">self</span>,
|
||
input_channels<span style="color: #666666">=1</span>,
|
||
feature_maps<span style="color: #666666">=1</span>,
|
||
kernel_height<span style="color: #666666">=3</span>,
|
||
kernel_width<span style="color: #666666">=3</span>,
|
||
v_stride<span style="color: #666666">=1</span>,
|
||
h_stride<span style="color: #666666">=1</span>,
|
||
pad<span style="color: #666666">=</span><span style="color: #BA2121">"same"</span>,
|
||
act_func<span style="color: #666666">=</span>LRELU,
|
||
optimized<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,
|
||
) <span style="color: #666666">-></span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Add a Convolution2DLayer to the CNN, i.e. a convolutional layer with a 2 dimensional kernel. Should be</span>
|
||
<span style="color: #BA2121; font-style: italic"> the first layer added to the CNN</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I input_channels (int) specifies amount of input channels. For monochrome images, use input_channels</span>
|
||
<span style="color: #BA2121; font-style: italic"> = 1, and input_channels = 3 for colored images, where each channel represents one of R, G and B</span>
|
||
<span style="color: #BA2121; font-style: italic"> II feature_maps (int) amount of feature maps in CNN</span>
|
||
<span style="color: #BA2121; font-style: italic"> III kernel_height (int) height of the kernel, also called 'convolutional filter' in literature</span>
|
||
<span style="color: #BA2121; font-style: italic"> IV kernel_width (int) width of the kernel, also called 'convolutional filter' in literature</span>
|
||
<span style="color: #BA2121; font-style: italic"> V v_stride (int) value of vertical stride for dimentionality reduction</span>
|
||
<span style="color: #BA2121; font-style: italic"> VI h_stride (int) value of horizontal stride for dimentionality reduction</span>
|
||
<span style="color: #BA2121; font-style: italic"> VII pad (str) default = "same" ensures output size is the same as input size (given stride=1)</span>
|
||
<span style="color: #BA2121; font-style: italic"> VIII act_func (activationFunctions) default = "LRELU", nonlinear activation function</span>
|
||
<span style="color: #BA2121; font-style: italic"> IX optimized (bool) default = True, uses Convolution2DLayerOPT if True which is much faster when</span>
|
||
<span style="color: #BA2121; font-style: italic"> compared to Convolution2DLayer, which is a more straightforward, understandable implementation</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> optimized:
|
||
conv_layer <span style="color: #666666">=</span> Convolution2DLayerOPT(
|
||
input_channels,
|
||
feature_maps,
|
||
kernel_height,
|
||
kernel_width,
|
||
v_stride,
|
||
h_stride,
|
||
pad,
|
||
act_func,
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>seed,
|
||
reset_weights_independently<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||
)
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
conv_layer <span style="color: #666666">=</span> Convolution2DLayer(
|
||
input_channels,
|
||
feature_maps,
|
||
kernel_height,
|
||
kernel_width,
|
||
v_stride,
|
||
h_stride,
|
||
pad,
|
||
act_func,
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>seed,
|
||
reset_weights_independently<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||
)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>layers<span style="color: #666666">.</span>append(conv_layer)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">add_PoolingLayer</span>(
|
||
<span style="color: #008000">self</span>, kernel_height<span style="color: #666666">=2</span>, kernel_width<span style="color: #666666">=2</span>, v_stride<span style="color: #666666">=1</span>, h_stride<span style="color: #666666">=1</span>, pooling<span style="color: #666666">=</span><span style="color: #BA2121">"max"</span>
|
||
) <span style="color: #666666">-></span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Add a Pooling2DLayer to the CNN, i.e. a pooling layer that reduces the dimentionality of</span>
|
||
<span style="color: #BA2121; font-style: italic"> the image data. It is not necessary to use a Pooling2DLayer when creating a CNN, but it</span>
|
||
<span style="color: #BA2121; font-style: italic"> can be used to speed up the training</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I kernel_height (int) height of the kernel used for pooling</span>
|
||
<span style="color: #BA2121; font-style: italic"> II kernel_width (int) width of the kernel used for pooling</span>
|
||
<span style="color: #BA2121; font-style: italic"> III v_stride (int) value of vertical stride for dimentionality reduction</span>
|
||
<span style="color: #BA2121; font-style: italic"> IV h_stride (int) value of horizontal stride for dimentionality reduction</span>
|
||
<span style="color: #BA2121; font-style: italic"> V pooling (str) either "max" or "average", describes type of pooling performed</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
pooling_layer <span style="color: #666666">=</span> Pooling2DLayer(
|
||
kernel_height, kernel_width, v_stride, h_stride, pooling, <span style="color: #008000">self</span><span style="color: #666666">.</span>seed
|
||
)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>layers<span style="color: #666666">.</span>append(pooling_layer)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">fit</span>(
|
||
<span style="color: #008000">self</span>,
|
||
X: np<span style="color: #666666">.</span>ndarray,
|
||
t: np<span style="color: #666666">.</span>ndarray,
|
||
epochs: <span style="color: #008000">int</span> <span style="color: #666666">=</span> <span style="color: #666666">100</span>,
|
||
lam: <span style="color: #008000">float</span> <span style="color: #666666">=</span> <span style="color: #666666">0</span>,
|
||
batches: <span style="color: #008000">int</span> <span style="color: #666666">=</span> <span style="color: #666666">1</span>,
|
||
X_val: np<span style="color: #666666">.</span>ndarray <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>,
|
||
t_val: np<span style="color: #666666">.</span>ndarray <span style="color: #666666">=</span> <span style="color: #008000; font-weight: bold">None</span>,
|
||
) <span style="color: #666666">-></span> <span style="color: #008000">dict</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Fits the CNN to input X for a given amount of epochs. Performs feedforward and backpropagation passes,</span>
|
||
<span style="color: #BA2121; font-style: italic"> can utilize batches, regulariziation and validation if desired.</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> X (numpy array) with input data in format [images, input channels,</span>
|
||
<span style="color: #BA2121; font-style: italic"> image height, image_width]</span>
|
||
<span style="color: #BA2121; font-style: italic"> t (numpy array) target labels for input data</span>
|
||
<span style="color: #BA2121; font-style: italic"> epochs (int) amount of epochs</span>
|
||
<span style="color: #BA2121; font-style: italic"> lam (float) regulariziation term lambda</span>
|
||
<span style="color: #BA2121; font-style: italic"> batches (int) amount of batches input data splits into</span>
|
||
<span style="color: #BA2121; font-style: italic"> X_val (numpy array) validation data</span>
|
||
<span style="color: #BA2121; font-style: italic"> t_val (numpy array) target labels for validation data</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> scores (dict) a dictionary with "train_error", "train_acc", "val_error", val_acc" keys</span>
|
||
<span style="color: #BA2121; font-style: italic"> that contain numpy arrays with float values of all accuracies/errors over all epochs.</span>
|
||
<span style="color: #BA2121; font-style: italic"> Can be used to create plots. Also used to update the progress bar during training</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># setup</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>seed <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>seed(<span style="color: #008000">self</span><span style="color: #666666">.</span>seed)
|
||
|
||
<span style="color: #408080; font-style: italic"># initialize weights</span>
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_initialize_weights(X)
|
||
|
||
<span style="color: #408080; font-style: italic"># create arrays for score metrics</span>
|
||
scores <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_initialize_scores(epochs)
|
||
|
||
<span style="color: #008000; font-weight: bold">assert</span> batches <span style="color: #666666"><=</span> t<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]
|
||
batch_size <span style="color: #666666">=</span> X<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> batches
|
||
|
||
<span style="color: #008000; font-weight: bold">try</span>:
|
||
<span style="color: #008000; font-weight: bold">for</span> epoch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(epochs):
|
||
<span style="color: #008000; font-weight: bold">for</span> batch <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(batches):
|
||
<span style="color: #408080; font-style: italic"># minibatch gradient descent</span>
|
||
<span style="color: #408080; font-style: italic"># If the for loop has reached the last batch, take all thats left</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> batch <span style="color: #666666">==</span> batches <span style="color: #666666">-</span> <span style="color: #666666">1</span>:
|
||
X_batch <span style="color: #666666">=</span> X[batch <span style="color: #666666">*</span> batch_size :, :, :, :]
|
||
t_batch <span style="color: #666666">=</span> t[batch <span style="color: #666666">*</span> batch_size :, :]
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
X_batch <span style="color: #666666">=</span> X[
|
||
batch <span style="color: #666666">*</span> batch_size : (batch <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #666666">*</span> batch_size, :, :, :
|
||
]
|
||
t_batch <span style="color: #666666">=</span> t[batch <span style="color: #666666">*</span> batch_size : (batch <span style="color: #666666">+</span> <span style="color: #666666">1</span>) <span style="color: #666666">*</span> batch_size, :]
|
||
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_feedforward(X_batch)
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_backpropagate(t_batch, lam)
|
||
|
||
<span style="color: #408080; font-style: italic"># reset schedulers for each epoch (some schedulers pass in this call)</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> layer <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>layers:
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">isinstance</span>(layer, FullyConnectedLayer):
|
||
layer<span style="color: #666666">.</span>_reset_scheduler()
|
||
|
||
<span style="color: #408080; font-style: italic"># computing performance metrics</span>
|
||
scores <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_compute_scores(scores, epoch, X, t, X_val, t_val)
|
||
|
||
<span style="color: #408080; font-style: italic"># printing progress bar</span>
|
||
print_length <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_progress_bar(
|
||
epoch,
|
||
epochs,
|
||
scores,
|
||
)
|
||
<span style="color: #408080; font-style: italic"># allows for stopping training at any point and seeing the result</span>
|
||
<span style="color: #008000; font-weight: bold">except</span> <span style="color: #D2413A; font-weight: bold">KeyboardInterrupt</span>:
|
||
<span style="color: #008000; font-weight: bold">pass</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># visualization of training progression (similiar to tensorflow progression bar)</span>
|
||
sys<span style="color: #666666">.</span>stdout<span style="color: #666666">.</span>write(<span style="color: #BA2121">"</span><span style="color: #BB6622; font-weight: bold">\r</span><span style="color: #BA2121">"</span> <span style="color: #666666">+</span> <span style="color: #BA2121">" "</span> <span style="color: #666666">*</span> print_length)
|
||
sys<span style="color: #666666">.</span>stdout<span style="color: #666666">.</span>flush()
|
||
<span style="color: #008000">self</span><span style="color: #666666">.</span>_progress_bar(
|
||
epochs,
|
||
epochs,
|
||
scores,
|
||
)
|
||
sys<span style="color: #666666">.</span>stdout<span style="color: #666666">.</span>write(<span style="color: #BA2121">""</span>)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> scores
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_feedforward</span>(<span style="color: #008000">self</span>, X_batch) <span style="color: #666666">-></span> np<span style="color: #666666">.</span>ndarray:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Performs the feedforward pass for all layers in the CNN. Called from fit()</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
a <span style="color: #666666">=</span> X_batch
|
||
<span style="color: #008000; font-weight: bold">for</span> layer <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>layers:
|
||
a <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_feedforward(a)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> a
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_backpropagate</span>(<span style="color: #008000">self</span>, t_batch, lam) <span style="color: #666666">-></span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Performs backpropagation for all layers in the CNN. Called from fit()</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> <span style="color: #008000">len</span>(<span style="color: #008000">self</span><span style="color: #666666">.</span>layers) <span style="color: #666666">>=</span> <span style="color: #666666">2</span>
|
||
reversed_layers <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>layers[::<span style="color: #666666">-1</span>]
|
||
|
||
<span style="color: #408080; font-style: italic"># for every layer, backwards</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #008000">len</span>(reversed_layers) <span style="color: #666666">-</span> <span style="color: #666666">1</span>):
|
||
layer <span style="color: #666666">=</span> reversed_layers[i]
|
||
prev_layer <span style="color: #666666">=</span> reversed_layers[i <span style="color: #666666">+</span> <span style="color: #666666">1</span>]
|
||
|
||
<span style="color: #408080; font-style: italic"># OutputLayer</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">isinstance</span>(layer, OutputLayer):
|
||
prev_a <span style="color: #666666">=</span> prev_layer<span style="color: #666666">.</span>get_prev_a()
|
||
weights_next, delta_next <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_backpropagate(t_batch, prev_a, lam)
|
||
|
||
<span style="color: #408080; font-style: italic"># FullyConnectedLayer</span>
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">isinstance</span>(layer, FullyConnectedLayer):
|
||
<span style="color: #008000; font-weight: bold">assert</span> (
|
||
delta_next <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
), <span style="color: #BA2121">"No OutputLayer to follow FullyConnectedLayer"</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> (
|
||
weights_next <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
), <span style="color: #BA2121">"No OutputLayer to follow FullyConnectedLayer"</span>
|
||
prev_a <span style="color: #666666">=</span> prev_layer<span style="color: #666666">.</span>get_prev_a()
|
||
weights_next, delta_next <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_backpropagate(
|
||
weights_next, delta_next, prev_a, lam
|
||
)
|
||
|
||
<span style="color: #408080; font-style: italic"># FlattenLayer</span>
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">isinstance</span>(layer, FlattenLayer):
|
||
<span style="color: #008000; font-weight: bold">assert</span> (
|
||
delta_next <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
), <span style="color: #BA2121">"No FullyConnectedLayer to follow FlattenLayer"</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> (
|
||
weights_next <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
), <span style="color: #BA2121">"No FullyConnectedLayer to follow FlattenLayer"</span>
|
||
delta_next <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_backpropagate(weights_next, delta_next)
|
||
|
||
<span style="color: #408080; font-style: italic"># Convolution2DLayer and Convolution2DLayerOPT</span>
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">isinstance</span>(layer, Convolution2DLayer):
|
||
<span style="color: #008000; font-weight: bold">assert</span> (
|
||
delta_next <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>
|
||
), <span style="color: #BA2121">"No FlattenLayer to follow Convolution2DLayer"</span>
|
||
delta_next <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_backpropagate(delta_next)
|
||
|
||
<span style="color: #408080; font-style: italic"># Pooling2DLayer</span>
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">isinstance</span>(layer, Pooling2DLayer):
|
||
<span style="color: #008000; font-weight: bold">assert</span> delta_next <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>, <span style="color: #BA2121">"No Layer to follow Pooling2DLayer"</span>
|
||
delta_next <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_backpropagate(delta_next)
|
||
|
||
<span style="color: #408080; font-style: italic"># Catch error</span>
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
<span style="color: #008000; font-weight: bold">raise</span> <span style="color: #D2413A; font-weight: bold">NotImplementedError</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_compute_scores</span>(
|
||
<span style="color: #008000">self</span>,
|
||
scores: <span style="color: #008000">dict</span>,
|
||
epoch: <span style="color: #008000">int</span>,
|
||
X: np<span style="color: #666666">.</span>ndarray,
|
||
t: np<span style="color: #666666">.</span>ndarray,
|
||
X_val: np<span style="color: #666666">.</span>ndarray,
|
||
t_val: np<span style="color: #666666">.</span>ndarray,
|
||
) <span style="color: #666666">-></span> <span style="color: #008000">dict</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Computes scores such as training error, training accuracy, validation error</span>
|
||
<span style="color: #BA2121; font-style: italic"> and validation accuracy for the CNN depending on if a validation set is used</span>
|
||
<span style="color: #BA2121; font-style: italic"> and if the CNN performs classification or regression</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> scores (dict) a dictionary with "train_error", "train_acc", "val_error", val_acc" keys</span>
|
||
<span style="color: #BA2121; font-style: italic"> that contain numpy arrays with float values of all accuracies/errors over all epochs.</span>
|
||
<span style="color: #BA2121; font-style: italic"> Can be used to create plots. Also used to update the progress bar during training</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
pred_train <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>predict(X)
|
||
cost_function_train <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>cost_func(t)
|
||
train_error <span style="color: #666666">=</span> cost_function_train(pred_train)
|
||
scores[<span style="color: #BA2121">"train_error"</span>][epoch] <span style="color: #666666">=</span> train_error
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> X_val <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span> <span style="color: #AA22FF; font-weight: bold">and</span> t_val <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
cost_function_val <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>cost_func(t_val)
|
||
pred_val <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>predict(X_val)
|
||
val_error <span style="color: #666666">=</span> cost_function_val(pred_val)
|
||
scores[<span style="color: #BA2121">"val_error"</span>][epoch] <span style="color: #666666">=</span> val_error
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">!=</span> <span style="color: #BA2121">"Regression"</span>:
|
||
train_acc <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_accuracy(pred_train, t)
|
||
scores[<span style="color: #BA2121">"train_acc"</span>][epoch] <span style="color: #666666">=</span> train_acc
|
||
<span style="color: #008000; font-weight: bold">if</span> X_val <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span> <span style="color: #AA22FF; font-weight: bold">and</span> t_val <span style="color: #AA22FF; font-weight: bold">is</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
val_acc <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_accuracy(pred_val, t_val)
|
||
scores[<span style="color: #BA2121">"val_acc"</span>][epoch] <span style="color: #666666">=</span> val_acc
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> scores
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_initialize_scores</span>(<span style="color: #008000">self</span>, epochs) <span style="color: #666666">-></span> <span style="color: #008000">dict</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Initializes scores such as training error, training accuracy, validation error</span>
|
||
<span style="color: #BA2121; font-style: italic"> and validation accuracy for the CNN</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> A dictionary with "train_error", "train_acc", "val_error", val_acc" keys that</span>
|
||
<span style="color: #BA2121; font-style: italic"> will contain numpy arrays with float values of all accuracies/errors over all epochs</span>
|
||
<span style="color: #BA2121; font-style: italic"> when passed through the _compute_scores() function during fit()</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
scores <span style="color: #666666">=</span> <span style="color: #008000">dict</span>()
|
||
|
||
train_errors <span style="color: #666666">=</span> np<span style="color: #666666">.</span>empty(epochs)
|
||
train_errors<span style="color: #666666">.</span>fill(np<span style="color: #666666">.</span>nan)
|
||
val_errors <span style="color: #666666">=</span> np<span style="color: #666666">.</span>empty(epochs)
|
||
val_errors<span style="color: #666666">.</span>fill(np<span style="color: #666666">.</span>nan)
|
||
|
||
train_accs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>empty(epochs)
|
||
train_accs<span style="color: #666666">.</span>fill(np<span style="color: #666666">.</span>nan)
|
||
val_accs <span style="color: #666666">=</span> np<span style="color: #666666">.</span>empty(epochs)
|
||
val_accs<span style="color: #666666">.</span>fill(np<span style="color: #666666">.</span>nan)
|
||
|
||
scores[<span style="color: #BA2121">"train_error"</span>] <span style="color: #666666">=</span> train_errors
|
||
scores[<span style="color: #BA2121">"val_error"</span>] <span style="color: #666666">=</span> val_errors
|
||
scores[<span style="color: #BA2121">"train_acc"</span>] <span style="color: #666666">=</span> train_accs
|
||
scores[<span style="color: #BA2121">"val_acc"</span>] <span style="color: #666666">=</span> val_accs
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> scores
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_initialize_weights</span>(<span style="color: #008000">self</span>, X: np<span style="color: #666666">.</span>ndarray) <span style="color: #666666">-></span> <span style="color: #008000; font-weight: bold">None</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Initializes weights for all layers in CNN</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I X (np.ndarray) input of format [img, feature_maps, height, width]</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
prev_nodes <span style="color: #666666">=</span> X
|
||
<span style="color: #008000; font-weight: bold">for</span> layer <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>layers:
|
||
prev_nodes <span style="color: #666666">=</span> layer<span style="color: #666666">.</span>_reset_weights(prev_nodes)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">predict</span>(<span style="color: #008000">self</span>, X: np<span style="color: #666666">.</span>ndarray, <span style="color: #666666">*</span>, threshold<span style="color: #666666">=0.5</span>) <span style="color: #666666">-></span> np<span style="color: #666666">.</span>ndarray:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Predicts output of input X</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I X (np.ndarray) input [img, feature_maps, height, width]</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
|
||
prediction <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_feedforward(X)
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">==</span> <span style="color: #BA2121">"Binary"</span>:
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>where(prediction <span style="color: #666666">></span> threshold, <span style="color: #666666">1</span>, <span style="color: #666666">0</span>)
|
||
<span style="color: #008000; font-weight: bold">elif</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>pred_format <span style="color: #666666">==</span> <span style="color: #BA2121">"Multi-class"</span>:
|
||
class_prediction <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(prediction<span style="color: #666666">.</span>shape)
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(prediction<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>]):
|
||
class_prediction[i, np<span style="color: #666666">.</span>argmax(prediction[i, :])] <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> class_prediction
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
<span style="color: #008000; font-weight: bold">return</span> prediction
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_accuracy</span>(<span style="color: #008000">self</span>, prediction: np<span style="color: #666666">.</span>ndarray, target: np<span style="color: #666666">.</span>ndarray) <span style="color: #666666">-></span> <span style="color: #008000">float</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Calculates accuracy of given prediction to target</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Parameters:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> I prediction (np.ndarray): output of predict() fuction</span>
|
||
<span style="color: #BA2121; font-style: italic"> (1s and 0s in case of classification, and real numbers in case of regression)</span>
|
||
<span style="color: #BA2121; font-style: italic"> II target (np.ndarray): vector of true values (What the network should predict)</span>
|
||
|
||
<span style="color: #BA2121; font-style: italic"> Returns:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> A floating point number representing the percentage of correctly classified instances.</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000; font-weight: bold">assert</span> prediction<span style="color: #666666">.</span>size <span style="color: #666666">==</span> target<span style="color: #666666">.</span>size
|
||
<span style="color: #008000; font-weight: bold">return</span> np<span style="color: #666666">.</span>average((target <span style="color: #666666">==</span> prediction))
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_progress_bar</span>(<span style="color: #008000">self</span>, epoch: <span style="color: #008000">int</span>, epochs: <span style="color: #008000">int</span>, scores: <span style="color: #008000">dict</span>) <span style="color: #666666">-></span> <span style="color: #008000">int</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Displays progress of training</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
progression <span style="color: #666666">=</span> epoch <span style="color: #666666">/</span> epochs
|
||
epoch <span style="color: #666666">-=</span> <span style="color: #666666">1</span>
|
||
print_length <span style="color: #666666">=</span> <span style="color: #666666">40</span>
|
||
num_equals <span style="color: #666666">=</span> <span style="color: #008000">int</span>(progression <span style="color: #666666">*</span> print_length)
|
||
num_not <span style="color: #666666">=</span> print_length <span style="color: #666666">-</span> num_equals
|
||
arrow <span style="color: #666666">=</span> <span style="color: #BA2121">">"</span> <span style="color: #008000; font-weight: bold">if</span> num_equals <span style="color: #666666">></span> <span style="color: #666666">0</span> <span style="color: #008000; font-weight: bold">else</span> <span style="color: #BA2121">""</span>
|
||
bar <span style="color: #666666">=</span> <span style="color: #BA2121">"["</span> <span style="color: #666666">+</span> <span style="color: #BA2121">"="</span> <span style="color: #666666">*</span> (num_equals <span style="color: #666666">-</span> <span style="color: #666666">1</span>) <span style="color: #666666">+</span> arrow <span style="color: #666666">+</span> <span style="color: #BA2121">"-"</span> <span style="color: #666666">*</span> num_not <span style="color: #666666">+</span> <span style="color: #BA2121">"]"</span>
|
||
perc_print <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_fmt(progression <span style="color: #666666">*</span> <span style="color: #666666">100</span>, N<span style="color: #666666">=5</span>)
|
||
line <span style="color: #666666">=</span> <span style="color: #BA2121">f" </span><span style="color: #BB6688; font-weight: bold">{</span>bar<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> </span><span style="color: #BB6688; font-weight: bold">{</span>perc_print<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">% "</span>
|
||
|
||
<span style="color: #008000; font-weight: bold">for</span> key, score <span style="color: #AA22FF; font-weight: bold">in</span> scores<span style="color: #666666">.</span>items():
|
||
<span style="color: #008000; font-weight: bold">if</span> np<span style="color: #666666">.</span>isnan(score[epoch]) <span style="color: #666666">==</span> <span style="color: #008000; font-weight: bold">False</span>:
|
||
value <span style="color: #666666">=</span> <span style="color: #008000">self</span><span style="color: #666666">.</span>_fmt(score[epoch], N<span style="color: #666666">=4</span>)
|
||
line <span style="color: #666666">+=</span> <span style="color: #BA2121">f"| </span><span style="color: #BB6688; font-weight: bold">{</span>key<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">: </span><span style="color: #BB6688; font-weight: bold">{</span>value<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121"> "</span>
|
||
<span style="color: #008000">print</span>(line, end<span style="color: #666666">=</span><span style="color: #BA2121">"</span><span style="color: #BB6622; font-weight: bold">\r</span><span style="color: #BA2121">"</span>)
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">len</span>(line)
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">_fmt</span>(<span style="color: #008000">self</span>, value: <span style="color: #008000">int</span>, N<span style="color: #666666">=4</span>) <span style="color: #666666">-></span> <span style="color: #008000">str</span>:
|
||
<span style="color: #BA2121; font-style: italic">"""</span>
|
||
<span style="color: #BA2121; font-style: italic"> Description:</span>
|
||
<span style="color: #BA2121; font-style: italic"> ------------</span>
|
||
<span style="color: #BA2121; font-style: italic"> Formats decimal numbers for progress bar</span>
|
||
<span style="color: #BA2121; font-style: italic"> """</span>
|
||
<span style="color: #008000; font-weight: bold">if</span> value <span style="color: #666666">></span> <span style="color: #666666">0</span>:
|
||
v <span style="color: #666666">=</span> value
|
||
<span style="color: #008000; font-weight: bold">elif</span> value <span style="color: #666666"><</span> <span style="color: #666666">0</span>:
|
||
v <span style="color: #666666">=</span> <span style="color: #666666">-10</span> <span style="color: #666666">*</span> value
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
v <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
n <span style="color: #666666">=</span> <span style="color: #666666">1</span> <span style="color: #666666">+</span> math<span style="color: #666666">.</span>floor(math<span style="color: #666666">.</span>log10(v))
|
||
<span style="color: #008000; font-weight: bold">if</span> n <span style="color: #666666">>=</span> N <span style="color: #666666">-</span> <span style="color: #666666">1</span>:
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #008000">str</span>(<span style="color: #008000">round</span>(value))
|
||
<span style="color: #408080; font-style: italic"># or overflow</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> <span style="color: #BA2121">f"</span><span style="color: #BB6688; font-weight: bold">{</span>value<span style="color: #BB6688; font-weight: bold">:</span><span style="color: #BA2121">.</span><span style="color: #BB6688; font-weight: bold">{</span>N<span style="color: #666666">-</span>n<span style="color: #666666">-1</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">f</span><span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">"</span>
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="usage-of-cnn-code" class="anchor">Usage of CNN code </h3>
|
||
|
||
<p>Using the CNN codebase is very simple. We begin by initiating a CNN
|
||
object, which takes a cost function, a scheduler and a seed as its
|
||
arguments. If a scheduler is not provided, it will per default
|
||
initiate an Adam scheduler with eta=1e-4, and if a seed is not
|
||
provided, the CNN will not be seeded, meaning it will run with a
|
||
different random seed every run. Below we demonstrate an initiation of
|
||
our CNN.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">adam_scheduler <span style="color: #666666">=</span> Adam(eta<span style="color: #666666">=1e-3</span>, rho<span style="color: #666666">=0.9</span>, rho2<span style="color: #666666">=0.999</span>)
|
||
cnn <span style="color: #666666">=</span> CNN(cost_func<span style="color: #666666">=</span>CostCrossEntropy, scheduler<span style="color: #666666">=</span>adam_scheduler, seed<span style="color: #666666">=2023</span>)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>Now that we have our CNN object, we can begin to add layers to it!
|
||
Many of the add_layer functions have default values, for example
|
||
add_Convolution2DLayer() has a default v_stride and h_stride of
|
||
1. However, these can of course be set to any value you please. Note
|
||
that the input channels of a subsequent convolutional layer must equal
|
||
the previous convolutional layer's feature maps.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">cnn<span style="color: #666666">.</span>add_Convolution2DLayer(
|
||
input_channels<span style="color: #666666">=1</span>,
|
||
feature_maps<span style="color: #666666">=1</span>,
|
||
kernel_height<span style="color: #666666">=3</span>,
|
||
kernel_width<span style="color: #666666">=3</span>,
|
||
act_func<span style="color: #666666">=</span>LRELU,
|
||
)
|
||
|
||
cnn<span style="color: #666666">.</span>add_FlattenLayer()
|
||
|
||
cnn<span style="color: #666666">.</span>add_FullyConnectedLayer(<span style="color: #666666">30</span>, LRELU)
|
||
|
||
cnn<span style="color: #666666">.</span>add_FullyConnectedLayer(<span style="color: #666666">20</span>, LRELU)
|
||
|
||
cnn<span style="color: #666666">.</span>add_OutputLayer(<span style="color: #666666">10</span>, softmax)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>Here we have created a CNN with the following architecture:</p>
|
||
|
||
<ol>
|
||
<li> A convolutional layer with 1 input channel, with a kernel height of 2 and a width of 2, which uses LRELU as its non-linearity function. This layer outputs 1 feature map, which feed into the subsequent layer.</li>
|
||
<li> A flatten layer</li>
|
||
<li> A hidden layer with 30 nodes, with LRELU as its activation function</li>
|
||
<li> Another hidden layer but with 20 nodes</li>
|
||
<li> The output layer, with softmax as its activation function and 10 nodes. We use 10 nodes because we will be using a dataset with 10 classes.</li>
|
||
</ol>
|
||
<p>Now, before we can train the model, we need to load in our data. We
|
||
will use the MNIST dataset and use 10000 \( 28 \times 28 images \).
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> fetch_openml
|
||
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
||
|
||
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">onehot</span>(target: np<span style="color: #666666">.</span>ndarray):
|
||
onehot <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros((target<span style="color: #666666">.</span>size, target<span style="color: #666666">.</span>max() <span style="color: #666666">+</span> <span style="color: #666666">1</span>))
|
||
onehot[np<span style="color: #666666">.</span>arange(target<span style="color: #666666">.</span>size), target] <span style="color: #666666">=</span> <span style="color: #666666">1</span>
|
||
<span style="color: #008000; font-weight: bold">return</span> onehot
|
||
|
||
<span style="color: #408080; font-style: italic"># get dataset</span>
|
||
dataset <span style="color: #666666">=</span> fetch_openml(<span style="color: #BA2121">"mnist_784"</span>, parser<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||
mnist <span style="color: #666666">=</span> dataset<span style="color: #666666">.</span>data<span style="color: #666666">.</span>to_numpy(dtype<span style="color: #666666">=</span><span style="color: #BA2121">"float"</span>)[:<span style="color: #666666">10000</span>, :]
|
||
|
||
<span style="color: #408080; font-style: italic"># scale data</span>
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(mnist<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>]):
|
||
mnist[:, i] <span style="color: #666666">/=</span> <span style="color: #666666">255</span>
|
||
|
||
<span style="color: #408080; font-style: italic"># reshape to add single input channel to data shape [inputs, input_channels, height, width]</span>
|
||
mnist <span style="color: #666666">=</span> mnist<span style="color: #666666">.</span>reshape(mnist<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], <span style="color: #666666">1</span>, <span style="color: #666666">28</span>, <span style="color: #666666">28</span>)
|
||
|
||
<span style="color: #408080; font-style: italic"># one hot encode target as we are doing multi-class classification</span>
|
||
target <span style="color: #666666">=</span> onehot(np<span style="color: #666666">.</span>array([<span style="color: #008000">int</span>(i) <span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> dataset<span style="color: #666666">.</span>target<span style="color: #666666">.</span>to_numpy()[:<span style="color: #666666">10000</span>]]))
|
||
|
||
<span style="color: #408080; font-style: italic"># split into training and validation data</span>
|
||
x_train, x_val, y_train, y_val <span style="color: #666666">=</span> train_test_split(mnist, target)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>Now we may train our model. Note that we can utilize regularization in
|
||
the CNN by using the lam (lambda) parameter in fit(), and utilize
|
||
different types of gradient descent by specifying the amount of
|
||
batches via the batches parameter as shown below.
|
||
</p>
|
||
|
||
<p>The functionfit() returns a score dictionary of the training error and
|
||
accuracy (and validation error and accuracy if a validation set is
|
||
provided) which can be used to plot the error and accuracy of the
|
||
model over epochs.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">scores <span style="color: #666666">=</span> cnn<span style="color: #666666">.</span>fit(
|
||
x_train,
|
||
y_train,
|
||
lam<span style="color: #666666">=1e-5</span>,
|
||
batches<span style="color: #666666">=10</span>,
|
||
epochs<span style="color: #666666">=100</span>,
|
||
X_val<span style="color: #666666">=</span>x_val,
|
||
t_val<span style="color: #666666">=</span>y_val,
|
||
)
|
||
|
||
plt<span style="color: #666666">.</span>plot(scores[<span style="color: #BA2121">"train_acc"</span>], label<span style="color: #666666">=</span><span style="color: #BA2121">"Training"</span>)
|
||
plt<span style="color: #666666">.</span>plot(scores[<span style="color: #BA2121">"val_acc"</span>], label<span style="color: #666666">=</span><span style="color: #BA2121">"Validation"</span>)
|
||
plt<span style="color: #666666">.</span>ylim([<span style="color: #666666">0.8</span>,<span style="color: #666666">1</span>])
|
||
plt<span style="color: #666666">.</span>xlabel(<span style="color: #BA2121">"Epochs"</span>)
|
||
plt<span style="color: #666666">.</span>ylabel(<span style="color: #BA2121">"Accuracy"</span>)
|
||
plt<span style="color: #666666">.</span>legend()
|
||
plt<span style="color: #666666">.</span>show()
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>Considering we only trained the model for 100 epochs without any tuning of the hyperparameters, this result is pretty good.</p>
|
||
|
||
<p>The codebase allows for great flexibility in CNN
|
||
architectures. Pooling layers can be added before, inbetween or after
|
||
convolutional layers, but due to the great optimizations made within
|
||
Convolution2DLayerOPT, we recommend using the v_stride and h_stride
|
||
parameters in add_Convolution2DLayer() to reduce the dimentionality of
|
||
the problem as the pooling layer is slow in comparison. To use the
|
||
unoptimized version of Convolution2DLayer, simply pass optimized=False
|
||
as an argument in add_Convolution2DLayer().
|
||
</p>
|
||
|
||
<p>If one wishes to perform binary classification using the CNN, simply
|
||
use the cost function 'CostLogReg' when initializing the CNN and use 1
|
||
node at the OutputLayer.
|
||
</p>
|
||
|
||
<p>Below we have created another, more untraditional architecture using
|
||
our code to demonstrate its flexibility and different attributes such
|
||
as asymmetric stride that might become useful when constructing your
|
||
own CNN.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">adam_scheduler <span style="color: #666666">=</span> Adam(eta<span style="color: #666666">=1e-3</span>, rho<span style="color: #666666">=0.9</span>, rho2<span style="color: #666666">=0.999</span>)
|
||
cnn <span style="color: #666666">=</span> CNN(cost_func<span style="color: #666666">=</span>CostCrossEntropy, scheduler<span style="color: #666666">=</span>adam_scheduler, seed<span style="color: #666666">=2023</span>)
|
||
|
||
cnn<span style="color: #666666">.</span>add_Convolution2DLayer(
|
||
input_channels<span style="color: #666666">=1</span>,
|
||
feature_maps<span style="color: #666666">=7</span>,
|
||
kernel_height<span style="color: #666666">=7</span>,
|
||
kernel_width<span style="color: #666666">=1</span>,
|
||
act_func<span style="color: #666666">=</span>LRELU,
|
||
)
|
||
|
||
cnn<span style="color: #666666">.</span>add_PoolingLayer(
|
||
kernel_height<span style="color: #666666">=2</span>,
|
||
kernel_width<span style="color: #666666">=2</span>,
|
||
pooling<span style="color: #666666">=</span><span style="color: #BA2121">"average"</span>,
|
||
)
|
||
|
||
cnn<span style="color: #666666">.</span>add_PoolingLayer(
|
||
kernel_height<span style="color: #666666">=2</span>,
|
||
kernel_width<span style="color: #666666">=2</span>,
|
||
pooling<span style="color: #666666">=</span><span style="color: #BA2121">"max"</span>,
|
||
)
|
||
|
||
cnn<span style="color: #666666">.</span>add_Convolution2DLayer(
|
||
input_channels<span style="color: #666666">=7</span>,
|
||
feature_maps<span style="color: #666666">=1</span>,
|
||
kernel_height<span style="color: #666666">=4</span>,
|
||
kernel_width<span style="color: #666666">=4</span>,
|
||
v_stride<span style="color: #666666">=2</span>,
|
||
h_stride<span style="color: #666666">=3</span>,
|
||
act_func<span style="color: #666666">=</span>LRELU,
|
||
optimized<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">False</span>,
|
||
)
|
||
|
||
cnn<span style="color: #666666">.</span>add_Convolution2DLayer(
|
||
input_channels<span style="color: #666666">=1</span>,
|
||
feature_maps<span style="color: #666666">=1</span>,
|
||
kernel_height<span style="color: #666666">=2</span>,
|
||
kernel_width<span style="color: #666666">=2</span>,
|
||
act_func<span style="color: #666666">=</span>sigmoid,
|
||
optimized<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>,
|
||
)
|
||
|
||
cnn<span style="color: #666666">.</span>add_PoolingLayer(
|
||
kernel_height<span style="color: #666666">=2</span>,
|
||
kernel_width<span style="color: #666666">=2</span>,
|
||
pooling<span style="color: #666666">=</span><span style="color: #BA2121">"max"</span>
|
||
)
|
||
|
||
cnn<span style="color: #666666">.</span>add_FlattenLayer()
|
||
|
||
cnn<span style="color: #666666">.</span>add_FullyConnectedLayer(<span style="color: #666666">100</span>, LRELU)
|
||
|
||
cnn<span style="color: #666666">.</span>add_FullyConnectedLayer(<span style="color: #666666">10</span>, sigmoid)
|
||
|
||
cnn<span style="color: #666666">.</span>add_FullyConnectedLayer(<span style="color: #666666">101</span>, identity)
|
||
|
||
cnn<span style="color: #666666">.</span>add_OutputLayer(<span style="color: #666666">10</span>, softmax)
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>Here we see the use of asymmetrical 1D kernels such as the \( 7 \times
|
||
1 \) kernel in the first convolutional layer, both max and average
|
||
pooling, asymmetric stride in the unoptimized convolutional layer,
|
||
more pooling, a flatten layer, a hidden layer with 100 nodes using
|
||
LRELU, another hidden layer with 10 hidden nodes that uses the sigmoid
|
||
activation function, and another hidden layer with 101 nodes which
|
||
utilizes no activation function (identity). Finally, we arrive at the
|
||
output layer with 10 nodes, which uses softmax as its activation
|
||
function.
|
||
</p>
|
||
<h3 id="additional-remarks" class="anchor">Additional Remarks </h3>
|
||
|
||
<p>The stride parameter controls the distance between each convolution
|
||
and the kernel/filter. If our image is padded, stride is the only
|
||
parameter that determines the size of the output from a convolutional
|
||
layer. However, if we decide not to perform any padding, the size of
|
||
the output feature map depends on both the stride and kernel size. It
|
||
is important to note that neither the stride nor the kernel has to be
|
||
symmetrical. This means that we can use a rectangular filter if we
|
||
choose, and the stride in the vertical direction (axis=0 in Python)
|
||
does not need to be the same as the stride in the horizontal direction
|
||
(axis=1 in Python). It may even be the case that asymmetric
|
||
combinations of stride or kernel dimensions, or both, yield better
|
||
results than symmetric values for these parameters.
|
||
</p>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">convolve</span>(image, kernel, stride<span style="color: #666666">=1</span>):
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">2</span>):
|
||
kernel <span style="color: #666666">=</span> np<span style="color: #666666">.</span>rot90(kernel)
|
||
|
||
k_half_height <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
k_half_width <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
|
||
conv_image <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(image<span style="color: #666666">.</span>shape)
|
||
pad_image <span style="color: #666666">=</span> padding(image, kernel)
|
||
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(k_half_height, conv_image<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">+</span> k_half_height, stride):
|
||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(k_half_width, conv_image<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">+</span> k_half_width, stride):
|
||
conv_image[i <span style="color: #666666">-</span> k_half_height, j <span style="color: #666666">-</span> k_half_width] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>sum(
|
||
pad_image[
|
||
i <span style="color: #666666">-</span> k_half_height : i <span style="color: #666666">+</span> k_half_height <span style="color: #666666">+</span> <span style="color: #666666">1</span>, j <span style="color: #666666">-</span> k_half_width : j <span style="color: #666666">+</span> k_half_width <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
||
]
|
||
<span style="color: #666666">*</span> kernel
|
||
)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> conv_image
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<h3 id="remarks-on-the-speed" class="anchor">Remarks on the speed </h3>
|
||
|
||
<p>Despite the naive convolution algorithm shown above working finely, it
|
||
is extremely slow, requiring approximately 20-30 seconds to process a
|
||
single image. The time complexity of 2D convolution, which is O(NMnm),
|
||
rapidly becomes a constraint and may, at worst, make computations
|
||
infeasible. Consequently, optimizing the naive 2D convolution
|
||
algorithm is a necessity, as the execution time of the algorithm
|
||
significantly increases as the input data size expands. This can pose
|
||
a bottleneck in applications that necessitate real-time processing of
|
||
large data volumes, such as image and video processing, deep learning,
|
||
and scientific simulations.
|
||
</p>
|
||
|
||
<p>To address this issue, we shall present two widely used optimization
|
||
techniques: the separable kernel approach and Fast Fourier Transform
|
||
(FFT). Both of these methods can drastically reduce the computational
|
||
complexity of convolution and enhance the overall efficiency of
|
||
processing substantial data quantities. While we shall refrain from
|
||
delving into the intricacies of these algorithms, we strongly
|
||
encourage you to examine at least the application of FFT to optimize
|
||
computations.
|
||
</p>
|
||
<h3 id="convolution-using-separable-kernels" class="anchor">Convolution using separable kernels </h3>
|
||
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">conv2DSep</span>(image, kernel, coef, stride<span style="color: #666666">=1</span>, pad<span style="color: #666666">=</span><span style="color: #BA2121">"zero"</span>):
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">2</span>):
|
||
kernel <span style="color: #666666">=</span> np<span style="color: #666666">.</span>rot90(kernel)
|
||
|
||
<span style="color: #408080; font-style: italic"># The kernel is quadratic, thus we only need one of its dimensions</span>
|
||
half_dim <span style="color: #666666">=</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">//</span> <span style="color: #666666">2</span>
|
||
|
||
ker1 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(kernel[<span style="color: #666666">0</span>, :])
|
||
ker2 <span style="color: #666666">=</span> np<span style="color: #666666">.</span>array(kernel[:, <span style="color: #666666">0</span>])
|
||
|
||
<span style="color: #008000; font-weight: bold">if</span> pad <span style="color: #666666">==</span> <span style="color: #BA2121">"zero"</span>:
|
||
conv_image <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(image<span style="color: #666666">.</span>shape)
|
||
pad_image <span style="color: #666666">=</span> padding(image, kernel)
|
||
<span style="color: #008000; font-weight: bold">else</span>:
|
||
conv_image <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(
|
||
(image<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">-</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>], image<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">-</span> kernel<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>])
|
||
)
|
||
pad_image <span style="color: #666666">=</span> image[:, :]
|
||
|
||
<span style="color: #008000; font-weight: bold">for</span> i <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(half_dim, conv_image<span style="color: #666666">.</span>shape[<span style="color: #666666">0</span>] <span style="color: #666666">+</span> half_dim, stride):
|
||
<span style="color: #008000; font-weight: bold">for</span> j <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(half_dim, conv_image<span style="color: #666666">.</span>shape[<span style="color: #666666">1</span>] <span style="color: #666666">+</span> half_dim, stride):
|
||
conv_image[i <span style="color: #666666">-</span> half_dim, j <span style="color: #666666">-</span> half_dim] <span style="color: #666666">=</span> (
|
||
pad_image[
|
||
i <span style="color: #666666">-</span> half_dim : i <span style="color: #666666">+</span> half_dim <span style="color: #666666">+</span> <span style="color: #666666">1</span>, j <span style="color: #666666">-</span> half_dim : j <span style="color: #666666">+</span> half_dim <span style="color: #666666">+</span> <span style="color: #666666">1</span>
|
||
]
|
||
<span style="color: #666666">@</span> ker1
|
||
<span style="color: #666666">@</span> ker2<span style="color: #666666">.</span>T
|
||
<span style="color: #666666">*</span> coef
|
||
)
|
||
|
||
<span style="color: #008000; font-weight: bold">return</span> conv_image
|
||
|
||
img_path <span style="color: #666666">=</span> img_path <span style="color: #666666">=</span> <span style="color: #BA2121">"data/IMG-2167.JPG"</span>
|
||
image_of_cute_dog <span style="color: #666666">=</span> imageio<span style="color: #666666">.</span>imread(img_path, mode<span style="color: #666666">=</span><span style="color: #BA2121">"L"</span>)
|
||
start_time <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||
filtered_image <span style="color: #666666">=</span> conv2DSep(image_of_cute_dog, kernel<span style="color: #666666">=</span>sobel_kernel, coef<span style="color: #666666">=1</span>)
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Time taken for convolution with seperated kernel on 128x128 image </span><span style="color: #BB6688; font-weight: bold">{</span>time<span style="color: #666666">.</span>time() <span style="color: #666666">-</span> start_time<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">'</span>)
|
||
plt<span style="color: #666666">.</span>imshow(filtered_image, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"gray"</span>, vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=255</span>, aspect<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||
plt<span style="color: #666666">.</span>show()
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>By taking advantage of the capabilities of separable kernels, we can
|
||
effectively cut the computational expense of filtering an image in
|
||
half. Yet, if we seek even more rapid processing, we can turn to the
|
||
Fast Fourier Transform (FFT) algorithm provided by the numpy
|
||
library. By utilizing FFT to transform the input image and filter into
|
||
the frequency domain, we can perform convolution in this domain. This
|
||
approach significantly reduces the number of operations needed and
|
||
results in a marked speedup relative to other convolution
|
||
techniques. In addition, it is worth noting that the FFT is widely
|
||
regarded as one of the most critical algorithms developed to date,
|
||
with applications ranging from digital signal processing to scientific
|
||
computing.
|
||
</p>
|
||
<h3 id="convolution-in-the-fourier-domain" class="anchor">Convolution in the Fourier domain </h3>
|
||
|
||
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
||
<div class="cell border-box-sizing code_cell rendered">
|
||
<div class="input">
|
||
<div class="inner_cell">
|
||
<div class="input_area">
|
||
<div class="highlight" style="background: #f8f8f8">
|
||
<pre style="line-height: 125%;">start_time <span style="color: #666666">=</span> time<span style="color: #666666">.</span>time()
|
||
img_fft <span style="color: #666666">=</span> np<span style="color: #666666">.</span>fft<span style="color: #666666">.</span>fft2(image_of_cute_dog)
|
||
kernel_fft <span style="color: #666666">=</span> np<span style="color: #666666">.</span>fft<span style="color: #666666">.</span>fft2(sobel_kernel, s<span style="color: #666666">=</span>image_of_cute_dog<span style="color: #666666">.</span>shape)
|
||
|
||
conv_image <span style="color: #666666">=</span> img_fft <span style="color: #666666">*</span> kernel_fft
|
||
|
||
filtered_image <span style="color: #666666">=</span> np<span style="color: #666666">.</span>fft<span style="color: #666666">.</span>ifft2(conv_image)
|
||
<span style="color: #008000">print</span>(<span style="color: #BA2121">f'Time take for convolution in the fourier domain: </span><span style="color: #BB6688; font-weight: bold">{</span>time<span style="color: #666666">.</span>time() <span style="color: #666666">-</span> start_time<span style="color: #BB6688; font-weight: bold">}</span><span style="color: #BA2121">'</span>)
|
||
plt<span style="color: #666666">.</span>imshow(filtered_image<span style="color: #666666">.</span>real, cmap<span style="color: #666666">=</span><span style="color: #BA2121">"gray"</span>, vmin<span style="color: #666666">=0</span>, vmax<span style="color: #666666">=255</span>, aspect<span style="color: #666666">=</span><span style="color: #BA2121">"auto"</span>)
|
||
plt<span style="color: #666666">.</span>show()
|
||
</pre>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="output_wrapper">
|
||
<div class="output">
|
||
<div class="output_area">
|
||
<div class="output_subarea output_stream output_stdout output_text">
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<p>It is evident that executing convolution in the Fourier domain yields
|
||
the quickest computation time. Nonetheless, one should exercise
|
||
caution, particularly when dealing with images of relatively small
|
||
dimensions, as one of the other methods may prove to be more
|
||
expeditious than FFT-enhanced convolution. The overhead involved in
|
||
transferring both the image and filter into the Fourier domain,
|
||
followed by their subsequent transformation back into the spatial
|
||
domain, results in a minor inconvenience. Therefore, it is imperative
|
||
to remain cognizant of this fact when utilizing FFT as the primary
|
||
optimization technique.
|
||
</p>
|
||
|
||
<p>
|
||
<!-- navigation buttons at the bottom of the page -->
|
||
<ul class="pagination">
|
||
<li><a href="._week44-bs047.html">«</a></li>
|
||
<li><a href="._week44-bs000.html">1</a></li>
|
||
<li><a href="">...</a></li>
|
||
<li><a href="._week44-bs040.html">41</a></li>
|
||
<li><a href="._week44-bs041.html">42</a></li>
|
||
<li><a href="._week44-bs042.html">43</a></li>
|
||
<li><a href="._week44-bs043.html">44</a></li>
|
||
<li><a href="._week44-bs044.html">45</a></li>
|
||
<li><a href="._week44-bs045.html">46</a></li>
|
||
<li><a href="._week44-bs046.html">47</a></li>
|
||
<li><a href="._week44-bs047.html">48</a></li>
|
||
<li class="active"><a href="._week44-bs048.html">49</a></li>
|
||
</ul>
|
||
<!-- ------------------- end of main content --------------- -->
|
||
</div> <!-- end container -->
|
||
<!-- include javascript, jQuery *first* -->
|
||
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
||
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
||
<!-- Bootstrap footer
|
||
<footer>
|
||
<a href="https://..."><img width="250" align=right src="https://..."></a>
|
||
</footer>
|
||
-->
|
||
<center style="font-size:80%">
|
||
<!-- copyright only on the titlepage -->
|
||
</center>
|
||
</body>
|
||
</html>
|
||
|