
% Default to the notebook output style

    


% Inherit from the specified cell style.




    
\documentclass[11pt]{article}

    
    
    \usepackage[T1]{fontenc}
    % Nicer default font (+ math font) than Computer Modern for most use cases
    \usepackage{mathpazo}

    % Basic figure setup, for now with no caption control since it's done
    % automatically by Pandoc (which extracts ![](path) syntax from Markdown).
    \usepackage{graphicx}
    % We will generate all images so they have a width \maxwidth. This means
    % that they will get their normal width if they fit onto the page, but
    % are scaled down if they would overflow the margins.
    \makeatletter
    \def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth
    \else\Gin@nat@width\fi}
    \makeatother
    \let\Oldincludegraphics\includegraphics
    % Set max figure width to be 80% of text width, for now hardcoded.
    \renewcommand{\includegraphics}[1]{\Oldincludegraphics[width=.8\maxwidth]{#1}}
    % Ensure that by default, figures have no caption (until we provide a
    % proper Figure object with a Caption API and a way to capture that
    % in the conversion process - todo).
    \usepackage{caption}
    \DeclareCaptionLabelFormat{nolabel}{}
    \captionsetup{labelformat=nolabel}

    \usepackage{adjustbox} % Used to constrain images to a maximum size 
    \usepackage{xcolor} % Allow colors to be defined
    \usepackage{enumerate} % Needed for markdown enumerations to work
    \usepackage{geometry} % Used to adjust the document margins
    \usepackage{amsmath} % Equations
    \usepackage{amssymb} % Equations
    \usepackage{textcomp} % defines textquotesingle
    % Hack from http://tex.stackexchange.com/a/47451/13684:
    \AtBeginDocument{%
        \def\PYZsq{\textquotesingle}% Upright quotes in Pygmentized code
    }
    \usepackage{upquote} % Upright quotes for verbatim code
    \usepackage{eurosym} % defines \euro
    \usepackage[mathletters]{ucs} % Extended unicode (utf-8) support
    \usepackage[utf8x]{inputenc} % Allow utf-8 characters in the tex document
    \usepackage{fancyvrb} % verbatim replacement that allows latex
    \usepackage{grffile} % extends the file name processing of package graphics 
                         % to support a larger range 
    % The hyperref package gives us a pdf with properly built
    % internal navigation ('pdf bookmarks' for the table of contents,
    % internal cross-reference links, web links for URLs, etc.)
    \usepackage{hyperref}
    \usepackage{longtable} % longtable support required by pandoc >1.10
    \usepackage{booktabs}  % table support for pandoc > 1.12.2
    \usepackage[inline]{enumitem} % IRkernel/repr support (it uses the enumerate* environment)
    \usepackage[normalem]{ulem} % ulem is needed to support strikethroughs (\sout)
                                % normalem makes italics be italics, not underlines
    

    
    
    % Colors for the hyperref package
    \definecolor{urlcolor}{rgb}{0,.145,.698}
    \definecolor{linkcolor}{rgb}{.71,0.21,0.01}
    \definecolor{citecolor}{rgb}{.12,.54,.11}

    % ANSI colors
    \definecolor{ansi-black}{HTML}{3E424D}
    \definecolor{ansi-black-intense}{HTML}{282C36}
    \definecolor{ansi-red}{HTML}{E75C58}
    \definecolor{ansi-red-intense}{HTML}{B22B31}
    \definecolor{ansi-green}{HTML}{00A250}
    \definecolor{ansi-green-intense}{HTML}{007427}
    \definecolor{ansi-yellow}{HTML}{DDB62B}
    \definecolor{ansi-yellow-intense}{HTML}{B27D12}
    \definecolor{ansi-blue}{HTML}{208FFB}
    \definecolor{ansi-blue-intense}{HTML}{0065CA}
    \definecolor{ansi-magenta}{HTML}{D160C4}
    \definecolor{ansi-magenta-intense}{HTML}{A03196}
    \definecolor{ansi-cyan}{HTML}{60C6C8}
    \definecolor{ansi-cyan-intense}{HTML}{258F8F}
    \definecolor{ansi-white}{HTML}{C5C1B4}
    \definecolor{ansi-white-intense}{HTML}{A1A6B2}

    % commands and environments needed by pandoc snippets
    % extracted from the output of `pandoc -s`
    \providecommand{\tightlist}{%
      \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
    \DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
    % Add ',fontsize=\small' for more characters per line
    \newenvironment{Shaded}{}{}
    \newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
    \newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}}
    \newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
    \newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
    \newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
    \newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
    \newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
    \newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}}
    \newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
    \newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
    \newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}}
    \newcommand{\RegionMarkerTok}[1]{{#1}}
    \newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
    \newcommand{\NormalTok}[1]{{#1}}
    
    % Additional commands for more recent versions of Pandoc
    \newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.53,0.00,0.00}{{#1}}}
    \newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
    \newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
    \newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.73,0.40,0.53}{{#1}}}
    \newcommand{\ImportTok}[1]{{#1}}
    \newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.73,0.13,0.13}{\textit{{#1}}}}
    \newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
    \newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
    \newcommand{\VariableTok}[1]{\textcolor[rgb]{0.10,0.09,0.49}{{#1}}}
    \newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
    \newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.40,0.40,0.40}{{#1}}}
    \newcommand{\BuiltInTok}[1]{{#1}}
    \newcommand{\ExtensionTok}[1]{{#1}}
    \newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.74,0.48,0.00}{{#1}}}
    \newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.49,0.56,0.16}{{#1}}}
    \newcommand{\InformationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
    \newcommand{\WarningTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
    
    
    % Define a nice break command that doesn't care if a line doesn't already
    % exist.
    \def\br{\hspace*{\fill} \\* }
    % Math Jax compatability definitions
    \def\gt{>}
    \def\lt{<}
    % Document parameters
    \title{NeuralNet}
    
    
    

    % Pygments definitions
    
\makeatletter
\def\PY@reset{\let\PY@it=\relax \let\PY@bf=\relax%
    \let\PY@ul=\relax \let\PY@tc=\relax%
    \let\PY@bc=\relax \let\PY@ff=\relax}
\def\PY@tok#1{\csname PY@tok@#1\endcsname}
\def\PY@toks#1+{\ifx\relax#1\empty\else%
    \PY@tok{#1}\expandafter\PY@toks\fi}
\def\PY@do#1{\PY@bc{\PY@tc{\PY@ul{%
    \PY@it{\PY@bf{\PY@ff{#1}}}}}}}
\def\PY#1#2{\PY@reset\PY@toks#1+\relax+\PY@do{#2}}

\expandafter\def\csname PY@tok@w\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
\expandafter\def\csname PY@tok@c\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@cp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.74,0.48,0.00}{##1}}}
\expandafter\def\csname PY@tok@k\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@kp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@kt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.69,0.00,0.25}{##1}}}
\expandafter\def\csname PY@tok@o\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@ow\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}}
\expandafter\def\csname PY@tok@nb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@nf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
\expandafter\def\csname PY@tok@nc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
\expandafter\def\csname PY@tok@nn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
\expandafter\def\csname PY@tok@ne\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.82,0.25,0.23}{##1}}}
\expandafter\def\csname PY@tok@nv\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@no\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}}
\expandafter\def\csname PY@tok@nl\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.63,0.00}{##1}}}
\expandafter\def\csname PY@tok@ni\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.60,0.60,0.60}{##1}}}
\expandafter\def\csname PY@tok@na\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.49,0.56,0.16}{##1}}}
\expandafter\def\csname PY@tok@nt\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@nd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}}
\expandafter\def\csname PY@tok@s\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@sd\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@si\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}}
\expandafter\def\csname PY@tok@se\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.13}{##1}}}
\expandafter\def\csname PY@tok@sr\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}}
\expandafter\def\csname PY@tok@ss\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@sx\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@m\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@gh\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
\expandafter\def\csname PY@tok@gu\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}}
\expandafter\def\csname PY@tok@gd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
\expandafter\def\csname PY@tok@gi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
\expandafter\def\csname PY@tok@gr\endcsname{\def\PY@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
\expandafter\def\csname PY@tok@ge\endcsname{\let\PY@it=\textit}
\expandafter\def\csname PY@tok@gs\endcsname{\let\PY@bf=\textbf}
\expandafter\def\csname PY@tok@gp\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
\expandafter\def\csname PY@tok@go\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
\expandafter\def\csname PY@tok@gt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}}
\expandafter\def\csname PY@tok@err\endcsname{\def\PY@bc##1{\setlength{\fboxsep}{0pt}\fcolorbox[rgb]{1.00,0.00,0.00}{1,1,1}{\strut ##1}}}
\expandafter\def\csname PY@tok@kc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@kd\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@kn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@kr\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@bp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
\expandafter\def\csname PY@tok@fm\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
\expandafter\def\csname PY@tok@vc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@vg\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@vi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@vm\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
\expandafter\def\csname PY@tok@sa\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@sb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@sc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@dl\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@s2\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@sh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@s1\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
\expandafter\def\csname PY@tok@mb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@mf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@mh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@mi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@il\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@mo\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
\expandafter\def\csname PY@tok@ch\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@cm\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@cpf\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@c1\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
\expandafter\def\csname PY@tok@cs\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}

\def\PYZbs{\char`\\}
\def\PYZus{\char`\_}
\def\PYZob{\char`\{}
\def\PYZcb{\char`\}}
\def\PYZca{\char`\^}
\def\PYZam{\char`\&}
\def\PYZlt{\char`\<}
\def\PYZgt{\char`\>}
\def\PYZsh{\char`\#}
\def\PYZpc{\char`\%}
\def\PYZdl{\char`\$}
\def\PYZhy{\char`\-}
\def\PYZsq{\char`\'}
\def\PYZdq{\char`\"}
\def\PYZti{\char`\~}
% for compatibility with earlier versions
\def\PYZat{@}
\def\PYZlb{[}
\def\PYZrb{]}
\makeatother


    % Exact colors from NB
    \definecolor{incolor}{rgb}{0.0, 0.0, 0.5}
    \definecolor{outcolor}{rgb}{0.545, 0.0, 0.0}



    
    % Prevent overflowing lines due to hard-to-break entities
    \sloppy 
    % Setup hyperref package
    \hypersetup{
      breaklinks=true,  % so long urls are correctly broken across lines
      colorlinks=true,
      urlcolor=urlcolor,
      linkcolor=linkcolor,
      citecolor=citecolor,
      }
    % Slightly bigger margins than the latex defaults
    
    \geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in}
    
    

    \begin{document}
    
    
    \maketitle
    
    

    
    \hypertarget{data-analysis-and-machine-learning-neural-networks-from-the-simple-perceptron-to-deep-learning}{%
\section{Data Analysis and Machine Learning: Neural networks, from the
simple perceptron to deep
learning}\label{data-analysis-and-machine-learning-neural-networks-from-the-simple-perceptron-to-deep-learning}}

\textbf{Morten Hjorth-Jensen}, Department of Physics, University of Oslo
and Department of Physics and Astronomy and National Superconducting
Cyclotron Laboratory, Michigan State University

Date: \textbf{Oct 4, 2018}

Copyright 1999-2018, Morten Hjorth-Jensen. Released under CC
Attribution-NonCommercial 4.0 license

\hypertarget{neural-networks}{%
\subsection{Neural networks}\label{neural-networks}}

Artificial neural networks are computational systems that can learn to
perform tasks by considering examples, generally without being
programmed with any task-specific rules. It is supposed to mimic a
biological system, wherein neurons interact by sending signals in the
form of mathematical functions between layers. All layers can contain an
arbitrary number of neurons, and each connection is represented by a
weight variable.

\hypertarget{artificial-neurons}{%
\subsection{Artificial neurons}\label{artificial-neurons}}

The field of artificial neural networks has a long history of
development, and is closely connected with the advancement of computer
science and computers in general. A model of artificial neurons was
first developed by McCulloch and Pitts in 1943 to study signal
processing in the brain and has later been refined by others. The
general idea is to mimic neural networks in the human brain, which is
composed of billions of neurons that communicate with each other by
sending electrical signals. Each neuron accumulates its incoming
signals, which must exceed an activation threshold to yield an output.
If the threshold is not overcome, the neuron remains inactive, i.e.~has
zero output.

This behaviour has inspired a simple mathematical model for an
artificial neuron.

    \hypertarget{artificialNeuron}{}

\[
\begin{equation}
 y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
\label{artificialNeuron} \tag{1}
\end{equation}
\]

    Here, the output \(y\) of the neuron is the value of its activation
function, which have as input a weighted sum of signals
\(x_i, \dots ,x_n\) received by \(n\) other neurons.

Conceptually, it is helpful to divide neural networks into four
categories: 1. general purpose neural networks for supervised learning,

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{1}
\item
  neural networks designed specifically for image processing, the most
  prominent example of this class being Convolutional Neural Networks
  (CNNs),
\item
  neural networks for sequential data such as Recurrent Neural Networks
  (RNNs), and
\item
  neural networks for unsupervised learning such as Deep Boltzmann
  Machines.
\end{enumerate}

In natural science, DNNs and CNNs have already found numerous
applications. In statistical physics, they have been applied to detect
phase transitions in 2D Ising and Potts models, lattice gauge theories,
and different phases of polymers, or solving the Navier-Stokes equation
in weather forecasting. Deep learning has also found interesting
applications in quantum physics. Various quantum phase transitions can
be detected and studied using DNNs and CNNs, topological phases, and
even non-equilibrium many-body localization. Representing quantum states
as DNNs quantum state tomography are among some of the impressive
achievements to reveal the potential of DNNs to facilitate the study of
quantum systems.

In quantum information theory, it has been shown that one can perform
gate decompositions with the help of neural.

The applications are not limited to the natural sciences. There is a
plethora of applications in essentially all disciplines, from the
humanities to life science and medicine.

\hypertarget{neural-network-types}{%
\subsection{Neural network types}\label{neural-network-types}}

An artificial neural network (ANN), is a computational model that
consists of layers of connected neurons, or nodes or units. We will
refer to these interchangeably as units or nodes, and sometimes as
neurons.

It is supposed to mimic a biological nervous system by letting each
neuron interact with other neurons by sending signals in the form of
mathematical functions between layers. A wide variety of different ANNs
have been developed, but most of them consist of an input layer, an
output layer and eventual layers in-between, called \emph{hidden
layers}. All layers can contain an arbitrary number of nodes, and each
connection between two nodes is associated with a weight variable.

Neural networks (also called neural nets) are neural-inspired nonlinear
models for supervised learning. As we will see, neural nets can be
viewed as natural, more powerful extensions of supervised learning
methods such as linear and logistic regression and soft-max methods we
discussed earlier.

\hypertarget{feed-forward-neural-networks}{%
\subsection{Feed-forward neural
networks}\label{feed-forward-neural-networks}}

The feed-forward neural network (FFNN) was the first and simplest type
of ANNs that were devised. In this network, the information moves in
only one direction: forward through the layers.

Nodes are represented by circles, while the arrows display the
connections between the nodes, including the direction of information
flow. Additionally, each arrow corresponds to a weight variable (figure
to come). We observe that each node in a layer is connected to
\emph{all} nodes in the subsequent layer, making this a so-called
\emph{fully-connected} FFNN.

\hypertarget{convolutional-neural-network}{%
\subsection{Convolutional Neural
Network}\label{convolutional-neural-network}}

A different variant of FFNNs are \emph{convolutional neural networks}
(CNNs), which have a connectivity pattern inspired by the animal visual
cortex. Individual neurons in the visual cortex only respond to stimuli
from small sub-regions of the visual field, called a receptive field.
This makes the neurons well-suited to exploit the strong spatially local
correlation present in natural images. The response of each neuron can
be approximated mathematically as a convolution operation. (figure to
come)

Convolutional neural networks emulate the behaviour of neurons in the
visual cortex by enforcing a \emph{local} connectivity pattern between
nodes of adjacent layers: Each node in a convolutional layer is
connected only to a subset of the nodes in the previous layer, in
contrast to the fully-connected FFNN. Often, CNNs consist of several
convolutional layers that learn local features of the input, with a
fully-connected layer at the end, which gathers all the local data and
produces the outputs. They have wide applications in image and video
recognition.

\hypertarget{recurrent-neural-networks}{%
\subsection{Recurrent neural networks}\label{recurrent-neural-networks}}

So far we have only mentioned ANNs where information flows in one
direction: forward. \emph{Recurrent neural networks} on the other hand,
have connections between nodes that form directed \emph{cycles}. This
creates a form of internal memory which are able to capture information
on what has been calculated before; the output is dependent on the
previous computations. Recurrent NNs make use of sequential information
by performing the same task for every element in a sequence, where each
element depends on previous elements. An example of such information is
sentences, making recurrent NNs especially well-suited for handwriting
and speech recognition.

\hypertarget{other-types-of-networks}{%
\subsection{Other types of networks}\label{other-types-of-networks}}

There are many other kinds of ANNs that have been developed. One type
that is specifically designed for interpolation in multidimensional
space is the radial basis function (RBF) network. RBFs are typically
made up of three layers: an input layer, a hidden layer with non-linear
radial symmetric activation functions and a linear output layer
(`'linear'' here means that each node in the output layer has a linear
activation function). The layers are normally fully-connected and there
are no cycles, thus RBFs can be viewed as a type of fully-connected
FFNN. They are however usually treated as a separate type of NN due the
unusual activation functions.

\hypertarget{multilayer-perceptrons}{%
\subsection{Multilayer perceptrons}\label{multilayer-perceptrons}}

One uses often so-called fully-connected feed-forward neural networks
with three or more layers (an input layer, one or more hidden layers and
an output layer) consisting of neurons that have non-linear activation
functions.

Such networks are often called \emph{multilayer perceptrons} (MLPs).

\hypertarget{why-multilayer-perceptrons}{%
\subsection{Why multilayer
perceptrons?}\label{why-multilayer-perceptrons}}

According to the \emph{Universal approximation theorem}, a feed-forward
neural network with just a single hidden layer containing a finite
number of neurons can approximate a continuous multidimensional function
to arbitrary accuracy, assuming the activation function for the hidden
layer is a \textbf{non-constant, bounded and monotonically-increasing
continuous function}.

Note that the requirements on the activation function only applies to
the hidden layer, the output nodes are always assumed to be linear, so
as to not restrict the range of output values.

\hypertarget{mathematical-model}{%
\subsection{Mathematical model}\label{mathematical-model}}

The output \(y\) is produced via the activation function \(f\)

    \[
y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
\]

    This function receives \(x_i\) as inputs. Here the activation
\(z=(\sum_{i=1}^n w_ix_i+b_i)\). In an FFNN of such neurons, the
\emph{inputs} \(x_i\) are the \emph{outputs} of the neurons in the
preceding layer. Furthermore, an MLP is fully-connected, which means
that each neuron receives a weighted sum of the outputs of \emph{all}
neurons in the previous layer.

\hypertarget{mathematical-model}{%
\subsection{Mathematical model}\label{mathematical-model}}

First, for each node \(i\) in the first hidden layer, we calculate a
weighted sum \(z_i^1\) of the input coordinates \(x_j\),

    \hypertarget{_auto1}{}

\[
\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
\label{_auto1} \tag{2}
\end{equation}
\]

    Here \(b_i\) is the so-called bias which is normally needed in case of
zero activation weights or inputs. How to fix the biases and the weights
will be discussed below. The value of \(z_i^1\) is the argument to the
activation function \(f_i\) of each node \(i\), The variable \(M\)
stands for all possible inputs to a given node \(i\) in the first layer.
We define the output \(y_i^1\) of all neurons in layer 1 as

    \hypertarget{outputLayer1}{}

\[
\begin{equation}
 y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j  + b_i^1\right)
\label{outputLayer1} \tag{3}
\end{equation}
\]

    where we assume that all nodes in the same layer have identical
activation functions, hence the notation \(f\). In general, we could
assume in the more general case that different layers have different
activation functions. In this case we would identify these functions
with a superscript \(l\) for the \(l\)-th layer,

    \hypertarget{generalLayer}{}

\[
\begin{equation}
 y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
\label{generalLayer} \tag{4}
\end{equation}
\]

    where \(N_l\) is the number of nodes in layer \(l\). When the output of
all the nodes in the first hidden layer are computed, the values of the
subsequent layer can be calculated and so forth until the output is
obtained.

\hypertarget{mathematical-model}{%
\subsection{Mathematical model}\label{mathematical-model}}

The output of neuron \(i\) in layer 2 is thus,

    \hypertarget{_auto2}{}

\[
\begin{equation}
 y_i^2 = f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right) 
\label{_auto2} \tag{5}
\end{equation}
\]

    \hypertarget{outputLayer2}{}

\[
\begin{equation} 
 = f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
\label{outputLayer2} \tag{6}
\end{equation}
\]

    where we have substituted \(y_k^1\) with the inputs \(x_k\). Finally,
the ANN output reads

    \hypertarget{_auto3}{}

\[
\begin{equation}
 y_i^3 = f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right) 
\label{_auto3} \tag{7}
\end{equation}
\]

    \hypertarget{_auto4}{}

\[
\begin{equation} 
 = f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
  + b_1^3\right]
\label{_auto4} \tag{8}
\end{equation}
\]

    \hypertarget{mathematical-model}{%
\subsection{Mathematical model}\label{mathematical-model}}

We can generalize this expression to an MLP with \(l\) hidden layers.
The complete functional form is,

    \hypertarget{completeNN}{}

\[
\begin{equation}
y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right] 
\label{completeNN} \tag{9}
\end{equation}
\]

    which illustrates a basic property of MLPs: The only independent
variables are the input values \(x_n\).

\hypertarget{mathematical-model}{%
\subsection{Mathematical model}\label{mathematical-model}}

This confirms that an MLP, despite its quite convoluted mathematical
form, is nothing more than an analytic function, specifically a mapping
of real-valued vectors
\(\hat{x} \in \mathbb{R}^n \rightarrow \hat{y} \in \mathbb{R}^m\).

Furthermore, the flexibility and universality of an MLP can be
illustrated by realizing that the expression is essentially a nested sum
of scaled activation functions of the form

    \hypertarget{_auto5}{}

\[
\begin{equation}
 f(x) = c_1 f(c_2 x + c_3) + c_4
\label{_auto5} \tag{10}
\end{equation}
\]

    where the parameters \(c_i\) are weights and biases. By adjusting these
parameters, the activation functions can be shifted up and down or left
and right, change slope or be rescaled which is the key to the
flexibility of a neural network.

\hypertarget{matrix-vector-notation}{%
\subsubsection{Matrix-vector notation}\label{matrix-vector-notation}}

We can introduce a more convenient notation for the activations in an A
NN.

Additionally, we can represent the biases and activations as layer-wise
column vectors \(\hat{b}_l\) and \(\hat{y}_l\), so that the \(i\)-th
element of each vector is the bias \(b_i^l\) and activation \(y_i^l\) of
node \(i\) in layer \(l\) respectively.

We have that \(\mathrm{W}_l\) is an \(N_{l-1} \times N_l\) matrix, while
\(\hat{b}_l\) and \(\hat{y}_l\) are \(N_l \times 1\) column vectors.
With this notation, the sum becomes a matrix-vector multiplication, and
we can write the equation for the activations of hidden layer 2
(assuming three nodes for simplicity) as

    \hypertarget{_auto6}{}

\[
\begin{equation}
 \hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) = 
 f_2\left(\left[\begin{array}{ccc}
    w^2_{11} &w^2_{12} &w^2_{13} \\
    w^2_{21} &w^2_{22} &w^2_{23} \\
    w^2_{31} &w^2_{32} &w^2_{33} \\
    \end{array} \right] \cdot
    \left[\begin{array}{c}
           y^1_1 \\
           y^1_2 \\
           y^1_3 \\
          \end{array}\right] + 
    \left[\begin{array}{c}
           b^2_1 \\
           b^2_2 \\
           b^2_3 \\
          \end{array}\right]\right).
\label{_auto6} \tag{11}
\end{equation}
\]

    \hypertarget{matrix-vector-notation-and-activation}{%
\subsubsection{Matrix-vector notation and
activation}\label{matrix-vector-notation-and-activation}}

The activation of node \(i\) in layer 2 is

    \hypertarget{_auto7}{}

\[
\begin{equation}
 y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) = 
 f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
\label{_auto7} \tag{12}
\end{equation}
\]

    This is not just a convenient and compact notation, but also a useful
and intuitive way to think about MLPs: The output is calculated by a
series of matrix-vector multiplications and vector additions that are
used as input to the activation functions. For each operation
\(\mathrm{W}_l \hat{y}_{l-1}\) we move forward one layer.

\hypertarget{activation-functions}{%
\subsubsection{Activation functions}\label{activation-functions}}

A property that characterizes a neural network, other than its
connectivity, is the choice of activation function(s). As described in,
the following restrictions are imposed on an activation function for a
FFNN to fulfill the universal approximation theorem

\begin{itemize}
\item
  Non-constant
\item
  Bounded
\item
  Monotonically-increasing
\item
  Continuous
\end{itemize}

\hypertarget{activation-functions-logistic-and-hyperbolic-ones}{%
\subsubsection{Activation functions, Logistic and Hyperbolic
ones}\label{activation-functions-logistic-and-hyperbolic-ones}}

The second requirement excludes all linear functions. Furthermore, in a
MLP with only linear activation functions, each layer simply performs a
linear transformation of its inputs.

Regardless of the number of layers, the output of the NN will be nothing
but a linear function of the inputs. Thus we need to introduce some kind
of non-linearity to the NN to be able to fit non-linear functions
Typical examples are the logistic \emph{Sigmoid}

    \[
f(x) = \frac{1}{1 + e^{-x}},
\]

    and the \emph{hyperbolic tangent} function

    \[
f(x) = \tanh(x)
\]

    \hypertarget{relevance}{%
\subsubsection{Relevance}\label{relevance}}

The \emph{sigmoid} function are more biologically plausible because the
output of inactive neurons are zero. Such activation function are called
\emph{one-sided}. However, it has been shown that the hyperbolic tangent
performs better than the sigmoid for training MLPs. has become the most
popular for \emph{deep neural networks}

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}1}]:} \PY{o}{\PYZpc{}}\PY{k}{matplotlib} inline
        
        \PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}The sigmoid function (or the logistic curve) is a }
        \PY{l+s+sd}{function that takes any real number, z, and outputs a number (0,1).}
        \PY{l+s+sd}{It is useful in neural networks for assigning weights on a relative scale.}
        \PY{l+s+sd}{The value z is the weighted sum of parameters involved in the learning algorithm.\PYZdq{}\PYZdq{}\PYZdq{}}
        
        \PY{k+kn}{import} \PY{n+nn}{numpy}
        \PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt}
        \PY{k+kn}{import} \PY{n+nn}{math} \PY{k}{as} \PY{n+nn}{mt}
        
        \PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{o}{.}\PY{l+m+mi}{1}\PY{p}{)}
        \PY{n}{sigma\PYZus{}fn} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{vectorize}\PY{p}{(}\PY{k}{lambda} \PY{n}{z}\PY{p}{:} \PY{l+m+mi}{1}\PY{o}{/}\PY{p}{(}\PY{l+m+mi}{1}\PY{o}{+}\PY{n}{numpy}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{n}{z}\PY{p}{)}\PY{p}{)}\PY{p}{)}
        \PY{n}{sigma} \PY{o}{=} \PY{n}{sigma\PYZus{}fn}\PY{p}{(}\PY{n}{z}\PY{p}{)}
        
        \PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
        \PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{sigma}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{0.1}\PY{p}{,} \PY{l+m+mf}{1.1}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,}\PY{l+m+mi}{5}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sigmoid function}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        
        \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
        
        \PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}Step Function\PYZdq{}\PYZdq{}\PYZdq{}}
        \PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{o}{.}\PY{l+m+mi}{02}\PY{p}{)}
        \PY{n}{step\PYZus{}fn} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{vectorize}\PY{p}{(}\PY{k}{lambda} \PY{n}{z}\PY{p}{:} \PY{l+m+mf}{1.0} \PY{k}{if} \PY{n}{z} \PY{o}{\PYZgt{}}\PY{o}{=} \PY{l+m+mf}{0.0} \PY{k}{else} \PY{l+m+mf}{0.0}\PY{p}{)}
        \PY{n}{step} \PY{o}{=} \PY{n}{step\PYZus{}fn}\PY{p}{(}\PY{n}{z}\PY{p}{)}
        
        \PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
        \PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{step}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{0.5}\PY{p}{,} \PY{l+m+mf}{1.5}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,}\PY{l+m+mi}{5}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{step function}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        
        \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
        
        \PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}Sine Function\PYZdq{}\PYZdq{}\PYZdq{}}
        \PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{,} \PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{,} \PY{l+m+mf}{0.1}\PY{p}{)}
        \PY{n}{t} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{sin}\PY{p}{(}\PY{n}{z}\PY{p}{)}
        
        \PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
        \PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{t}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{1.0}\PY{p}{,} \PY{l+m+mf}{1.0}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{,}\PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sine function}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        
        \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
        
        \PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}Plots a graph of the squashing function used by a rectified linear}
        \PY{l+s+sd}{unit\PYZdq{}\PYZdq{}\PYZdq{}}
        \PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{2}\PY{p}{,} \PY{l+m+mi}{2}\PY{p}{,} \PY{o}{.}\PY{l+m+mi}{1}\PY{p}{)}
        \PY{n}{zero} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{z}\PY{p}{)}\PY{p}{)}
        \PY{n}{y} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{max}\PY{p}{(}\PY{p}{[}\PY{n}{zero}\PY{p}{,} \PY{n}{z}\PY{p}{]}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
        
        \PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
        \PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{y}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{2.0}\PY{p}{,} \PY{l+m+mf}{2.0}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{2.0}\PY{p}{,} \PY{l+m+mf}{2.0}\PY{p}{]}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{Rectified linear unit}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
        
        \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \hypertarget{the-multilayer-perceptron-mlp}{%
\subsection{The multilayer perceptron
(MLP)}\label{the-multilayer-perceptron-mlp}}

The multilayer perceptron is a very popular, and easy to implement
approach, to deep learning. It consists of 1. A neural network with one
or more layers of nodes between the input and the output nodes.

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{1}
\item
  The multilayer network structure, or architecture, or topology,
  consists of an input layer, one or more hidden layers, and one output
  layer.
\item
  The input nodes pass values to the first hidden layer, its nodes pass
  the information on to the second and so on till we reach the output
  layer.
\end{enumerate}

As a convention it is normal to call a network with one layer of input
units, one layer of hidden units and one layer of output units as a
two-layer network. A network with two layers of hidden units is called a
three-layer network etc etc.

For an MLP network there is no direct connection between the output
nodes/neurons/units and the input nodes/neurons/units. Hereafter we will
call the various entities of a layer for nodes. There are also no
connections within a single layer.

The number of input nodes does not need to equal the number of output
nodes. This applies also to the hidden layers. Each layer may have its
own number of nodes and activation functions.

The hidden layers have their name from the fact that they are not linked
to observables and as we will see below when we define the so-called
activation \(\hat{z}\), we can think of this as a basis expansion of the
original inputs \(\hat{x}\). The difference however between neural
networks and say linear regression is that now these basis functions
(which will correspond to the weights in the network) are learned from
data. This results in an important difference between neural networks
and deep learning approaches on one side and methods like logistic
regression or linear regression and their modifications on the other
side.

\hypertarget{from-one-to-many-layers-the-universal-approximation-theorem}{%
\subsection{From one to many layers, the universal approximation
theorem}\label{from-one-to-many-layers-the-universal-approximation-theorem}}

A neural network with only one layer, what we called the simple
perceptron, is best suited if we have a standard binary model with clear
(linear) boundaries between the outcomes. As such it could equally well
be replaced by standard linear regression or logistic regression.
Networks with one or more hidden layers approximate systems with more
complex boundaries.

As stated earlier, an important theorem in studies of neural networks,
restated without proof here, is the
\href{http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873\&rep=rep1\&type=pdf}{universal
approximation theorem}.

It states that a feed-forward network with a single hidden layer
containing a finite number of neurons can approximate continuous
functions on compact subsets of real functions. The theorem thus states
that simple neural networks can represent a wide variety of interesting
functions when given appropriate parameters. It is the multilayer
feedforward architecture itself which gives neural networks the
potential of being universal approximators.

\hypertarget{deriving-the-back-propagation-code-for-a-multilayer-perceptron-model}{%
\subsection{Deriving the back propagation code for a multilayer
perceptron
model}\label{deriving-the-back-propagation-code-for-a-multilayer-perceptron-model}}

\textbf{Note: figures will be inserted later!}

As we have seen now in a feed forward network, we can express the final
output of our network in terms of basic matrix-vector multiplications.
The unknowwn quantities are our weights \(w_{ij}\) and we need to find
an algorithm for changing them so that our errors are as small as
possible. This leads us to the famous
\href{https://www.nature.com/articles/323533a0}{back propagation
algorithm}.

The questions we want to ask are how do changes in the biases and the
weights in our network change the cost function and how can we use the
final output to modify the weights?

To derive these equations let us start with a plain regression problem
and define our cost function as

    \[
{\cal C}(\hat{W})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2,
\]

    where the \(t_i\)s are our \(n\) targets (the values we want to
reproduce), while the outputs of the network after having propagated all
inputs \(\hat{x}\) are given by \(y_i\). Below we will demonstrate how
the basic equations arising from the back propagation algorithm can be
modified in order to study classification problems with \(K\) classes.

\hypertarget{definitions}{%
\subsection{Definitions}\label{definitions}}

With our definition of the targets \(\hat{t}\), the outputs of the
network \(\hat{y}\) and the inputs \(\hat{x}\) we define now the
activation \(z_j^l\) of node/neuron/unit \(j\) of the \(l\)-th layer as
a function of the bias, the weights which add up from the previous layer
\(l-1\) and the forward passes/outputs \(\hat{a}^{l-1}\) from the
previous layer as

    \[
z_j^l = \sum_{i=1}^{M_{l-1}}w_{ij}^la_j^{l-1}+b_j^l,
\]

    where \(b_k^l\) are the biases from layer \(l\). Here \(M_{l-1}\)
represents the total number of nodes/neurons/units of layer \(l-1\). The
figure here illustrates this equation. We can rewrite this in a more
compact form as the matrix-vector products we discussed earlier,

    \[
\hat{z}^l = \left(\hat{W}^l\right)^T\hat{a}^{l-1}+\hat{b}^l.
\]

    With the activation values \(\hat{z}^l\) we can in turn define the
output of layer \(l\) as \(\hat{a}^l = f(\hat{z}^l)\) where \(f\) is our
activation function. In the examples here we will use the sigmoid
function discussed in our logistic regression lectures. We will also use
the same activation function \(f\) for all layers and their nodes. It
means we have

    \[
a_j^l = f(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}.
\]

    \hypertarget{derivatives-and-the-chain-rule}{%
\subsection{Derivatives and the chain
rule}\label{derivatives-and-the-chain-rule}}

From the definition of the activation \(z_j^l\) we have

    \[
\frac{\partial z_j^l}{\partial w_{ji}^l} = a_i^{l-1},
\]

    and

    \[
\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l.
\]

    With our definition of the activation function we have that (note that
this function depends only on \(z_j^l\))

    \[
\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)).
\]

    \hypertarget{derivative-of-the-cost-function}{%
\subsection{Derivative of the cost
function}\label{derivative-of-the-cost-function}}

With these definitions we can now compute the derivative of the cost
function in terms of the weights.

Let us specialize to the output layer \(l=L\). Our cost function is

    \[
{\cal C}(\hat{W^L})  =  \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - t_i\right)^2,
\]

    The derivative of this function with respect to the weights is

    \[
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)\frac{\partial a_j^L}{\partial w_{jk}^{L}},
\]

    The last partial derivative can easily be computed and reads (by
applying the chain rule)

    \[
\frac{\partial a_j^L}{\partial w_{jk}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},
\]

    \hypertarget{bringing-it-together-first-back-propagation-equation}{%
\subsection{Bringing it together, first back propagation
equation}\label{bringing-it-together-first-back-propagation-equation}}

We have thus

    \[
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1},
\]

    Defining

    \[
\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
\]

    and using the Hadamard product of two vectors we can write this as

    \[
\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}L)}.
\]

    This is an important expression. The second term on the right handside
measures how fast the cost function is changing as a function of the
\(j\)th output activation. If, for example, the cost function doesn't
depend much on a particular output node \(j\), then \(\delta_j^L\) will
be small, which is what we would expect. The first term on the right,
measures how fast the activation function \(f\) is changing at a given
activation value \(z_j^L\).

Notice that everything in the above equations is easily computed. In
particular, we compute \(z_j^L\) while computing the behaviour of the
network, and it is only a small additional overhead to compute
\(f'(z^L_j)\). The exact form of the derivative with respect to the
output depends on the form of the cost function. However, provided the
cost function is known there should be little trouble in calculating

    \[
\frac{\partial {\cal C}}{\partial (a_j^L)}
\]

    With the definition of \(\delta_j^L\) we have a more compact definition
of the derivative of the cost function in terms of the weights, namely

    \[
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1}.
\]

    \hypertarget{derivatives-in-terms-of-z_jl}{%
\subsection{\texorpdfstring{Derivatives in terms of
\(z_j^L\)}{Derivatives in terms of z\_j\^{}L}}\label{derivatives-in-terms-of-z_jl}}

It is also easy to see that our previous equation can be written as

    \[
\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
\]

    which can also be interpreted as the partial derivative of the cost
function with respect to the biases \(b_j^L\), namely

    \[
\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
\]

    That is, the error \(\delta_j^L\) is exactly equal to the rate of change
of the cost function as a function of the bias. \#\# Bringing it
together

We have now three equations that are essential for the computations of
the derivatives of the cost function at the output layer. These
equations are needed to start the algorithm and they are

\textbf{The starting equations.}

    \hypertarget{_auto8}{}

\[
\begin{equation}
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L}  =  \delta_j^La_k^{L-1},
\label{_auto8} \tag{13}
\end{equation}
\]

    and

    \hypertarget{_auto9}{}

\[
\begin{equation}
\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
\label{_auto9} \tag{14}
\end{equation}
\]

    and

    \hypertarget{_auto10}{}

\[
\begin{equation}
\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
\label{_auto10} \tag{15}
\end{equation}
\]

    An interesting consequence of the above equations is that when the
activation \(a_k^{L-1}\) is small, the gradient term, that is the
derivative of the cost function with respect to the weights, will also
tend to be small. We say then that the weight learns slowly, meaning
that it changes slowly when we minimize the weights via say gradient
descent. In this case we say the system learns slowly.

Another interesting feature is that is when the activation function,
represented by the sigmoid function here, is rather flat when we move
towards its end values \(0\) and \(1\) (see the above Python codes). In
these cases, the derivatives of the activation function will also be
close to zero, meaning again that the gradients will be small and the
network learns slowly again.

We need a fourth equation and we are set. We are going to propagate
backwards in order to the determine the weights and biases. In order to
do so we need to represent the error in the layer before the final one
\(L-1\) in terms of the errors in the final output layer.

\hypertarget{final-back-propagating-equation}{%
\subsection{Final back propagating
equation}\label{final-back-propagating-equation}}

We have that (replacing \(L\) with a general layer \(l\))

    \[
\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
\]

    We want to express this in terms of the equations for layer \(l+1\).
Using the chain rule and summing over all \(k\) entries we have

    \[
\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
\]

    and recalling that

    \[
z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_j^{l}+b_j^{l+1},
\]

    we obtain

    \[
\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
\]

    This is our final equation.

We are now ready to set up the algorithm for back propagation and
learning the weights and biases.

\hypertarget{setting-up-the-back-propagation-algorithm}{%
\subsection{Setting up the Back propagation
algorithm}\label{setting-up-the-back-propagation-algorithm}}

The four equations provide us with a way of computing the gradient of
the cost function. Let us write this out in the form of an algorithm.

First, we set up the input data \(\hat{x}\) and the activations
\(\hat{z}_1\) of the input layer and compute the activation function and
the pertinent outputs \(\hat{a}^1\).

Secondly, we perform then the feed forward till we reach the output
layer and compute all \(\hat{z}_l\) of the input layer and compute the
activation function and the pertinent outputs \(\hat{a}^l\) for
\(l=2,3,\dots,L\).

Thereafter we compute the ouput error \(\hat{\delta}^L\) by computing
all

    \[
\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
\]

    Then we compute the back propagate error for each \(l=L-1,L-2,\dots,2\)
as

    \[
\delta_j^l =\sum_k \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
\]

    Finally, we update the weights and the biases using gradient descent for
each \(l=L-1,L-2,dots,2\) and update the weights and biases according to
the rules

    \[
w_{jk}^l\leftarrow  = w_{jk}^l- \eta \delta_j^la_k^{l-1},
\]

    \[
b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^L},
\]

    The parameter \(\eta\) is the learning parameter discussed in connection
with the gradient descent methods. Here it is convenient to use
stochastic gradient descent (see the examples below) with mini-batches
with an outer loop that steps through multiple epochs of training.

\hypertarget{setting-up-a-multi-layer-perceptron-model-for-classification}{%
\subsection{Setting up a Multi-layer perceptron model for
classification}\label{setting-up-a-multi-layer-perceptron-model-for-classification}}

We are now gong to develop an example based on the MNIST data base. This
is a classification problem and we need to use our cross-entropy
function we discussed in connection with logistic regression. The
cross-entropy defines our cost function for the classificaton problems
with neural networks.

In binary classification with two classes \((0, 1)\) we define the
logistic/sigmoid function as the probability that a particular input is
in class \(0\) or \(1\). This is possible because the logistic function
takes any input from the real numbers and inputs a number between 0 and
1, and can therefore be interpreted as a probability. It also has other
nice properties, such as a derivative that is simple to calculate.

For an input \(\boldsymbol{a}\) from the hidden layer, the probability
that the input \(\boldsymbol{x}\) is in class 0 or 1 is just. We let
\(\theta\) represent the unknown weights and biases to be adjusted by
our equations). The variable \(x\) represents our activation values
\(z\). We have

    \[
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{x}} ,
\]

    and

    \[
P(y = 1 \mid \boldsymbol{x}, \boldsymbol{\theta}) = 1 - P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) ,
\]

    where \(y \in \{0, 1\}\) and \(\boldsymbol{\theta}\) represents the
weights and biases of our network.

\hypertarget{defining-the-cost-function}{%
\subsection{Defining the cost
function}\label{defining-the-cost-function}}

Our cost function is given as (see the Logistic regression lectures)

    \[
\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n
y_i \ln[P(y_i = 0)] + (1 - y_i) \ln [1 - P(y_i = 0)] = \sum_{i=1}^n \mathcal{L}_i(\boldsymbol{\theta}) .
\]

    This last equality means that we can interpret our \emph{cost} function
as a sum over the \emph{loss} function for each point in the dataset
\(\mathcal{L}_i(\boldsymbol{\theta})\).\\
The negative sign is just so that we can think about our algorithm as
minimizing a positive number, rather than maximizing a negative number.

In \emph{multiclass} classification it is common to treat each integer
label as a so called \emph{one-hot} vector:

\(y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,\)
and

\(y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,\)

i.e.~a binary bit string of length \(C\), where \(C = 10\) is the number
of classes in the MNIST dataset (numbers from \(0\) to \(9\))..

If \(\boldsymbol{x}_i\) is the \(i\)-th input (image), \(y_{ic}\) refers
to the \(c\)-th component of the \(i\)-th output vector
\(\boldsymbol{y}_i\).\\
The probability of \(\boldsymbol{x}_i\) being in class \(c\) will be
given by the softmax function:

    \[
P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) = \frac{\exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_c)}}
{\sum_{c'=0}^{C-1} \exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_{c'})}} ,
\]

    which reduces to the logistic function in the binary case.\\
The likelihood of this \(C\)-class classifier is now given as:

    \[
P(\mathcal{D} \mid \boldsymbol{\theta}) = \prod_{i=1}^n \prod_{c=0}^{C-1} [P(y_{ic} = 1)]^{y_{ic}} .
\]

    Again we take the negative log-likelihood to define our cost function:

    \[
\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n \sum_{c=0}^{C-1}
y_{ic} \ln[P(y_{ic} = 1)] = \sum_{i=1}^n
\mathcal{L}_i(\boldsymbol{\theta}) .
\]

    The back propagation equations need now only a small change, namely the
definition of a new cost function. We are thus ready to use the same
equations as before! We leave it as an exercise in project 2 to derive
these equations.

\hypertarget{developing-a-code-for-doing-neural-networks-with-back-propagation}{%
\subsection{Developing a code for doing neural networks with back
propagation}\label{developing-a-code-for-doing-neural-networks-with-back-propagation}}

One can identify a set of key steps when using neural networks to solve
supervised learning problems:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  Collect and pre-process data
\item
  Define model and architecture
\item
  Choose cost function and optimizer
\item
  Train the model
\item
  Evaluate model performance on test data
\item
  Adjust hyperparameters (if necessary, network architecture)
\end{enumerate}

\hypertarget{collect-and-pre-process-data}{%
\subsection{Collect and pre-process
data}\label{collect-and-pre-process-data}}

Here we will be using the MNIST dataset, which is readily available
through the \textbf{scikit-learn} package. You may also find it for
example \href{http://yann.lecun.com/exdb/mnist/}{here}.\\
The \emph{MNIST} (Modified National Institute of Standards and
Technology) database is a large database of handwritten digits that is
commonly used for training various image processing systems.\\
The MNIST dataset consists of 70 000 images of size 28x28 pixels, each
labeled from 0 to 9.\\
The scikit-learn dataset we will use consists of a selection of 1797
images of size \(8\times 8\) collected and processed from this database.

To feed data into a feed-forward neural network we need to represent the
inputs as a feature matrix \(X = (n_{inputs}, n_{features})\). Each row
represents an \emph{input}, in this case a handwritten digit, and each
column represents a \emph{feature}, in this case a pixel. The correct
answers, also known as \emph{labels} or \emph{targets} are represented
as a 1D array of integers \(Y = (n_{inputs}) = (5, 3, 1, 8,...)\).

As an example, say we want to build a neural network using supervised
learning to predict Body-Mass Index (BMI) from measurements of height
(in m)\\
and weight (in kg). If we have measurements of 5 people the feature
matrix could be for example:

\[ X = \begin{bmatrix}
1.85 & 81\\
1.71 & 65\\
1.95 & 103\\
1.55 & 42\\
1.63 & 56
\end{bmatrix} ,\]

and the targets would be:

\[ Y = (23.7, 22.2, 27.1, 17.5, 21.1) \]

Since each input image is a 2D matrix, we need to flatten the image
(i.e. ``unravel'' the 2D matrix into a 1D array) to turn the data into a
feature matrix. This means we lose all spatial information in the image,
such as locality and translational invariance. More complicated
architectures such as Convolutional Neural Networks can take advantage
of such information, and are most commonly applied when analyzing
images.

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}2}]:} \PY{c+c1}{\PYZsh{} import necessary packages}
        \PY{k+kn}{import} \PY{n+nn}{numpy} \PY{k}{as} \PY{n+nn}{np}
        \PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt}
        \PY{k+kn}{from} \PY{n+nn}{sklearn} \PY{k}{import} \PY{n}{datasets}
        
        
        \PY{c+c1}{\PYZsh{} ensure the same random numbers appear every time}
        \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{seed}\PY{p}{(}\PY{l+m+mi}{0}\PY{p}{)}
        
        \PY{c+c1}{\PYZsh{} display images in notebook}
        \PY{o}{\PYZpc{}}\PY{k}{matplotlib} inline
        \PY{n}{plt}\PY{o}{.}\PY{n}{rcParams}\PY{p}{[}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{figure.figsize}\PY{l+s+s1}{\PYZsq{}}\PY{p}{]} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{12}\PY{p}{,}\PY{l+m+mi}{12}\PY{p}{)}
        
        
        \PY{c+c1}{\PYZsh{} download MNIST dataset}
        \PY{n}{digits} \PY{o}{=} \PY{n}{datasets}\PY{o}{.}\PY{n}{load\PYZus{}digits}\PY{p}{(}\PY{p}{)}
        
        \PY{c+c1}{\PYZsh{} define inputs and labels}
        \PY{n}{inputs} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{images}
        \PY{n}{labels} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{target}
        
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{inputs = (n\PYZus{}inputs, pixel\PYZus{}width, pixel\PYZus{}height) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{labels = (n\PYZus{}inputs) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{labels}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
        
        
        \PY{c+c1}{\PYZsh{} flatten the image}
        \PY{c+c1}{\PYZsh{} the value \PYZhy{}1 means dimension is inferred from the remaining dimensions: 8x8 = 64}
        \PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{inputs}\PY{p}{)}
        \PY{n}{inputs} \PY{o}{=} \PY{n}{inputs}\PY{o}{.}\PY{n}{reshape}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{o}{\PYZhy{}}\PY{l+m+mi}{1}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{X = (n\PYZus{}inputs, n\PYZus{}features) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
        
        
        \PY{c+c1}{\PYZsh{} choose some random images to display}
        \PY{n}{indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
        \PY{n}{random\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}\PY{n}{indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{l+m+mi}{5}\PY{p}{)}
        
        \PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{image} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{digits}\PY{o}{.}\PY{n}{images}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{]}\PY{p}{)}\PY{p}{:}
            \PY{n}{plt}\PY{o}{.}\PY{n}{subplot}\PY{p}{(}\PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{n}{i}\PY{o}{+}\PY{l+m+mi}{1}\PY{p}{)}
            \PY{n}{plt}\PY{o}{.}\PY{n}{axis}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{off}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
            \PY{n}{plt}\PY{o}{.}\PY{n}{imshow}\PY{p}{(}\PY{n}{image}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{n}{plt}\PY{o}{.}\PY{n}{cm}\PY{o}{.}\PY{n}{gray\PYZus{}r}\PY{p}{,} \PY{n}{interpolation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{nearest}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
            \PY{n}{plt}\PY{o}{.}\PY{n}{title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Label: }\PY{l+s+si}{\PYZpc{}d}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{digits}\PY{o}{.}\PY{n}{target}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{]}\PY{p}{)}
        \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
inputs = (n\_inputs, pixel\_width, pixel\_height) = (1797, 8, 8)
labels = (n\_inputs) = (1797,)
X = (n\_inputs, n\_features) = (1797, 64)

    \end{Verbatim}

    \begin{center}
    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_98_1.png}
    \end{center}
    { \hspace*{\fill} \\}
    
    \hypertarget{train-and-test-datasets}{%
\subsection{Train and test datasets}\label{train-and-test-datasets}}

Performing analysis before partitioning the dataset is a major error,
that can lead to incorrect conclusions.

We will reserve \(80 \%\) of our dataset for training and \(20 \%\) for
testing.

It is important that the train and test datasets are drawn randomly from
our dataset, to ensure no bias in the sampling.\\
Say you are taking measurements of weather data to predict the weather
in the coming 5 days. You don't want to train your model on measurements
taken from the hours 00.00 to 12.00, and then test it on data collected
from 12.00 to 24.00.

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}3}]:} \PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{model\PYZus{}selection} \PY{k}{import} \PY{n}{train\PYZus{}test\PYZus{}split}
        
        \PY{c+c1}{\PYZsh{} one\PYZhy{}liner from scikit\PYZhy{}learn library}
        \PY{n}{train\PYZus{}size} \PY{o}{=} \PY{l+m+mf}{0.8}
        \PY{n}{test\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n}{train\PYZus{}size}
        \PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{train\PYZus{}test\PYZus{}split}\PY{p}{(}\PY{n}{inputs}\PY{p}{,} \PY{n}{labels}\PY{p}{,} \PY{n}{train\PYZus{}size}\PY{o}{=}\PY{n}{train\PYZus{}size}\PY{p}{,}
                                                            \PY{n}{test\PYZus{}size}\PY{o}{=}\PY{n}{test\PYZus{}size}\PY{p}{)}
        
        \PY{c+c1}{\PYZsh{} equivalently in numpy}
        \PY{k}{def} \PY{n+nf}{train\PYZus{}test\PYZus{}split\PYZus{}numpy}\PY{p}{(}\PY{n}{inputs}\PY{p}{,} \PY{n}{labels}\PY{p}{,} \PY{n}{train\PYZus{}size}\PY{p}{,} \PY{n}{test\PYZus{}size}\PY{p}{)}\PY{p}{:}
            \PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{inputs}\PY{p}{)}
            \PY{n}{inputs\PYZus{}shuffled} \PY{o}{=} \PY{n}{inputs}\PY{o}{.}\PY{n}{copy}\PY{p}{(}\PY{p}{)}
            \PY{n}{labels\PYZus{}shuffled} \PY{o}{=} \PY{n}{labels}\PY{o}{.}\PY{n}{copy}\PY{p}{(}\PY{p}{)}
            
            \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{shuffle}\PY{p}{(}\PY{n}{inputs\PYZus{}shuffled}\PY{p}{)}
            \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{shuffle}\PY{p}{(}\PY{n}{labels\PYZus{}shuffled}\PY{p}{)}
            
            \PY{n}{train\PYZus{}end} \PY{o}{=} \PY{n+nb}{int}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{o}{*}\PY{n}{train\PYZus{}size}\PY{p}{)}
            \PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test} \PY{o}{=} \PY{n}{inputs\PYZus{}shuffled}\PY{p}{[}\PY{p}{:}\PY{n}{train\PYZus{}end}\PY{p}{]}\PY{p}{,} \PY{n}{inputs\PYZus{}shuffled}\PY{p}{[}\PY{n}{train\PYZus{}end}\PY{p}{:}\PY{p}{]}
            \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{labels\PYZus{}shuffled}\PY{p}{[}\PY{p}{:}\PY{n}{train\PYZus{}end}\PY{p}{]}\PY{p}{,} \PY{n}{labels\PYZus{}shuffled}\PY{p}{[}\PY{n}{train\PYZus{}end}\PY{p}{:}\PY{p}{]}
            
            \PY{k}{return} \PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test}
        
        \PY{c+c1}{\PYZsh{}X\PYZus{}train, X\PYZus{}test, Y\PYZus{}train, Y\PYZus{}test = train\PYZus{}test\PYZus{}split\PYZus{}numpy(inputs, labels, train\PYZus{}size, test\PYZus{}size)}
        
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Number of training images: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}\PY{p}{)}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Number of test images: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}\PY{p}{)}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
Number of training images: 1437
Number of test images: 360

    \end{Verbatim}

    \hypertarget{define-model-and-architecture}{%
\subsection{Define model and
architecture}\label{define-model-and-architecture}}

Our simple feed-forward neural network will consist of an \emph{input}
layer, a single \emph{hidden} layer and an \emph{output} layer. The
activation \(y\) of each neuron is a weighted sum of inputs, passed
through an activation function:

\[ z = \sum_{i=1}^n w_i a_i ,\]

\[ y = f(z) ,\]

where \(f\) is the activation function, \(a_i\) represents input from
neuron \(i\) in the preceding layer and \(w_i\) is the weight to input
\(i\).\\
The activation of the neurons in the input layer is just the features
(e.g.~a pixel value).

The simplest activation function for a neuron is the \emph{Heaviside}
function:

\[ f(z) = 
\begin{cases}
1,  &  z > 0\\
0,  & \text{otherwise}
\end{cases}
\]

A feed-forward neural network with this activation is known as a
\emph{perceptron}.\\
For a binary classifier (i.e.~two classes, 0 or 1, dog or not-dog) we
can also use this in our output layer.\\
This activation can be generalized to \(k\) classes (using e.g.~the
\emph{one-against-all} strategy), and we call these architectures
\emph{multiclass perceptrons}.

However, it is now common to use the terms Single Layer Perceptron (SLP)
(1 hidden layer) and\\
Multilayer Perceptron (MLP) (2 or more hidden layers) to refer to
feed-forward neural networks with any activation function.

Typical choices for activation functions include the sigmoid function,
hyperbolic tangent, and Rectified Linear Unit (ReLU).\\
We will be using the sigmoid function \(\sigma(x)\):

\[ f(x) = \sigma(x) = \frac{1}{1 + e^{-x}} ,\]

which is inspired by probability theory (see logistic regression) and
was most commonly used until about 2011.

\hypertarget{layers}{%
\subsection{Layers}\label{layers}}

\begin{itemize}
\tightlist
\item
  Input
\end{itemize}

Since each input image has 8x8 = 64 pixels or features, we have an input
layer of 64 neurons.

\begin{itemize}
\tightlist
\item
  Hidden layer
\end{itemize}

We will use 50 neurons in the hidden layer receiving input from the
neurons in the input layer.\\
Since each neuron in the hidden layer is connected to the 64 inputs we
have 64x50 = 3200 weights to the hidden layer.

\begin{itemize}
\tightlist
\item
  Output
\end{itemize}

If we were building a binary classifier, it would be sufficient with a
single neuron in the output layer, which could output 0 or 1 according
to the Heaviside function. This would be an example of a \emph{hard}
classifier, meaning it outputs the class of the input directly. However,
if we are dealing with noisy data it is often beneficial to use a
\emph{soft} classifier, which outputs the probability of being in class
0 or 1.

For a soft binary classifier, we could use a single neuron and interpret
the output as either being the probability of being in class 0 or the
probability of being in class 1. Alternatively we could use 2 neurons,
and interpret each neuron as the probability of being in each class.

Since we are doing multiclass classification, with 10 categories, it is
natural to use 10 neurons in the output layer. We number the neurons
\(j = 0,1,...,9\). The activation of each output neuron \(j\) will be
according to the \emph{softmax} function:

\[ P(\text{class $j$} \mid \text{input $\boldsymbol{a}$}) = \frac{\exp{(\boldsymbol{a}^T \boldsymbol{w}_j)}}
{\sum_{c=0}^{9} \exp{(\boldsymbol{a}^T \boldsymbol{w}_c)}} ,\]

i.e.~each neuron \(j\) outputs the probability of being in class \(j\)
given an input from the hidden layer \(\boldsymbol{a}\), with
\(\boldsymbol{w}_j\) the weights of neuron \(j\) to the inputs.\\
The denominator is a normalization factor to ensure the outputs
(probabilities) sum up to 1.\\
The exponent is just the weighted sum of inputs as before:

\[ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.\]

Since each neuron in the output layer is connected to the 50 inputs from
the hidden layer we have 50x10 = 500 weights to the output layer.

\hypertarget{weights-and-biases}{%
\subsection{Weights and biases}\label{weights-and-biases}}

Typically weights are initialized with small values distributed around
zero, drawn from a uniform or normal distribution. Setting all weights
to zero means all neurons give the same output, making the network
useless.

Adding a bias value to the weighted sum of inputs allows the neural
network to represent a greater range of values. Without it, any input
with the value 0 will be mapped to zero (before being passed through the
activation). The bias unit has an output of 1, and a weight to each
neuron \(j\), \(b_j\):

\[ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j.\]

The bias weights \(\boldsymbol{b}\) are often initialized to zero, but a
small value like \(0.01\) ensures all neurons have some output which can
be backpropagated in the first training cycle.

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}5}]:} \PY{c+c1}{\PYZsh{} building our neural network}
        
        \PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{n}{n\PYZus{}features} \PY{o}{=} \PY{n}{X\PYZus{}train}\PY{o}{.}\PY{n}{shape}
        \PY{n}{n\PYZus{}hidden\PYZus{}neurons} \PY{o}{=} \PY{l+m+mi}{50}
        \PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{l+m+mi}{10}
        
        \PY{c+c1}{\PYZsh{} we make the weights normally distributed using numpy.random.randn}
        
        \PY{c+c1}{\PYZsh{} weights and bias in the hidden layer}
        \PY{n}{hidden\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n}{n\PYZus{}features}\PY{p}{,} \PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)}
        \PY{n}{hidden\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
        
        \PY{c+c1}{\PYZsh{} weights and bias in the output layer}
        \PY{n}{output\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{)}
        \PY{n}{output\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n}{n\PYZus{}categories}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
\end{Verbatim}


    \hypertarget{feed-forward-pass}{%
\subsection{Feed-forward pass}\label{feed-forward-pass}}

Denote \(F\) the number of features, \(H\) the number of hidden neurons
and \(C\) the number of categories.\\
For each input image we calculate a weighted sum of input features
(pixel values) to each neuron \(j\) in the hidden layer \(l\):

\[ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},\]

this is then passed through our activation function

\[ a_{j}^{l} = f(z_{j}^{l}) .\]

We calculate a weighted sum of inputs (activations in the hidden layer)
to each neuron \(j\) in the output layer:

\[ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.\]

Finally we calculate the output of neuron \(j\) in the output layer
using the softmax function:

\[ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .\]

\hypertarget{matrix-multiplication}{%
\subsection{Matrix multiplication}\label{matrix-multiplication}}

Since our data has the dimensions \(X = (n_{inputs}, n_{features})\) and
our weights to the hidden layer have the dimensions\\
\(W_{hidden} = (n_{features}, n_{hidden})\), we can easily feed the
network all our training data in one go by taking the matrix product

\[ X W^{h} = (n_{inputs}, n_{hidden}),\]

and obtain a matrix that holds the weighted sum of inputs to the hidden
layer for each input image and each hidden neuron.\\
We also add the bias to obtain a matrix of weighted sums to the hidden
layer \(Z^{h}\):

\[ \hat{z}^{l} = \hat{X} \hat{W}^{l} + \hat{b}^{l} ,\]

meaning the same bias (1D array with size equal number of hidden
neurons) is added to each input image.\\
This is then passed through the activation:

\[ \hat{a}^{l} = f(\hat{z}^l) .\]

This is fed to the output layer:

\[ \hat{z}^{L} = \hat{a}^{L} \hat{W}^{L} + \hat{b}^{L} .\]

Finally we receive our output values for each image and each category by
passing it through the softmax function:

\[ output = softmax (\hat{z}^{L}) = (n_{inputs}, n_{categories}) .\]

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}6}]:} \PY{c+c1}{\PYZsh{} setup the feed\PYZhy{}forward pass, subscript h = hidden layer}
        
        \PY{k}{def} \PY{n+nf}{sigmoid}\PY{p}{(}\PY{n}{x}\PY{p}{)}\PY{p}{:}
            \PY{k}{return} \PY{l+m+mi}{1}\PY{o}{/}\PY{p}{(}\PY{l+m+mi}{1} \PY{o}{+} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{n}{x}\PY{p}{)}\PY{p}{)}
        
        \PY{k}{def} \PY{n+nf}{feed\PYZus{}forward}\PY{p}{(}\PY{n}{X}\PY{p}{)}\PY{p}{:}
            \PY{c+c1}{\PYZsh{} weighted sum of inputs to the hidden layer}
            \PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{hidden\PYZus{}bias}
            \PY{c+c1}{\PYZsh{} activation in the hidden layer}
            \PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n}{z\PYZus{}h}\PY{p}{)}
            
            \PY{c+c1}{\PYZsh{} weighted sum of inputs to the output layer}
            \PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{output\PYZus{}bias}
            \PY{c+c1}{\PYZsh{} softmax output}
            \PY{c+c1}{\PYZsh{} axis 0 holds each input and axis 1 the probabilities of each category}
            \PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n}{z\PYZus{}o}\PY{p}{)}
            \PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
            
            \PY{k}{return} \PY{n}{probabilities}
        
        \PY{n}{probabilities} \PY{o}{=} \PY{n}{feed\PYZus{}forward}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{probabilities = (n\PYZus{}inputs, n\PYZus{}categories) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{probabilities}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{probability that image 0 is in category 0,1,2,...,9 = }\PY{l+s+se}{\PYZbs{}n}\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{probabilities}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{p}{)}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{probabilities sum up to: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{probabilities}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{p}{)}\PY{p}{)}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
        
        \PY{c+c1}{\PYZsh{} we obtain a prediction by taking the class with the highest likelihood}
        \PY{k}{def} \PY{n+nf}{predict}\PY{p}{(}\PY{n}{X}\PY{p}{)}\PY{p}{:}
            \PY{n}{probabilities} \PY{o}{=} \PY{n}{feed\PYZus{}forward}\PY{p}{(}\PY{n}{X}\PY{p}{)}
            \PY{k}{return} \PY{n}{np}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n}{probabilities}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
        
        \PY{n}{predictions} \PY{o}{=} \PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{predictions = (n\PYZus{}inputs) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{predictions}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{prediction for image 0: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{predictions}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{p}{)}\PY{p}{)}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{correct label for image 0: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{p}{)}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
probabilities = (n\_inputs, n\_categories) = (1437, 10)
probability that image 0 is in category 0,1,2,{\ldots},9 = 
[3.89940599e-05 1.79115580e-01 1.47286800e-02 7.96733555e-01
 3.28982767e-04 1.49752254e-07 9.19699482e-05 4.42365585e-03
 3.57722690e-06 4.53485505e-03]
probabilities sum up to: 1.0000000000000002

predictions = (n\_inputs) = (1437,)
prediction for image 0: 3
correct label for image 0: 6

    \end{Verbatim}

    \hypertarget{choose-cost-function-and-optimizer}{%
\subsection{Choose cost function and
optimizer}\label{choose-cost-function-and-optimizer}}

To measure how well our neural network is doing we need to introduce a
cost function.\\
We will call the function that gives the error of a single sample output
the \emph{loss} function, and the function that gives the total error of
our network across all samples the \emph{cost} function. A typical
choice for multiclass classification is the \emph{cross-entropy} loss,
also known as the negative log likelihood.

In \emph{multiclass} classification it is common to treat each integer
label as a so called \emph{one-hot} vector:

\[ y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,\]

\[ y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,\]

i.e.~a binary bit string of length \(C\), where \(C = 10\) is the number
of classes in the MNIST dataset.

Let \(y_{ic}\) denote the \(c\)-th component of the \(i\)-th one-hot
vector.\\
We define the cost function \(\mathcal{C}\) as a sum over the
cross-entropy loss for each point \(\boldsymbol{x}_i\) in the dataset:

    \hypertarget{_auto11}{}

\[
\begin{equation} \mathcal{C}(\boldsymbol{\theta}) = \sum_{i=1}^N \mathcal{L}_i (\boldsymbol{\theta}) 
\label{_auto11} \tag{16}
\end{equation}
\]

    \hypertarget{_auto12}{}

\[
\begin{equation} 
 = -\sum_{i=1}^N \sum_{c=0}^{C-1} y_{ic} \log P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) .
\label{_auto12} \tag{17}
\end{equation}
\]

    In the one-hot representation only one of the terms in the loss function
is non-zero, namely the probability of the correct category \(c'\)\\
(i.e.~the category \(c'\) such that \(y_{ic'} = 1\)). This means that
the cross entropy loss only punishes you for how wrong you got the
correct label. The probability of category \(c\) is given by the softmax
function. The vector \(\boldsymbol{\theta}\) represents the parameters
of our network, i.e.~all the weights and biases.

A full derivation is given in the appendix at the end.

\hypertarget{optimizing-the-cost-function}{%
\subsection{Optimizing the cost
function}\label{optimizing-the-cost-function}}

The network is trained by finding the weights and biases that minimize
the cost function. One of the most widely used classes of methods is
\emph{gradient descent} and its generalizations. The idea behind
gradient descent is simply to adjust the weights in the direction where
the gradient of the cost function is large and negative. This ensures we
flow toward a \emph{local} minimum of the cost function.\\
Each parameter \(\theta\) is iteratively adjusted according to the rule

\[ \theta_{i+1} = \theta_i - \eta \nabla \mathcal{C}(\theta_i) ,\]

where \(\eta\) is known as the \emph{learning rate}, which controls how
big a step we take towards the minimum.\\
This update can be repeated for any number of iterations, or until we
are satisfied with the result.

A simple and effective improvement is a variant called \emph{Batch
Gradient Descent}.\\
Instead of calculating the gradient on the whole dataset, we calculate
an approximation of the gradient on a subset of the data called a
\emph{minibatch}.\\
If there are \(N\) data points and we have a minibatch size of \(M\),
the total number of batches is \(N/M\).\\
We denote each minibatch \(B_k\), with \(k = 1, 2,...,N/M\). The
gradient then becomes:

\[ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad
\frac{1}{M} \sum_{i \in B_k} \nabla \mathcal{L}_i(\theta) ,\]

i.e.~instead of averaging the loss over the entire dataset, we average
over a minibatch.

This has two important benefits:\\
1. Introducing stochasticity decreases the chance that the algorithm
becomes stuck in a local minima.

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\setcounter{enumi}{1}
\tightlist
\item
  It significantly speeds up the calculation, since we do not have to
  use the entire dataset to calculate the gradient.
\end{enumerate}

\hypertarget{regularization}{%
\subsection{Regularization}\label{regularization}}

It is common to add an extra term to the cost function, proportional to
the size of the weights. This is equivalent to constraining the size of
the weights, so that they do not grow out of control. Constraining the
size of the weights means that the weights cannot grow arbitrarily large
to fit the training data, and in this way reduces \emph{overfitting}.

We will measure the size of the weights using the so called
\emph{L2-norm}, meaning our cost function becomes:

\[ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad
\frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) + \lambda \lvert \lvert \boldsymbol{w} \rvert \rvert_2^2 
= \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}(\theta) + \lambda \sum_{ij} w_{ij}^2,\]

i.e.~we sum up all the weights squared. The factor \(\lambda\) is known
as a regularization parameter.

In order to train the model, we need to calculate the derivative of the
cost function with respect to every bias and weight in the network. In
total our network has \((64 + 1)\times 50=3250\) weights in the hidden
layer and \((50 + 1)\times 10=510\) weights to the output layer (\(+1\)
for the bias), and the gradient must be calculated for every parameter.
We use the \emph{backpropagation} algorithm discussed above. This is a
clever use of the chain rule that allows us to calculate the gradient
efficently.

\hypertarget{matrix-multiplication}{%
\subsection{Matrix multiplication}\label{matrix-multiplication}}

To more efficently train our network these equations are implemented
using matrix operations.\\
The error in the output layer is calculated simply as

\[ \delta_L = \hat{y} - y = (n_{inputs}, n_{categories}) .\]

The gradient for the output weights is calculated as

\[ \nabla W_{L} = \hat{a}^T \delta_L   = (n_{hidden}, n_{categories}) ,\]

where \(\hat{a} = (n_{inputs}, n_{hidden})\). This simply means that we
are summing up the gradients for each input.\\
Since we are going backwards we have to transpose the activation matrix.

The gradient with respect to the output bias is then

\[ \nabla \hat{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .\]

The error in the hidden layer is

\[ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,\]

where \(f'(a_{h})\) is the derivative of the activation in the hidden
layer. The matrix products mean that we are summing up the products for
each neuron in the output layer. The symbol \(\circ\) denotes the
\emph{Hadamard product}, meaning element-wise multiplication.

This again gives us the gradients in the hidden layer:

\[ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,\]

\[ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .\]

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}7}]:} \PY{c+c1}{\PYZsh{} to categorical turns our integer vector into a onehot representation}
        \PY{c+c1}{\PYZsh{}from keras.utils import to\PYZus{}categorical}
        
        \PY{c+c1}{\PYZsh{} calculate the accuracy score of our model}
        \PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{metrics} \PY{k}{import} \PY{n}{accuracy\PYZus{}score}
        
        \PY{c+c1}{\PYZsh{} one\PYZhy{}hot in numpy}
        \PY{k}{def} \PY{n+nf}{to\PYZus{}categorical\PYZus{}numpy}\PY{p}{(}\PY{n}{integer\PYZus{}vector}\PY{p}{)}\PY{p}{:}
            \PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{integer\PYZus{}vector}\PY{p}{)}
            \PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{max}\PY{p}{(}\PY{n}{integer\PYZus{}vector}\PY{p}{)} \PY{o}{+} \PY{l+m+mi}{1}
            \PY{n}{onehot\PYZus{}vector} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{)}\PY{p}{)}
            \PY{n}{onehot\PYZus{}vector}\PY{p}{[}\PY{n+nb}{range}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{)}\PY{p}{,} \PY{n}{integer\PYZus{}vector}\PY{p}{]} \PY{o}{=} \PY{l+m+mi}{1}
            
            \PY{k}{return} \PY{n}{onehot\PYZus{}vector}
        
        \PY{c+c1}{\PYZsh{}Y\PYZus{}train\PYZus{}onehot, Y\PYZus{}test\PYZus{}onehot = to\PYZus{}categorical(Y\PYZus{}train), to\PYZus{}categorical(Y\PYZus{}test)}
        \PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{,} \PY{n}{Y\PYZus{}test\PYZus{}onehot} \PY{o}{=} \PY{n}{to\PYZus{}categorical\PYZus{}numpy}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{,} \PY{n}{to\PYZus{}categorical\PYZus{}numpy}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{)}
        
        \PY{k}{def} \PY{n+nf}{feed\PYZus{}forward\PYZus{}train}\PY{p}{(}\PY{n}{X}\PY{p}{)}\PY{p}{:}
            \PY{c+c1}{\PYZsh{} weighted sum of inputs to the hidden layer}
            \PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{hidden\PYZus{}bias}
            \PY{c+c1}{\PYZsh{} activation in the hidden layer}
            \PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n}{z\PYZus{}h}\PY{p}{)}
            
            \PY{c+c1}{\PYZsh{} weighted sum of inputs to the output layer}
            \PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{output\PYZus{}bias}
            \PY{c+c1}{\PYZsh{} softmax output}
            \PY{c+c1}{\PYZsh{} axis 0 holds each input and axis 1 the probabilities of each category}
            \PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n}{z\PYZus{}o}\PY{p}{)}
            \PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
            
            \PY{c+c1}{\PYZsh{} for backpropagation need activations in hidden and output layers}
            \PY{k}{return} \PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{probabilities}
        
        \PY{k}{def} \PY{n+nf}{backpropagation}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n}{Y}\PY{p}{)}\PY{p}{:}
            \PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{probabilities} \PY{o}{=} \PY{n}{feed\PYZus{}forward\PYZus{}train}\PY{p}{(}\PY{n}{X}\PY{p}{)}
            
            \PY{c+c1}{\PYZsh{} error in the output layer}
            \PY{n}{error\PYZus{}output} \PY{o}{=} \PY{n}{probabilities} \PY{o}{\PYZhy{}} \PY{n}{Y}
            \PY{c+c1}{\PYZsh{} error in the hidden layer}
            \PY{n}{error\PYZus{}hidden} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n}{output\PYZus{}weights}\PY{o}{.}\PY{n}{T}\PY{p}{)} \PY{o}{*} \PY{n}{a\PYZus{}h} \PY{o}{*} \PY{p}{(}\PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n}{a\PYZus{}h}\PY{p}{)}
            
            \PY{c+c1}{\PYZsh{} gradients for the output layer}
            \PY{n}{output\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}output}\PY{p}{)}
            \PY{n}{output\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
            
            \PY{c+c1}{\PYZsh{} gradient for the hidden layer}
            \PY{n}{hidden\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}hidden}\PY{p}{)}
            \PY{n}{hidden\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}hidden}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
        
            \PY{k}{return} \PY{n}{output\PYZus{}weights\PYZus{}gradient}\PY{p}{,} \PY{n}{output\PYZus{}bias\PYZus{}gradient}\PY{p}{,} \PY{n}{hidden\PYZus{}weights\PYZus{}gradient}\PY{p}{,} \PY{n}{hidden\PYZus{}bias\PYZus{}gradient}
        
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Old accuracy on training data: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{)}\PY{p}{)}
        
        \PY{n}{eta} \PY{o}{=} \PY{l+m+mf}{0.01}
        \PY{n}{lmbd} \PY{o}{=} \PY{l+m+mf}{0.01}
        \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{l+m+mi}{1000}\PY{p}{)}\PY{p}{:}
            \PY{c+c1}{\PYZsh{} calculate gradients}
            \PY{n}{dWo}\PY{p}{,} \PY{n}{dBo}\PY{p}{,} \PY{n}{dWh}\PY{p}{,} \PY{n}{dBh} \PY{o}{=} \PY{n}{backpropagation}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{)}
            
            \PY{c+c1}{\PYZsh{} regularization term gradients}
            \PY{n}{dWo} \PY{o}{+}\PY{o}{=} \PY{n}{lmbd} \PY{o}{*} \PY{n}{output\PYZus{}weights}
            \PY{n}{dWh} \PY{o}{+}\PY{o}{=} \PY{n}{lmbd} \PY{o}{*} \PY{n}{hidden\PYZus{}weights}
            
            \PY{c+c1}{\PYZsh{} update weights and biases}
            \PY{n}{output\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dWo}
            \PY{n}{output\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dBo}
            \PY{n}{hidden\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dWh}
            \PY{n}{hidden\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dBh}
        
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{New accuracy on training data: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{)}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
Old accuracy on training data: 0.16423103688239388

    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:4: RuntimeWarning: overflow encountered in exp
  after removing the cwd from sys.path.

    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]
New accuracy on training data: 0.10090466249130133

    \end{Verbatim}

    \hypertarget{improving-performance}{%
\subsection{Improving performance}\label{improving-performance}}

As we can see the network does not seem to be learning at all. It seems
to be just guessing the label for each image.\\
In order to obtain a network that does something useful, we will have to
do a bit more work.

The choice of \emph{hyperparameters} such as learning rate and
regularization parameter is hugely influential for the performance of
the network. Typically a \emph{grid-search} is performed, wherein we
test different hyperparameters separated by orders of magnitude. For
example we could test the learning rates
\(\eta = 10^{-6}, 10^{-5},...,10^{-1}\) with different regularization
parameters \(\lambda = 10^{-6},...,10^{-0}\).

Next, we haven't implemented minibatching yet, which introduces
stochasticity and is though to act as an important regularizer on the
weights. We call a feed-forward + backward pass with a minibatch an
\emph{iteration}, and a full training period going through the entire
dataset (\(n/M\) batches) an \emph{epoch}.

If this does not improve network performance, you may want to consider
altering the network architecture, adding more neurons or hidden
layers.\\
Andrew Ng goes through some of these considerations in this
\href{https://youtu.be/F1ka6a13S9I}{video}. You can find a summary of
the video
\href{https://kevinzakka.github.io/2016/09/26/applying-deep-learning/}{here}.

\hypertarget{full-object-oriented-implementation}{%
\subsection{Full object-oriented
implementation}\label{full-object-oriented-implementation}}

It is very natural to think of the network as an object, with specific
instances of the network being realizations of this object with
different hyperparameters. An implementation using Python classes
provides a clean structure and interface, and the full implementation of
our neural network is given below.

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}8}]:} \PY{k}{class} \PY{n+nc}{NeuralNetwork}\PY{p}{:}
            \PY{k}{def} \PY{n+nf}{\PYZus{}\PYZus{}init\PYZus{}\PYZus{}}\PY{p}{(}
                \PY{n+nb+bp}{self}\PY{p}{,}
                \PY{n}{X\PYZus{}data}\PY{p}{,}
                \PY{n}{Y\PYZus{}data}\PY{p}{,}
                \PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{o}{=}\PY{l+m+mi}{50}\PY{p}{,}
                \PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{l+m+mi}{10}\PY{p}{,}
                \PY{n}{epochs}\PY{o}{=}\PY{l+m+mi}{10}\PY{p}{,}
                \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{l+m+mi}{100}\PY{p}{,}
                \PY{n}{eta}\PY{o}{=}\PY{l+m+mf}{0.1}\PY{p}{,}
                \PY{n}{lmbd}\PY{o}{=}\PY{l+m+mf}{0.0}\PY{p}{,}
        
            \PY{p}{)}\PY{p}{:}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data\PYZus{}full} \PY{o}{=} \PY{n}{X\PYZus{}data}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data\PYZus{}full} \PY{o}{=} \PY{n}{Y\PYZus{}data}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n}{X\PYZus{}data}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features} \PY{o}{=} \PY{n}{X\PYZus{}data}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons} \PY{o}{=} \PY{n}{n\PYZus{}hidden\PYZus{}neurons}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{n}{n\PYZus{}categories}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs} \PY{o}{=} \PY{n}{epochs}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{n}{batch\PYZus{}size}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{/}\PY{o}{/} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{=} \PY{n}{eta}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{=} \PY{n}{lmbd}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}biases\PYZus{}and\PYZus{}weights}\PY{p}{(}\PY{p}{)}
        
            \PY{k}{def} \PY{n+nf}{create\PYZus{}biases\PYZus{}and\PYZus{}weights}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{)}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
        
            \PY{k}{def} \PY{n+nf}{feed\PYZus{}forward}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                \PY{c+c1}{\PYZsh{} feed\PYZhy{}forward for training}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}h}\PY{p}{)}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias}
        
                \PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}o}\PY{p}{)}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
        
            \PY{k}{def} \PY{n+nf}{feed\PYZus{}forward\PYZus{}out}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{X}\PY{p}{)}\PY{p}{:}
                \PY{c+c1}{\PYZsh{} feed\PYZhy{}forward for output}
                \PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias}
                \PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n}{z\PYZus{}h}\PY{p}{)}
        
                \PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias}
                
                \PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n}{z\PYZus{}o}\PY{p}{)}
                \PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
                \PY{k}{return} \PY{n}{probabilities}
        
            \PY{k}{def} \PY{n+nf}{backpropagation}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                \PY{n}{error\PYZus{}output} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{probabilities} \PY{o}{\PYZhy{}} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data}
                \PY{n}{error\PYZus{}hidden} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}\PY{o}{.}\PY{n}{T}\PY{p}{)} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h} \PY{o}{*} \PY{p}{(}\PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h}\PY{p}{)}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}output}\PY{p}{)}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}hidden}\PY{p}{)}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}hidden}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
        
                \PY{k}{if} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{\PYZgt{}} \PY{l+m+mf}{0.0}\PY{p}{:}
                    \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights\PYZus{}gradient} \PY{o}{+}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}
                    \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights\PYZus{}gradient} \PY{o}{+}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights}
        
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights\PYZus{}gradient}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias\PYZus{}gradient}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights\PYZus{}gradient}
                \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias\PYZus{}gradient}
        
            \PY{k}{def} \PY{n+nf}{predict}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{X}\PY{p}{)}\PY{p}{:}
                \PY{n}{probabilities} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{feed\PYZus{}forward\PYZus{}out}\PY{p}{(}\PY{n}{X}\PY{p}{)}
                \PY{k}{return} \PY{n}{np}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n}{probabilities}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
        
            \PY{k}{def} \PY{n+nf}{predict\PYZus{}probabilities}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{X}\PY{p}{)}\PY{p}{:}
                \PY{n}{probabilities} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{feed\PYZus{}forward\PYZus{}out}\PY{p}{(}\PY{n}{X}\PY{p}{)}
                \PY{k}{return} \PY{n}{probabilities}
        
            \PY{k}{def} \PY{n+nf}{train}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                \PY{n}{data\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
        
                \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs}\PY{p}{)}\PY{p}{:}
                    \PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations}\PY{p}{)}\PY{p}{:}
                        \PY{c+c1}{\PYZsh{} pick datapoints with replacement}
                        \PY{n}{chosen\PYZus{}datapoints} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}
                            \PY{n}{data\PYZus{}indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{replace}\PY{o}{=}\PY{k+kc}{False}
                        \PY{p}{)}
        
                        \PY{c+c1}{\PYZsh{} minibatch training data}
                        \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data\PYZus{}full}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}
                        \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data\PYZus{}full}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}
        
                        \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{feed\PYZus{}forward}\PY{p}{(}\PY{p}{)}
                        \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{backpropagation}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \hypertarget{evaluate-model-performance-on-test-data}{%
\subsection{Evaluate model performance on test
data}\label{evaluate-model-performance-on-test-data}}

To measure the performance of our network we evaluate how well it does
it data it has never seen before, i.e.~the test data.\\
We measure the performance of the network using the \emph{accuracy}
score.\\
The accuracy is as you would expect just the number of images correctly
labeled divided by the total number of images. A perfect classifier will
have an accuracy score of \(1\).

\[ \text{Accuracy} = \frac{\sum_{i=1}^n I(\hat{y}_i = y_i)}{n} ,\]

where \(I\) is the indicator function, \(1\) if \(\hat{y}_i = y_i\) and
\(0\) otherwise.

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}9}]:} \PY{n}{epochs} \PY{o}{=} \PY{l+m+mi}{100}
        \PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{100}
        
        \PY{n}{dnn} \PY{o}{=} \PY{n}{NeuralNetwork}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{,} \PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{,} \PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,}
                            \PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{o}{=}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{n}{n\PYZus{}categories}\PY{p}{)}
        \PY{n}{dnn}\PY{o}{.}\PY{n}{train}\PY{p}{(}\PY{p}{)}
        \PY{n}{test\PYZus{}predict} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
        
        \PY{c+c1}{\PYZsh{} accuracy score from scikit library}
        \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Accuracy score on test set: }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}predict}\PY{p}{)}\PY{p}{)}
        
        \PY{c+c1}{\PYZsh{} equivalent in numpy}
        \PY{k}{def} \PY{n+nf}{accuracy\PYZus{}score\PYZus{}numpy}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}pred}\PY{p}{)}\PY{p}{:}
            \PY{k}{return} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{Y\PYZus{}test} \PY{o}{==} \PY{n}{Y\PYZus{}pred}\PY{p}{)} \PY{o}{/} \PY{n+nb}{len}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{)}
        
        \PY{c+c1}{\PYZsh{}print(\PYZdq{}Accuracy score on test set: \PYZdq{}, accuracy\PYZus{}score\PYZus{}numpy(Y\PYZus{}test, test\PYZus{}predict))}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
Accuracy score on test set:  0.9305555555555556

    \end{Verbatim}

    \hypertarget{adjust-hyperparameters}{%
\subsection{Adjust hyperparameters}\label{adjust-hyperparameters}}

We now perform a grid search to find the optimal hyperparameters for the
network.\\
Note that we are only using 1 layer with 50 neurons, and human
performance is estimated to be around \(98\%\) (\(2\%\) error rate).

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}10}]:} \PY{n}{eta\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
         \PY{n}{lmbd\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
         \PY{c+c1}{\PYZsh{} store the models for later use}
         \PY{n}{DNN\PYZus{}numpy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
         
         \PY{c+c1}{\PYZsh{} grid search}
         \PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
                 \PY{n}{dnn} \PY{o}{=} \PY{n}{NeuralNetwork}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{,} \PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{,} \PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,}
                                     \PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{o}{=}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{n}{n\PYZus{}categories}\PY{p}{)}
                 \PY{n}{dnn}\PY{o}{.}\PY{n}{train}\PY{p}{(}\PY{p}{)}
                 
                 \PY{n}{DNN\PYZus{}numpy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{dnn}
                 
                 \PY{n}{test\PYZus{}predict} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
                 
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate  = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Accuracy score on test set: }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}predict}\PY{p}{)}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
Learning rate  =  1e-05
Lambda =  1e-05
Accuracy score on test set:  0.18888888888888888

Learning rate  =  1e-05
Lambda =  0.0001
Accuracy score on test set:  0.18333333333333332

Learning rate  =  1e-05
Lambda =  0.001
Accuracy score on test set:  0.20277777777777778

Learning rate  =  1e-05
Lambda =  0.01
Accuracy score on test set:  0.2

Learning rate  =  1e-05
Lambda =  0.1
Accuracy score on test set:  0.12222222222222222

Learning rate  =  1e-05
Lambda =  1.0
Accuracy score on test set:  0.18888888888888888

Learning rate  =  1e-05
Lambda =  10.0
Accuracy score on test set:  0.1527777777777778

Learning rate  =  0.0001
Lambda =  1e-05
Accuracy score on test set:  0.5916666666666667

Learning rate  =  0.0001
Lambda =  0.0001
Accuracy score on test set:  0.5583333333333333

Learning rate  =  0.0001
Lambda =  0.001
Accuracy score on test set:  0.5361111111111111

Learning rate  =  0.0001
Lambda =  0.01
Accuracy score on test set:  0.5777777777777777

Learning rate  =  0.0001
Lambda =  0.1
Accuracy score on test set:  0.6055555555555555

Learning rate  =  0.0001
Lambda =  1.0
Accuracy score on test set:  0.6416666666666667

Learning rate  =  0.0001
Lambda =  10.0
Accuracy score on test set:  0.8111111111111111

Learning rate  =  0.001
Lambda =  1e-05
Accuracy score on test set:  0.8833333333333333

Learning rate  =  0.001
Lambda =  0.0001
Accuracy score on test set:  0.9

Learning rate  =  0.001
Lambda =  0.001
Accuracy score on test set:  0.8666666666666667

Learning rate  =  0.001
Lambda =  0.01
Accuracy score on test set:  0.875

Learning rate  =  0.001
Lambda =  0.1
Accuracy score on test set:  0.8666666666666667

Learning rate  =  0.001
Lambda =  1.0
Accuracy score on test set:  0.9416666666666667

Learning rate  =  0.001
Lambda =  10.0
Accuracy score on test set:  0.9416666666666667

Learning rate  =  0.01
Lambda =  1e-05
Accuracy score on test set:  0.9472222222222222

Learning rate  =  0.01
Lambda =  0.0001
Accuracy score on test set:  0.9333333333333333

Learning rate  =  0.01
Lambda =  0.001
Accuracy score on test set:  0.9416666666666667

Learning rate  =  0.01
Lambda =  0.01
Accuracy score on test set:  0.9361111111111111

Learning rate  =  0.01
Lambda =  0.1
Accuracy score on test set:  0.9527777777777777

Learning rate  =  0.01
Lambda =  1.0
Accuracy score on test set:  0.9333333333333333

Learning rate  =  0.01
Lambda =  10.0
Accuracy score on test set:  0.23055555555555557


    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:4: RuntimeWarning: overflow encountered in exp
  after removing the cwd from sys.path.

    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]
Learning rate  =  0.1
Lambda =  1e-05
Accuracy score on test set:  0.10555555555555556

Learning rate  =  0.1
Lambda =  0.0001
Accuracy score on test set:  0.10555555555555556

Learning rate  =  0.1
Lambda =  0.001
Accuracy score on test set:  0.11666666666666667

Learning rate  =  0.1
Lambda =  0.01
Accuracy score on test set:  0.07777777777777778

Learning rate  =  0.1
Lambda =  0.1
Accuracy score on test set:  0.08611111111111111

Learning rate  =  0.1
Lambda =  1.0
Accuracy score on test set:  0.10555555555555556

Learning rate  =  0.1
Lambda =  10.0
Accuracy score on test set:  0.125


    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:44: RuntimeWarning: overflow encountered in exp
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:45: RuntimeWarning: invalid value encountered in true\_divide

    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]
Learning rate  =  1.0
Lambda =  1e-05
Accuracy score on test set:  0.07777777777777778

Learning rate  =  1.0
Lambda =  0.0001
Accuracy score on test set:  0.07777777777777778

Learning rate  =  1.0
Lambda =  0.001
Accuracy score on test set:  0.07777777777777778

Learning rate  =  1.0
Lambda =  0.01
Accuracy score on test set:  0.07777777777777778

Learning rate  =  1.0
Lambda =  0.1
Accuracy score on test set:  0.07777777777777778

Learning rate  =  1.0
Lambda =  1.0
Accuracy score on test set:  0.10555555555555556

Learning rate  =  1.0
Lambda =  10.0
Accuracy score on test set:  0.07777777777777778

Learning rate  =  10.0
Lambda =  1e-05
Accuracy score on test set:  0.07777777777777778

Learning rate  =  10.0
Lambda =  0.0001
Accuracy score on test set:  0.07777777777777778

Learning rate  =  10.0
Lambda =  0.001
Accuracy score on test set:  0.07777777777777778

Learning rate  =  10.0
Lambda =  0.01
Accuracy score on test set:  0.07777777777777778

Learning rate  =  10.0
Lambda =  0.1
Accuracy score on test set:  0.07777777777777778

Learning rate  =  10.0
Lambda =  1.0
Accuracy score on test set:  0.07777777777777778

Learning rate  =  10.0
Lambda =  10.0
Accuracy score on test set:  0.07777777777777778


    \end{Verbatim}

    \hypertarget{visualization}{%
\subsection{Visualization}\label{visualization}}

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}11}]:} \PY{c+c1}{\PYZsh{} visual representation of grid search}
         \PY{c+c1}{\PYZsh{} uses seaborn heatmap, you can also do this with matplotlib imshow}
         \PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
         
         \PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         \PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         
         \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
                 \PY{n}{dnn} \PY{o}{=} \PY{n}{DNN\PYZus{}numpy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
                 
                 \PY{n}{train\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)} 
                 \PY{n}{test\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
         
                 \PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{train\PYZus{}pred}\PY{p}{)}
                 \PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}pred}\PY{p}{)}
         
                 
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:4: RuntimeWarning: overflow encountered in exp
  after removing the cwd from sys.path.

    \end{Verbatim}

    \begin{center}
    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_117_1.png}
    \end{center}
    { \hspace*{\fill} \\}
    
    \begin{center}
    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_117_2.png}
    \end{center}
    { \hspace*{\fill} \\}
    
    \hypertarget{scikit-learn-implementation}{%
\subsection{scikit-learn
implementation}\label{scikit-learn-implementation}}

\textbf{scikit-learn} focuses more on traditional machine learning
methods, such as regression, clustering, decision trees, etc. As such,
it has only two types of neural networks: Multi Layer Perceptron
outputting continuous values, \emph{MPLRegressor}, and Multi Layer
Perceptron outputting labels, \emph{MLPClassifier}. We will see how
simple it is to use these classes.

\textbf{scikit-learn} implements a few improvements from our neural
network, such as early stopping, a varying learning rate, different
optimization methods, etc. We would therefore expect a better
performance overall.

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}12}]:} \PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{neural\PYZus{}network} \PY{k}{import} \PY{n}{MLPClassifier}
         \PY{c+c1}{\PYZsh{} store models for later use}
         \PY{n}{DNN\PYZus{}scikit} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
         
         \PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
                 \PY{n}{dnn} \PY{o}{=} \PY{n}{MLPClassifier}\PY{p}{(}\PY{n}{hidden\PYZus{}layer\PYZus{}sizes}\PY{o}{=}\PY{p}{(}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{logistic}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,}
                                     \PY{n}{alpha}\PY{o}{=}\PY{n}{lmbd}\PY{p}{,} \PY{n}{learning\PYZus{}rate\PYZus{}init}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{max\PYZus{}iter}\PY{o}{=}\PY{n}{epochs}\PY{p}{)}
                 \PY{n}{dnn}\PY{o}{.}\PY{n}{fit}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}
                 
                 \PY{n}{DNN\PYZus{}scikit}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{dnn}
                 
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate  = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Accuracy score on test set: }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{dnn}\PY{o}{.}\PY{n}{score}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{)}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
/usr/local/lib/python3.7/site-packages/sklearn/neural\_network/multilayer\_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
  \% self.max\_iter, ConvergenceWarning)

    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]
Learning rate  =  1e-05
Lambda =  1e-05
Accuracy score on test set:  0.20833333333333334

Learning rate  =  1e-05
Lambda =  0.0001
Accuracy score on test set:  0.16944444444444445

Learning rate  =  1e-05
Lambda =  0.001
Accuracy score on test set:  0.18055555555555555

Learning rate  =  1e-05
Lambda =  0.01
Accuracy score on test set:  0.17222222222222222

Learning rate  =  1e-05
Lambda =  0.1
Accuracy score on test set:  0.23333333333333334

Learning rate  =  1e-05
Lambda =  1.0
Accuracy score on test set:  0.2916666666666667

Learning rate  =  1e-05
Lambda =  10.0
Accuracy score on test set:  0.1527777777777778

Learning rate  =  0.0001
Lambda =  1e-05
Accuracy score on test set:  0.8666666666666667

Learning rate  =  0.0001
Lambda =  0.0001
Accuracy score on test set:  0.8416666666666667

Learning rate  =  0.0001
Lambda =  0.001
Accuracy score on test set:  0.9277777777777778

Learning rate  =  0.0001
Lambda =  0.01
Accuracy score on test set:  0.8861111111111111

Learning rate  =  0.0001
Lambda =  0.1
Accuracy score on test set:  0.8805555555555555

Learning rate  =  0.0001
Lambda =  1.0
Accuracy score on test set:  0.875

Learning rate  =  0.0001
Lambda =  10.0
Accuracy score on test set:  0.875

Learning rate  =  0.001
Lambda =  1e-05
Accuracy score on test set:  0.9833333333333333

Learning rate  =  0.001
Lambda =  0.0001
Accuracy score on test set:  0.9833333333333333

Learning rate  =  0.001
Lambda =  0.001
Accuracy score on test set:  0.9805555555555555

Learning rate  =  0.001
Lambda =  0.01
Accuracy score on test set:  0.9861111111111112

Learning rate  =  0.001
Lambda =  0.1
Accuracy score on test set:  0.9805555555555555

Learning rate  =  0.001
Lambda =  1.0
Accuracy score on test set:  0.9805555555555555

Learning rate  =  0.001
Lambda =  10.0
Accuracy score on test set:  0.9527777777777777

Learning rate  =  0.01
Lambda =  1e-05
Accuracy score on test set:  0.9861111111111112

Learning rate  =  0.01
Lambda =  0.0001
Accuracy score on test set:  0.9916666666666667

Learning rate  =  0.01
Lambda =  0.001
Accuracy score on test set:  0.9861111111111112

Learning rate  =  0.01
Lambda =  0.01
Accuracy score on test set:  0.9833333333333333

Learning rate  =  0.01
Lambda =  0.1
Accuracy score on test set:  0.9944444444444445

Learning rate  =  0.01
Lambda =  1.0
Accuracy score on test set:  0.975

Learning rate  =  0.01
Lambda =  10.0
Accuracy score on test set:  0.9416666666666667

Learning rate  =  0.1
Lambda =  1e-05
Accuracy score on test set:  0.9416666666666667

Learning rate  =  0.1
Lambda =  0.0001
Accuracy score on test set:  0.75

Learning rate  =  0.1
Lambda =  0.001
Accuracy score on test set:  0.8972222222222223

Learning rate  =  0.1
Lambda =  0.01
Accuracy score on test set:  0.8944444444444445

Learning rate  =  0.1
Lambda =  0.1
Accuracy score on test set:  0.9277777777777778

Learning rate  =  0.1
Lambda =  1.0
Accuracy score on test set:  0.8972222222222223

Learning rate  =  0.1
Lambda =  10.0
Accuracy score on test set:  0.7388888888888889

Learning rate  =  1.0
Lambda =  1e-05
Accuracy score on test set:  0.08611111111111111

Learning rate  =  1.0
Lambda =  0.0001
Accuracy score on test set:  0.10555555555555556

Learning rate  =  1.0
Lambda =  0.001
Accuracy score on test set:  0.11388888888888889

Learning rate  =  1.0
Lambda =  0.01
Accuracy score on test set:  0.1527777777777778

Learning rate  =  1.0
Lambda =  0.1
Accuracy score on test set:  0.15

Learning rate  =  1.0
Lambda =  1.0
Accuracy score on test set:  0.08888888888888889

Learning rate  =  1.0
Lambda =  10.0
Accuracy score on test set:  0.15

Learning rate  =  10.0
Lambda =  1e-05
Accuracy score on test set:  0.1

Learning rate  =  10.0
Lambda =  0.0001
Accuracy score on test set:  0.11944444444444445

Learning rate  =  10.0
Lambda =  0.001
Accuracy score on test set:  0.18055555555555555

Learning rate  =  10.0
Lambda =  0.01
Accuracy score on test set:  0.10555555555555556

Learning rate  =  10.0
Lambda =  0.1
Accuracy score on test set:  0.08333333333333333

Learning rate  =  10.0
Lambda =  1.0
Accuracy score on test set:  0.09166666666666666

Learning rate  =  10.0
Lambda =  10.0
Accuracy score on test set:  0.08055555555555556


    \end{Verbatim}

    \hypertarget{visualization}{%
\subsection{Visualization}\label{visualization}}

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}13}]:} \PY{c+c1}{\PYZsh{} optional}
         \PY{c+c1}{\PYZsh{} visual representation of grid search}
         \PY{c+c1}{\PYZsh{} uses seaborn heatmap, could probably do this in matplotlib}
         \PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
         
         \PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         \PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         
         \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
                 \PY{n}{dnn} \PY{o}{=} \PY{n}{DNN\PYZus{}scikit}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
                 
                 \PY{n}{train\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)} 
                 \PY{n}{test\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
         
                 \PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{train\PYZus{}pred}\PY{p}{)}
                 \PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}pred}\PY{p}{)}
         
                 
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{center}
    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_121_0.png}
    \end{center}
    { \hspace*{\fill} \\}
    
    \begin{center}
    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_121_1.png}
    \end{center}
    { \hspace*{\fill} \\}
    
    \hypertarget{building-neural-networks-in-tensorflow-and-keras}{%
\subsection{Building neural networks in Tensorflow and
Keras}\label{building-neural-networks-in-tensorflow-and-keras}}

Now we want to build on the experience gained from our neural network
implementation in NumPy and scikit-learn and use it to construct a
neural network in Tensorflow. Once we have constructed a neural network
in NumPy and Tensorflow, building one in Keras is really quite trivial,
though the performance may suffer.

In our previous example we used only one hidden layer, and in this we
will use two. From this it should be quite clear how to build one using
an arbitrary number of hidden layers, using data structures such as
Python lists or NumPy arrays.

\hypertarget{tensorflow}{%
\subsection{Tensorflow}\label{tensorflow}}

Tensorflow is an open source library machine learning library developed
by the Google Brain team for internal use. It was released under the
Apache 2.0 open source license in November 9, 2015.

Tensorflow is a computational framework that allows you to construct
machine learning models at different levels of abstraction, from
high-level, object-oriented APIs like Keras, down to the C++ kernels
that Tensorflow is built upon. The higher levels of abstraction are
simpler to use, but less flexible, and our choice of implementation
should reflect the problems we are trying to solve.

\href{https://www.tensorflow.org/guide/graphs}{Tensorflow uses}
so-called graphs to represent your computation in terms of the
dependencies between individual operations, such that you first build a
Tensorflow \emph{graph} to represent your model, and then create a
Tensorflow \emph{session} to run the graph.

In this guide we will analyze the same data as we did in our NumPy and
scikit-learn tutorial, gathered from the MNIST database of images. We
will give an introduction to the lower level Python Application Program
Interfaces (APIs), and see how we use them to build our graph. Then we
will build (effectively) the same graph in Keras, to see just how simple
solving a machine learning problem can be.

To install tensorflow on Unix/Linux systems, use pip as

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}13}]:} \PY{n}{pip3} \PY{n}{install} \PY{n}{tensorflow}
\end{Verbatim}


    and/or if you use \textbf{anaconda}, just write (or install from the
graphical user interface)

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}14}]:} \PY{n}{conda} \PY{n}{install} \PY{n}{tensorflow}
\end{Verbatim}


    \hypertarget{collect-and-pre-process-data}{%
\subsection{Collect and pre-process
data}\label{collect-and-pre-process-data}}

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}14}]:} \PY{c+c1}{\PYZsh{} import necessary packages}
         \PY{k+kn}{import} \PY{n+nn}{numpy} \PY{k}{as} \PY{n+nn}{np}
         \PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt}
         \PY{k+kn}{from} \PY{n+nn}{sklearn} \PY{k}{import} \PY{n}{datasets}
         
         
         \PY{c+c1}{\PYZsh{} ensure the same random numbers appear every time}
         \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{seed}\PY{p}{(}\PY{l+m+mi}{0}\PY{p}{)}
         
         \PY{c+c1}{\PYZsh{} display images in notebook}
         \PY{o}{\PYZpc{}}\PY{k}{matplotlib} inline
         \PY{n}{plt}\PY{o}{.}\PY{n}{rcParams}\PY{p}{[}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{figure.figsize}\PY{l+s+s1}{\PYZsq{}}\PY{p}{]} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{12}\PY{p}{,}\PY{l+m+mi}{12}\PY{p}{)}
         
         
         \PY{c+c1}{\PYZsh{} download MNIST dataset}
         \PY{n}{digits} \PY{o}{=} \PY{n}{datasets}\PY{o}{.}\PY{n}{load\PYZus{}digits}\PY{p}{(}\PY{p}{)}
         
         \PY{c+c1}{\PYZsh{} define inputs and labels}
         \PY{n}{inputs} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{images}
         \PY{n}{labels} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{target}
         
         \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{inputs = (n\PYZus{}inputs, pixel\PYZus{}width, pixel\PYZus{}height) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
         \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{labels = (n\PYZus{}inputs) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{labels}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
         
         
         \PY{c+c1}{\PYZsh{} flatten the image}
         \PY{c+c1}{\PYZsh{} the value \PYZhy{}1 means dimension is inferred from the remaining dimensions: 8x8 = 64}
         \PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{inputs}\PY{p}{)}
         \PY{n}{inputs} \PY{o}{=} \PY{n}{inputs}\PY{o}{.}\PY{n}{reshape}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{o}{\PYZhy{}}\PY{l+m+mi}{1}\PY{p}{)}
         \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{X = (n\PYZus{}inputs, n\PYZus{}features) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
         
         
         \PY{c+c1}{\PYZsh{} choose some random images to display}
         \PY{n}{indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
         \PY{n}{random\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}\PY{n}{indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{l+m+mi}{5}\PY{p}{)}
         
         \PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{image} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{digits}\PY{o}{.}\PY{n}{images}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{]}\PY{p}{)}\PY{p}{:}
             \PY{n}{plt}\PY{o}{.}\PY{n}{subplot}\PY{p}{(}\PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{n}{i}\PY{o}{+}\PY{l+m+mi}{1}\PY{p}{)}
             \PY{n}{plt}\PY{o}{.}\PY{n}{axis}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{off}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
             \PY{n}{plt}\PY{o}{.}\PY{n}{imshow}\PY{p}{(}\PY{n}{image}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{n}{plt}\PY{o}{.}\PY{n}{cm}\PY{o}{.}\PY{n}{gray\PYZus{}r}\PY{p}{,} \PY{n}{interpolation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{nearest}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
             \PY{n}{plt}\PY{o}{.}\PY{n}{title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Label: }\PY{l+s+si}{\PYZpc{}d}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{digits}\PY{o}{.}\PY{n}{target}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{]}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
inputs = (n\_inputs, pixel\_width, pixel\_height) = (1797, 8, 8)
labels = (n\_inputs) = (1797,)
X = (n\_inputs, n\_features) = (1797, 64)

    \end{Verbatim}

    \begin{center}
    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_127_1.png}
    \end{center}
    { \hspace*{\fill} \\}
    
    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}15}]:} \PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{utils} \PY{k}{import} \PY{n}{to\PYZus{}categorical}
         \PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{model\PYZus{}selection} \PY{k}{import} \PY{n}{train\PYZus{}test\PYZus{}split}
         
         \PY{c+c1}{\PYZsh{} one\PYZhy{}hot representation of labels}
         \PY{n}{labels} \PY{o}{=} \PY{n}{to\PYZus{}categorical}\PY{p}{(}\PY{n}{labels}\PY{p}{)}
         
         \PY{c+c1}{\PYZsh{} split into train and test data}
         \PY{n}{train\PYZus{}size} \PY{o}{=} \PY{l+m+mf}{0.8}
         \PY{n}{test\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n}{train\PYZus{}size}
         \PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{train\PYZus{}test\PYZus{}split}\PY{p}{(}\PY{n}{inputs}\PY{p}{,} \PY{n}{labels}\PY{p}{,} \PY{n}{train\PYZus{}size}\PY{o}{=}\PY{n}{train\PYZus{}size}\PY{p}{,}
                                                             \PY{n}{test\PYZus{}size}\PY{o}{=}\PY{n}{test\PYZus{}size}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
Using TensorFlow backend.

    \end{Verbatim}

    \begin{Verbatim}[commandchars=\\\{\}]

        ---------------------------------------------------------------------------

        ModuleNotFoundError                       Traceback (most recent call last)

        <ipython-input-15-b532db97c8ce> in <module>()
    ----> 1 from keras.utils import to\_categorical
          2 from sklearn.model\_selection import train\_test\_split
          3 
          4 \# one-hot representation of labels
          5 labels = to\_categorical(labels)


        /usr/local/lib/python3.7/site-packages/keras/\_\_init\_\_.py in <module>()
          1 from \_\_future\_\_ import absolute\_import
          2 
    ----> 3 from . import utils
          4 from . import activations
          5 from . import applications


        /usr/local/lib/python3.7/site-packages/keras/utils/\_\_init\_\_.py in <module>()
          4 from . import data\_utils
          5 from . import io\_utils
    ----> 6 from . import conv\_utils
          7 
          8 \# Globally-importable utils.


        /usr/local/lib/python3.7/site-packages/keras/utils/conv\_utils.py in <module>()
          7 from six.moves import range
          8 import numpy as np
    ----> 9 from .. import backend as K
         10 
         11 


        /usr/local/lib/python3.7/site-packages/keras/backend/\_\_init\_\_.py in <module>()
         87 elif \_BACKEND == 'tensorflow':
         88     sys.stderr.write('Using TensorFlow backend.\textbackslash{}n')
    ---> 89     from .tensorflow\_backend import *
         90 else:
         91     \# Try and load external backend.


        /usr/local/lib/python3.7/site-packages/keras/backend/tensorflow\_backend.py in <module>()
          3 from \_\_future\_\_ import print\_function
          4 
    ----> 5 import tensorflow as tf
          6 from tensorflow.python.framework import ops as tf\_ops
          7 from tensorflow.python.training import moving\_averages


        ModuleNotFoundError: No module named 'tensorflow'

    \end{Verbatim}

    \hypertarget{using-tensorflow-backend}{%
\subsection{Using TensorFlow backend}\label{using-tensorflow-backend}}

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  Define model and architecture
\item
  Choose cost function and optimizer
\end{enumerate}

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}17}]:} \PY{k+kn}{import} \PY{n+nn}{tensorflow} \PY{k}{as} \PY{n+nn}{tf}
         
         \PY{k}{class} \PY{n+nc}{NeuralNetworkTensorflow}\PY{p}{:}
             \PY{k}{def} \PY{n+nf}{\PYZus{}\PYZus{}init\PYZus{}\PYZus{}}\PY{p}{(}
                 \PY{n+nb+bp}{self}\PY{p}{,}
                 \PY{n}{X\PYZus{}train}\PY{p}{,}
                 \PY{n}{Y\PYZus{}train}\PY{p}{,}
                 \PY{n}{X\PYZus{}test}\PY{p}{,}
                 \PY{n}{Y\PYZus{}test}\PY{p}{,}
                 \PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{o}{=}\PY{l+m+mi}{100}\PY{p}{,}
                 \PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{o}{=}\PY{l+m+mi}{50}\PY{p}{,}
                 \PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{l+m+mi}{2}\PY{p}{,}
                 \PY{n}{epochs}\PY{o}{=}\PY{l+m+mi}{10}\PY{p}{,}
                 \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{l+m+mi}{100}\PY{p}{,}
                 \PY{n}{eta}\PY{o}{=}\PY{l+m+mf}{0.1}\PY{p}{,}
                 \PY{n}{lmbd}\PY{o}{=}\PY{l+m+mf}{0.0}\PY{p}{,}
             \PY{p}{)}\PY{p}{:}
                 
                 \PY{c+c1}{\PYZsh{} keep track of number of steps}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{global\PYZus{}step} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{Variable}\PY{p}{(}\PY{l+m+mi}{0}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{int32}\PY{p}{,} \PY{n}{trainable}\PY{o}{=}\PY{k+kc}{False}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{global\PYZus{}step}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
                 
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}train} \PY{o}{=} \PY{n}{X\PYZus{}train}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}train} \PY{o}{=} \PY{n}{Y\PYZus{}train}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}test} \PY{o}{=} \PY{n}{X\PYZus{}test}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{Y\PYZus{}test}
                 
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n}{X\PYZus{}train}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features} \PY{o}{=} \PY{n}{X\PYZus{}train}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1} \PY{o}{=} \PY{n}{n\PYZus{}neurons\PYZus{}layer1}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2} \PY{o}{=} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{n}{n\PYZus{}categories}
                 
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs} \PY{o}{=} \PY{n}{epochs}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{n}{batch\PYZus{}size}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{/}\PY{o}{/} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{=} \PY{n}{eta}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{=} \PY{n}{lmbd}
                 
                 \PY{c+c1}{\PYZsh{} build network piece by piece}
                 \PY{c+c1}{\PYZsh{} name scopes (with) are used to enforce creation of new variables}
                 \PY{c+c1}{\PYZsh{} https://www.tensorflow.org/guide/variables}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}placeholders}\PY{p}{(}\PY{p}{)}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}DNN}\PY{p}{(}\PY{p}{)}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}loss}\PY{p}{(}\PY{p}{)}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}optimiser}\PY{p}{(}\PY{p}{)}
                 \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}accuracy}\PY{p}{(}\PY{p}{)}
             
             \PY{k}{def} \PY{n+nf}{create\PYZus{}placeholders}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                 \PY{c+c1}{\PYZsh{} placeholders are fine here, but \PYZdq{}Datasets\PYZdq{} are the preferred method}
                 \PY{c+c1}{\PYZsh{} of streaming data into a model}
                 \PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{data}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{placeholder}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{,} \PY{n}{shape}\PY{o}{=}\PY{p}{(}\PY{k+kc}{None}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features}\PY{p}{)}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{X\PYZus{}data}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{placeholder}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{,} \PY{n}{shape}\PY{o}{=}\PY{p}{(}\PY{k+kc}{None}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{)}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{Y\PYZus{}data}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
             
             \PY{k}{def} \PY{n+nf}{create\PYZus{}DNN}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                 \PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{DNN}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
                     \PY{c+c1}{\PYZsh{} the weights are stored to calculate regularization loss later}
                     
                     \PY{c+c1}{\PYZsh{} Fully connected layer 1}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc1} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{weight\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc1}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
                     \PY{n}{b\PYZus{}fc1} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{bias\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc1}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
                     \PY{n}{a\PYZus{}fc1} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{sigmoid}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc1}\PY{p}{)} \PY{o}{+} \PY{n}{b\PYZus{}fc1}\PY{p}{)}
                     
                     \PY{c+c1}{\PYZsh{} Fully connected layer 2}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc2} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{weight\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc2}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
                     \PY{n}{b\PYZus{}fc2} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{bias\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc2}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
                     \PY{n}{a\PYZus{}fc2} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{sigmoid}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}fc1}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc2}\PY{p}{)} \PY{o}{+} \PY{n}{b\PYZus{}fc2}\PY{p}{)}
                     
                     \PY{c+c1}{\PYZsh{} Output layer}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}out} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{weight\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{out}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
                     \PY{n}{b\PYZus{}out} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{bias\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{out}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}out} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}fc2}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}out}\PY{p}{)} \PY{o}{+} \PY{n}{b\PYZus{}out}
             
             \PY{k}{def} \PY{n+nf}{create\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                 \PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{loss}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
                     \PY{n}{softmax\PYZus{}loss} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{reduce\PYZus{}mean}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{softmax\PYZus{}cross\PYZus{}entropy\PYZus{}with\PYZus{}logits\PYZus{}v2}\PY{p}{(}\PY{n}{labels}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y}\PY{p}{,} \PY{n}{logits}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}out}\PY{p}{)}\PY{p}{)}
                     
                     \PY{n}{regularizer\PYZus{}loss\PYZus{}fc1} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{l2\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc1}\PY{p}{)}
                     \PY{n}{regularizer\PYZus{}loss\PYZus{}fc2} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{l2\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc2}\PY{p}{)}
                     \PY{n}{regularizer\PYZus{}loss\PYZus{}out} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{l2\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}out}\PY{p}{)}
                     \PY{n}{regularizer\PYZus{}loss} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd}\PY{o}{*}\PY{p}{(}\PY{n}{regularizer\PYZus{}loss\PYZus{}fc1} \PY{o}{+} \PY{n}{regularizer\PYZus{}loss\PYZus{}fc2} \PY{o}{+} \PY{n}{regularizer\PYZus{}loss\PYZus{}out}\PY{p}{)}
                     
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{loss} \PY{o}{=} \PY{n}{softmax\PYZus{}loss} \PY{o}{+} \PY{n}{regularizer\PYZus{}loss}
         
             \PY{k}{def} \PY{n+nf}{create\PYZus{}accuracy}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                 \PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{accuracy}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
                     \PY{n}{probabilities} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{softmax}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}out}\PY{p}{)}
                     \PY{n}{predictions} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n}{probabilities}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
                     \PY{n}{labels} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
                     
                     \PY{n}{correct\PYZus{}predictions} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{equal}\PY{p}{(}\PY{n}{predictions}\PY{p}{,} \PY{n}{labels}\PY{p}{)}
                     \PY{n}{correct\PYZus{}predictions} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{cast}\PY{p}{(}\PY{n}{correct\PYZus{}predictions}\PY{p}{,} \PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{accuracy} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{reduce\PYZus{}mean}\PY{p}{(}\PY{n}{correct\PYZus{}predictions}\PY{p}{)}
             
             \PY{k}{def} \PY{n+nf}{create\PYZus{}optimiser}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                 \PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{optimizer}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{optimizer} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{train}\PY{o}{.}\PY{n}{GradientDescentOptimizer}\PY{p}{(}\PY{n}{learning\PYZus{}rate}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta}\PY{p}{)}\PY{o}{.}\PY{n}{minimize}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{global\PYZus{}step}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{global\PYZus{}step}\PY{p}{)}
                     
             \PY{k}{def} \PY{n+nf}{weight\PYZus{}variable}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{shape}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}\PY{p}{:}
                 \PY{n}{initial} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{truncated\PYZus{}normal}\PY{p}{(}\PY{n}{shape}\PY{p}{,} \PY{n}{stddev}\PY{o}{=}\PY{l+m+mf}{0.1}\PY{p}{)}
                 \PY{k}{return} \PY{n}{tf}\PY{o}{.}\PY{n}{Variable}\PY{p}{(}\PY{n}{initial}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{n}{name}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{dtype}\PY{p}{)}
             
             \PY{k}{def} \PY{n+nf}{bias\PYZus{}variable}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{shape}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}\PY{p}{:}
                 \PY{n}{initial} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{constant}\PY{p}{(}\PY{l+m+mf}{0.1}\PY{p}{,} \PY{n}{shape}\PY{o}{=}\PY{n}{shape}\PY{p}{)}
                 \PY{k}{return} \PY{n}{tf}\PY{o}{.}\PY{n}{Variable}\PY{p}{(}\PY{n}{initial}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{n}{name}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{dtype}\PY{p}{)}
             
             \PY{k}{def} \PY{n+nf}{fit}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
                 \PY{n}{data\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
         
                 \PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{Session}\PY{p}{(}\PY{p}{)} \PY{k}{as} \PY{n}{sess}\PY{p}{:}
                     \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{global\PYZus{}variables\PYZus{}initializer}\PY{p}{(}\PY{p}{)}\PY{p}{)}
                     \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs}\PY{p}{)}\PY{p}{:}
                         \PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations}\PY{p}{)}\PY{p}{:}
                             \PY{n}{chosen\PYZus{}datapoints} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}\PY{n}{data\PYZus{}indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{replace}\PY{o}{=}\PY{k+kc}{False}\PY{p}{)}
                             \PY{n}{batch\PYZus{}X}\PY{p}{,} \PY{n}{batch\PYZus{}Y} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}train}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}train}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}
                     
                             \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{p}{[}\PY{n}{DNN}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{DNN}\PY{o}{.}\PY{n}{optimizer}\PY{p}{]}\PY{p}{,}
                                 \PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n}{batch\PYZus{}X}\PY{p}{,}
                                            \PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n}{batch\PYZus{}Y}\PY{p}{\PYZcb{}}\PY{p}{)}
                             \PY{n}{accuracy} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{n}{DNN}\PY{o}{.}\PY{n}{accuracy}\PY{p}{,}
                                 \PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n}{batch\PYZus{}X}\PY{p}{,}
                                            \PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n}{batch\PYZus{}Y}\PY{p}{\PYZcb{}}\PY{p}{)}
                             \PY{n}{step} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{n}{DNN}\PY{o}{.}\PY{n}{global\PYZus{}step}\PY{p}{)}
             
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{train\PYZus{}loss}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{p}{[}\PY{n}{DNN}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{DNN}\PY{o}{.}\PY{n}{accuracy}\PY{p}{]}\PY{p}{,}
                         \PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}train}\PY{p}{,}
                                    \PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}train}\PY{p}{\PYZcb{}}\PY{p}{)}
                 
                     \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{test\PYZus{}loss}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{p}{[}\PY{n}{DNN}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{DNN}\PY{o}{.}\PY{n}{accuracy}\PY{p}{]}\PY{p}{,}
                         \PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}test}\PY{p}{,}
                                    \PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}test}\PY{p}{\PYZcb{}}\PY{p}{)}
\end{Verbatim}


    \hypertarget{optimizing-and-using-gradient-descent}{%
\subsection{Optimizing and using gradient
descent}\label{optimizing-and-using-gradient-descent}}

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}18}]:} \PY{n}{epochs} \PY{o}{=} \PY{l+m+mi}{100}
         \PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{100}
         \PY{n}{n\PYZus{}neurons\PYZus{}layer1} \PY{o}{=} \PY{l+m+mi}{100}
         \PY{n}{n\PYZus{}neurons\PYZus{}layer2} \PY{o}{=} \PY{l+m+mi}{50}
         \PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{l+m+mi}{10}
         \PY{n}{eta\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
         \PY{n}{lmbd\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}19}]:} \PY{n}{DNN\PYZus{}tf} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
                 
         \PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
                 \PY{n}{DNN} \PY{o}{=} \PY{n}{NeuralNetworkTensorflow}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{,}
                                               \PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{,}
                                               \PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{)}
                 \PY{n}{DNN}\PY{o}{.}\PY{n}{fit}\PY{p}{(}\PY{p}{)}
                 
                 \PY{n}{DNN\PYZus{}tf}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}
                 
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test accuracy: }\PY{l+s+si}{\PYZpc{}.3f}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{DNN}\PY{o}{.}\PY{n}{test\PYZus{}accuracy}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}20}]:} \PY{c+c1}{\PYZsh{} optional}
         \PY{c+c1}{\PYZsh{} visual representation of grid search}
         \PY{c+c1}{\PYZsh{} uses seaborn heatmap, could probably do this in matplotlib}
         \PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
         
         \PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         \PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         
         \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
                 \PY{n}{DNN} \PY{o}{=} \PY{n}{DNN\PYZus{}tf}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
         
                 \PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{train\PYZus{}accuracy}
                 \PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{test\PYZus{}accuracy}
         
                 
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}21}]:} \PY{c+c1}{\PYZsh{} optional}
         \PY{c+c1}{\PYZsh{} we can use log files to visualize our graph in Tensorboard}
         \PY{n}{writer} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{summary}\PY{o}{.}\PY{n}{FileWriter}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{logs/}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
         \PY{n}{writer}\PY{o}{.}\PY{n}{add\PYZus{}graph}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{get\PYZus{}default\PYZus{}graph}\PY{p}{(}\PY{p}{)}\PY{p}{)}
\end{Verbatim}


    \hypertarget{using-keras}{%
\subsection{Using Keras}\label{using-keras}}

Keras is a high level
\href{https://en.wikipedia.org/wiki/Application_programming_interface}{neural
network} that supports Tensorflow, CTNK and Theano as backends.\\
If you have Tensorflow installed Keras is available through the
\emph{tf.keras} module.\\
If you have Anaconda installed you may run the following command

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}22}]:} \PY{n}{conda} \PY{n}{install} \PY{n}{keras}
\end{Verbatim}


    Alternatively, if you have Tensorflow or one of the other supported
backends install you may use the pip package manager:

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}23}]:} \PY{n}{pip3} \PY{n}{install} \PY{n}{keras}
\end{Verbatim}


    or look up the \href{https://keras.io/}{instructions here}.

    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}24}]:} \PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{models} \PY{k}{import} \PY{n}{Sequential}
         \PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{layers} \PY{k}{import} \PY{n}{Dense}
         \PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{regularizers} \PY{k}{import} \PY{n}{l2}
         \PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{optimizers} \PY{k}{import} \PY{n}{SGD}
         
         \PY{k}{def} \PY{n+nf}{create\PYZus{}neural\PYZus{}network\PYZus{}keras}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{,} \PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}\PY{p}{:}
             \PY{n}{model} \PY{o}{=} \PY{n}{Sequential}\PY{p}{(}\PY{p}{)}
             \PY{n}{model}\PY{o}{.}\PY{n}{add}\PY{p}{(}\PY{n}{Dense}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sigmoid}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{kernel\PYZus{}regularizer}\PY{o}{=}\PY{n}{l2}\PY{p}{(}\PY{n}{lmbd}\PY{p}{)}\PY{p}{)}\PY{p}{)}
             \PY{n}{model}\PY{o}{.}\PY{n}{add}\PY{p}{(}\PY{n}{Dense}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sigmoid}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{kernel\PYZus{}regularizer}\PY{o}{=}\PY{n}{l2}\PY{p}{(}\PY{n}{lmbd}\PY{p}{)}\PY{p}{)}\PY{p}{)}
             \PY{n}{model}\PY{o}{.}\PY{n}{add}\PY{p}{(}\PY{n}{Dense}\PY{p}{(}\PY{n}{n\PYZus{}categories}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{softmax}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{)}
             
             \PY{n}{sgd} \PY{o}{=} \PY{n}{SGD}\PY{p}{(}\PY{n}{lr}\PY{o}{=}\PY{n}{eta}\PY{p}{)}
             \PY{n}{model}\PY{o}{.}\PY{n}{compile}\PY{p}{(}\PY{n}{loss}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{categorical\PYZus{}crossentropy}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{optimizer}\PY{o}{=}\PY{n}{sgd}\PY{p}{,} \PY{n}{metrics}\PY{o}{=}\PY{p}{[}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{accuracy}\PY{l+s+s1}{\PYZsq{}}\PY{p}{]}\PY{p}{)}
             
             \PY{k}{return} \PY{n}{model}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}25}]:} \PY{n}{DNN\PYZus{}keras} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
                 
         \PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
                 \PY{n}{DNN} \PY{o}{=} \PY{n}{create\PYZus{}neural\PYZus{}network\PYZus{}keras}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{,}
                                                  \PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{)}
                 \PY{n}{DNN}\PY{o}{.}\PY{n}{fit}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{verbose}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
                 \PY{n}{scores} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{evaluate}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{)}
                 
                 \PY{n}{DNN\PYZus{}keras}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}
                 
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test accuracy: }\PY{l+s+si}{\PYZpc{}.3f}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{scores}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}\PY{p}{)}
                 \PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
\end{Verbatim}


    \begin{Verbatim}[commandchars=\\\{\}]
{\color{incolor}In [{\color{incolor}26}]:} \PY{c+c1}{\PYZsh{} optional}
         \PY{c+c1}{\PYZsh{} visual representation of grid search}
         \PY{c+c1}{\PYZsh{} uses seaborn heatmap, could probably do this in matplotlib}
         \PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
         
         \PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         \PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
         
         \PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
             \PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
                 \PY{n}{DNN} \PY{o}{=} \PY{n}{DNN\PYZus{}keras}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
         
                 \PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{evaluate}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
                 \PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{evaluate}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{)}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
         
                 
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
         
         \PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
         \PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
         \PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
\end{Verbatim}



    % Add a bibliography block to the postdoc
    
    
    
    \end{document}
