mhjensen
3333 lines
178 KiB
TeX
3333 lines
178 KiB
TeX
|
|
% Default to the notebook output style
|
|
|
|
|
|
|
|
|
|
% Inherit from the specified cell style.
|
|
|
|
|
|
|
|
|
|
|
|
\documentclass[11pt]{article}
|
|
|
|
|
|
|
|
\usepackage[T1]{fontenc}
|
|
% Nicer default font (+ math font) than Computer Modern for most use cases
|
|
\usepackage{mathpazo}
|
|
|
|
% Basic figure setup, for now with no caption control since it's done
|
|
% automatically by Pandoc (which extracts  syntax from Markdown).
|
|
\usepackage{graphicx}
|
|
% We will generate all images so they have a width \maxwidth. This means
|
|
% that they will get their normal width if they fit onto the page, but
|
|
% are scaled down if they would overflow the margins.
|
|
\makeatletter
|
|
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth
|
|
\else\Gin@nat@width\fi}
|
|
\makeatother
|
|
\let\Oldincludegraphics\includegraphics
|
|
% Set max figure width to be 80% of text width, for now hardcoded.
|
|
\renewcommand{\includegraphics}[1]{\Oldincludegraphics[width=.8\maxwidth]{#1}}
|
|
% Ensure that by default, figures have no caption (until we provide a
|
|
% proper Figure object with a Caption API and a way to capture that
|
|
% in the conversion process - todo).
|
|
\usepackage{caption}
|
|
\DeclareCaptionLabelFormat{nolabel}{}
|
|
\captionsetup{labelformat=nolabel}
|
|
|
|
\usepackage{adjustbox} % Used to constrain images to a maximum size
|
|
\usepackage{xcolor} % Allow colors to be defined
|
|
\usepackage{enumerate} % Needed for markdown enumerations to work
|
|
\usepackage{geometry} % Used to adjust the document margins
|
|
\usepackage{amsmath} % Equations
|
|
\usepackage{amssymb} % Equations
|
|
\usepackage{textcomp} % defines textquotesingle
|
|
% Hack from http://tex.stackexchange.com/a/47451/13684:
|
|
\AtBeginDocument{%
|
|
\def\PYZsq{\textquotesingle}% Upright quotes in Pygmentized code
|
|
}
|
|
\usepackage{upquote} % Upright quotes for verbatim code
|
|
\usepackage{eurosym} % defines \euro
|
|
\usepackage[mathletters]{ucs} % Extended unicode (utf-8) support
|
|
\usepackage[utf8x]{inputenc} % Allow utf-8 characters in the tex document
|
|
\usepackage{fancyvrb} % verbatim replacement that allows latex
|
|
\usepackage{grffile} % extends the file name processing of package graphics
|
|
% to support a larger range
|
|
% The hyperref package gives us a pdf with properly built
|
|
% internal navigation ('pdf bookmarks' for the table of contents,
|
|
% internal cross-reference links, web links for URLs, etc.)
|
|
\usepackage{hyperref}
|
|
\usepackage{longtable} % longtable support required by pandoc >1.10
|
|
\usepackage{booktabs} % table support for pandoc > 1.12.2
|
|
\usepackage[inline]{enumitem} % IRkernel/repr support (it uses the enumerate* environment)
|
|
\usepackage[normalem]{ulem} % ulem is needed to support strikethroughs (\sout)
|
|
% normalem makes italics be italics, not underlines
|
|
|
|
|
|
|
|
|
|
% Colors for the hyperref package
|
|
\definecolor{urlcolor}{rgb}{0,.145,.698}
|
|
\definecolor{linkcolor}{rgb}{.71,0.21,0.01}
|
|
\definecolor{citecolor}{rgb}{.12,.54,.11}
|
|
|
|
% ANSI colors
|
|
\definecolor{ansi-black}{HTML}{3E424D}
|
|
\definecolor{ansi-black-intense}{HTML}{282C36}
|
|
\definecolor{ansi-red}{HTML}{E75C58}
|
|
\definecolor{ansi-red-intense}{HTML}{B22B31}
|
|
\definecolor{ansi-green}{HTML}{00A250}
|
|
\definecolor{ansi-green-intense}{HTML}{007427}
|
|
\definecolor{ansi-yellow}{HTML}{DDB62B}
|
|
\definecolor{ansi-yellow-intense}{HTML}{B27D12}
|
|
\definecolor{ansi-blue}{HTML}{208FFB}
|
|
\definecolor{ansi-blue-intense}{HTML}{0065CA}
|
|
\definecolor{ansi-magenta}{HTML}{D160C4}
|
|
\definecolor{ansi-magenta-intense}{HTML}{A03196}
|
|
\definecolor{ansi-cyan}{HTML}{60C6C8}
|
|
\definecolor{ansi-cyan-intense}{HTML}{258F8F}
|
|
\definecolor{ansi-white}{HTML}{C5C1B4}
|
|
\definecolor{ansi-white-intense}{HTML}{A1A6B2}
|
|
|
|
% commands and environments needed by pandoc snippets
|
|
% extracted from the output of `pandoc -s`
|
|
\providecommand{\tightlist}{%
|
|
\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
|
|
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
|
|
% Add ',fontsize=\small' for more characters per line
|
|
\newenvironment{Shaded}{}{}
|
|
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
|
|
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}}
|
|
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
|
|
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
|
|
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
|
|
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
|
|
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
|
|
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}}
|
|
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
|
|
\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
|
|
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}}
|
|
\newcommand{\RegionMarkerTok}[1]{{#1}}
|
|
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
|
|
\newcommand{\NormalTok}[1]{{#1}}
|
|
|
|
% Additional commands for more recent versions of Pandoc
|
|
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.53,0.00,0.00}{{#1}}}
|
|
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
|
|
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
|
|
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.73,0.40,0.53}{{#1}}}
|
|
\newcommand{\ImportTok}[1]{{#1}}
|
|
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.73,0.13,0.13}{\textit{{#1}}}}
|
|
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
|
|
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
|
|
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.10,0.09,0.49}{{#1}}}
|
|
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
|
|
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.40,0.40,0.40}{{#1}}}
|
|
\newcommand{\BuiltInTok}[1]{{#1}}
|
|
\newcommand{\ExtensionTok}[1]{{#1}}
|
|
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.74,0.48,0.00}{{#1}}}
|
|
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.49,0.56,0.16}{{#1}}}
|
|
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
|
|
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textbf{\textit{{#1}}}}}
|
|
|
|
|
|
% Define a nice break command that doesn't care if a line doesn't already
|
|
% exist.
|
|
\def\br{\hspace*{\fill} \\* }
|
|
% Math Jax compatability definitions
|
|
\def\gt{>}
|
|
\def\lt{<}
|
|
% Document parameters
|
|
\title{NeuralNet}
|
|
|
|
|
|
|
|
|
|
% Pygments definitions
|
|
|
|
\makeatletter
|
|
\def\PY@reset{\let\PY@it=\relax \let\PY@bf=\relax%
|
|
\let\PY@ul=\relax \let\PY@tc=\relax%
|
|
\let\PY@bc=\relax \let\PY@ff=\relax}
|
|
\def\PY@tok#1{\csname PY@tok@#1\endcsname}
|
|
\def\PY@toks#1+{\ifx\relax#1\empty\else%
|
|
\PY@tok{#1}\expandafter\PY@toks\fi}
|
|
\def\PY@do#1{\PY@bc{\PY@tc{\PY@ul{%
|
|
\PY@it{\PY@bf{\PY@ff{#1}}}}}}}
|
|
\def\PY#1#2{\PY@reset\PY@toks#1+\relax+\PY@do{#2}}
|
|
|
|
\expandafter\def\csname PY@tok@w\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}}
|
|
\expandafter\def\csname PY@tok@c\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@cp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.74,0.48,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@k\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@kp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@kt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.69,0.00,0.25}{##1}}}
|
|
\expandafter\def\csname PY@tok@o\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@ow\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@nb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@nf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@nc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@nn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@ne\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.82,0.25,0.23}{##1}}}
|
|
\expandafter\def\csname PY@tok@nv\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
|
|
\expandafter\def\csname PY@tok@no\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@nl\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.63,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@ni\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.60,0.60,0.60}{##1}}}
|
|
\expandafter\def\csname PY@tok@na\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.49,0.56,0.16}{##1}}}
|
|
\expandafter\def\csname PY@tok@nt\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@nd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.67,0.13,1.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@s\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@sd\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@si\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}}
|
|
\expandafter\def\csname PY@tok@se\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@sr\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.40,0.53}{##1}}}
|
|
\expandafter\def\csname PY@tok@ss\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
|
|
\expandafter\def\csname PY@tok@sx\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@m\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@gh\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@gu\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@gd\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@gi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@gr\endcsname{\def\PY@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@ge\endcsname{\let\PY@it=\textit}
|
|
\expandafter\def\csname PY@tok@gs\endcsname{\let\PY@bf=\textbf}
|
|
\expandafter\def\csname PY@tok@gp\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@go\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}}
|
|
\expandafter\def\csname PY@tok@gt\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}}
|
|
\expandafter\def\csname PY@tok@err\endcsname{\def\PY@bc##1{\setlength{\fboxsep}{0pt}\fcolorbox[rgb]{1.00,0.00,0.00}{1,1,1}{\strut ##1}}}
|
|
\expandafter\def\csname PY@tok@kc\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@kd\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@kn\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@kr\endcsname{\let\PY@bf=\textbf\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@bp\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.50,0.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@fm\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.00,0.00,1.00}{##1}}}
|
|
\expandafter\def\csname PY@tok@vc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
|
|
\expandafter\def\csname PY@tok@vg\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
|
|
\expandafter\def\csname PY@tok@vi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
|
|
\expandafter\def\csname PY@tok@vm\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.10,0.09,0.49}{##1}}}
|
|
\expandafter\def\csname PY@tok@sa\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@sb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@sc\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@dl\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@s2\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@sh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@s1\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.73,0.13,0.13}{##1}}}
|
|
\expandafter\def\csname PY@tok@mb\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@mf\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@mh\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@mi\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@il\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@mo\endcsname{\def\PY@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}}
|
|
\expandafter\def\csname PY@tok@ch\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@cm\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@cpf\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@c1\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
|
|
\expandafter\def\csname PY@tok@cs\endcsname{\let\PY@it=\textit\def\PY@tc##1{\textcolor[rgb]{0.25,0.50,0.50}{##1}}}
|
|
|
|
\def\PYZbs{\char`\\}
|
|
\def\PYZus{\char`\_}
|
|
\def\PYZob{\char`\{}
|
|
\def\PYZcb{\char`\}}
|
|
\def\PYZca{\char`\^}
|
|
\def\PYZam{\char`\&}
|
|
\def\PYZlt{\char`\<}
|
|
\def\PYZgt{\char`\>}
|
|
\def\PYZsh{\char`\#}
|
|
\def\PYZpc{\char`\%}
|
|
\def\PYZdl{\char`\$}
|
|
\def\PYZhy{\char`\-}
|
|
\def\PYZsq{\char`\'}
|
|
\def\PYZdq{\char`\"}
|
|
\def\PYZti{\char`\~}
|
|
% for compatibility with earlier versions
|
|
\def\PYZat{@}
|
|
\def\PYZlb{[}
|
|
\def\PYZrb{]}
|
|
\makeatother
|
|
|
|
|
|
% Exact colors from NB
|
|
\definecolor{incolor}{rgb}{0.0, 0.0, 0.5}
|
|
\definecolor{outcolor}{rgb}{0.545, 0.0, 0.0}
|
|
|
|
|
|
|
|
|
|
% Prevent overflowing lines due to hard-to-break entities
|
|
\sloppy
|
|
% Setup hyperref package
|
|
\hypersetup{
|
|
breaklinks=true, % so long urls are correctly broken across lines
|
|
colorlinks=true,
|
|
urlcolor=urlcolor,
|
|
linkcolor=linkcolor,
|
|
citecolor=citecolor,
|
|
}
|
|
% Slightly bigger margins than the latex defaults
|
|
|
|
\geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in}
|
|
|
|
|
|
|
|
\begin{document}
|
|
|
|
|
|
\maketitle
|
|
|
|
|
|
|
|
|
|
\hypertarget{data-analysis-and-machine-learning-neural-networks-from-the-simple-perceptron-to-deep-learning}{%
|
|
\section{Data Analysis and Machine Learning: Neural networks, from the
|
|
simple perceptron to deep
|
|
learning}\label{data-analysis-and-machine-learning-neural-networks-from-the-simple-perceptron-to-deep-learning}}
|
|
|
|
\textbf{Morten Hjorth-Jensen}, Department of Physics, University of Oslo
|
|
and Department of Physics and Astronomy and National Superconducting
|
|
Cyclotron Laboratory, Michigan State University
|
|
|
|
Date: \textbf{Oct 4, 2018}
|
|
|
|
Copyright 1999-2018, Morten Hjorth-Jensen. Released under CC
|
|
Attribution-NonCommercial 4.0 license
|
|
|
|
\hypertarget{neural-networks}{%
|
|
\subsection{Neural networks}\label{neural-networks}}
|
|
|
|
Artificial neural networks are computational systems that can learn to
|
|
perform tasks by considering examples, generally without being
|
|
programmed with any task-specific rules. It is supposed to mimic a
|
|
biological system, wherein neurons interact by sending signals in the
|
|
form of mathematical functions between layers. All layers can contain an
|
|
arbitrary number of neurons, and each connection is represented by a
|
|
weight variable.
|
|
|
|
\hypertarget{artificial-neurons}{%
|
|
\subsection{Artificial neurons}\label{artificial-neurons}}
|
|
|
|
The field of artificial neural networks has a long history of
|
|
development, and is closely connected with the advancement of computer
|
|
science and computers in general. A model of artificial neurons was
|
|
first developed by McCulloch and Pitts in 1943 to study signal
|
|
processing in the brain and has later been refined by others. The
|
|
general idea is to mimic neural networks in the human brain, which is
|
|
composed of billions of neurons that communicate with each other by
|
|
sending electrical signals. Each neuron accumulates its incoming
|
|
signals, which must exceed an activation threshold to yield an output.
|
|
If the threshold is not overcome, the neuron remains inactive, i.e.~has
|
|
zero output.
|
|
|
|
This behaviour has inspired a simple mathematical model for an
|
|
artificial neuron.
|
|
|
|
\hypertarget{artificialNeuron}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
y = f\left(\sum_{i=1}^n w_ix_i\right) = f(u)
|
|
\label{artificialNeuron} \tag{1}
|
|
\end{equation}
|
|
\]
|
|
|
|
Here, the output \(y\) of the neuron is the value of its activation
|
|
function, which have as input a weighted sum of signals
|
|
\(x_i, \dots ,x_n\) received by \(n\) other neurons.
|
|
|
|
Conceptually, it is helpful to divide neural networks into four
|
|
categories: 1. general purpose neural networks for supervised learning,
|
|
|
|
\begin{enumerate}
|
|
\def\labelenumi{\arabic{enumi}.}
|
|
\setcounter{enumi}{1}
|
|
\item
|
|
neural networks designed specifically for image processing, the most
|
|
prominent example of this class being Convolutional Neural Networks
|
|
(CNNs),
|
|
\item
|
|
neural networks for sequential data such as Recurrent Neural Networks
|
|
(RNNs), and
|
|
\item
|
|
neural networks for unsupervised learning such as Deep Boltzmann
|
|
Machines.
|
|
\end{enumerate}
|
|
|
|
In natural science, DNNs and CNNs have already found numerous
|
|
applications. In statistical physics, they have been applied to detect
|
|
phase transitions in 2D Ising and Potts models, lattice gauge theories,
|
|
and different phases of polymers, or solving the Navier-Stokes equation
|
|
in weather forecasting. Deep learning has also found interesting
|
|
applications in quantum physics. Various quantum phase transitions can
|
|
be detected and studied using DNNs and CNNs, topological phases, and
|
|
even non-equilibrium many-body localization. Representing quantum states
|
|
as DNNs quantum state tomography are among some of the impressive
|
|
achievements to reveal the potential of DNNs to facilitate the study of
|
|
quantum systems.
|
|
|
|
In quantum information theory, it has been shown that one can perform
|
|
gate decompositions with the help of neural.
|
|
|
|
The applications are not limited to the natural sciences. There is a
|
|
plethora of applications in essentially all disciplines, from the
|
|
humanities to life science and medicine.
|
|
|
|
\hypertarget{neural-network-types}{%
|
|
\subsection{Neural network types}\label{neural-network-types}}
|
|
|
|
An artificial neural network (ANN), is a computational model that
|
|
consists of layers of connected neurons, or nodes or units. We will
|
|
refer to these interchangeably as units or nodes, and sometimes as
|
|
neurons.
|
|
|
|
It is supposed to mimic a biological nervous system by letting each
|
|
neuron interact with other neurons by sending signals in the form of
|
|
mathematical functions between layers. A wide variety of different ANNs
|
|
have been developed, but most of them consist of an input layer, an
|
|
output layer and eventual layers in-between, called \emph{hidden
|
|
layers}. All layers can contain an arbitrary number of nodes, and each
|
|
connection between two nodes is associated with a weight variable.
|
|
|
|
Neural networks (also called neural nets) are neural-inspired nonlinear
|
|
models for supervised learning. As we will see, neural nets can be
|
|
viewed as natural, more powerful extensions of supervised learning
|
|
methods such as linear and logistic regression and soft-max methods we
|
|
discussed earlier.
|
|
|
|
\hypertarget{feed-forward-neural-networks}{%
|
|
\subsection{Feed-forward neural
|
|
networks}\label{feed-forward-neural-networks}}
|
|
|
|
The feed-forward neural network (FFNN) was the first and simplest type
|
|
of ANNs that were devised. In this network, the information moves in
|
|
only one direction: forward through the layers.
|
|
|
|
Nodes are represented by circles, while the arrows display the
|
|
connections between the nodes, including the direction of information
|
|
flow. Additionally, each arrow corresponds to a weight variable (figure
|
|
to come). We observe that each node in a layer is connected to
|
|
\emph{all} nodes in the subsequent layer, making this a so-called
|
|
\emph{fully-connected} FFNN.
|
|
|
|
\hypertarget{convolutional-neural-network}{%
|
|
\subsection{Convolutional Neural
|
|
Network}\label{convolutional-neural-network}}
|
|
|
|
A different variant of FFNNs are \emph{convolutional neural networks}
|
|
(CNNs), which have a connectivity pattern inspired by the animal visual
|
|
cortex. Individual neurons in the visual cortex only respond to stimuli
|
|
from small sub-regions of the visual field, called a receptive field.
|
|
This makes the neurons well-suited to exploit the strong spatially local
|
|
correlation present in natural images. The response of each neuron can
|
|
be approximated mathematically as a convolution operation. (figure to
|
|
come)
|
|
|
|
Convolutional neural networks emulate the behaviour of neurons in the
|
|
visual cortex by enforcing a \emph{local} connectivity pattern between
|
|
nodes of adjacent layers: Each node in a convolutional layer is
|
|
connected only to a subset of the nodes in the previous layer, in
|
|
contrast to the fully-connected FFNN. Often, CNNs consist of several
|
|
convolutional layers that learn local features of the input, with a
|
|
fully-connected layer at the end, which gathers all the local data and
|
|
produces the outputs. They have wide applications in image and video
|
|
recognition.
|
|
|
|
\hypertarget{recurrent-neural-networks}{%
|
|
\subsection{Recurrent neural networks}\label{recurrent-neural-networks}}
|
|
|
|
So far we have only mentioned ANNs where information flows in one
|
|
direction: forward. \emph{Recurrent neural networks} on the other hand,
|
|
have connections between nodes that form directed \emph{cycles}. This
|
|
creates a form of internal memory which are able to capture information
|
|
on what has been calculated before; the output is dependent on the
|
|
previous computations. Recurrent NNs make use of sequential information
|
|
by performing the same task for every element in a sequence, where each
|
|
element depends on previous elements. An example of such information is
|
|
sentences, making recurrent NNs especially well-suited for handwriting
|
|
and speech recognition.
|
|
|
|
\hypertarget{other-types-of-networks}{%
|
|
\subsection{Other types of networks}\label{other-types-of-networks}}
|
|
|
|
There are many other kinds of ANNs that have been developed. One type
|
|
that is specifically designed for interpolation in multidimensional
|
|
space is the radial basis function (RBF) network. RBFs are typically
|
|
made up of three layers: an input layer, a hidden layer with non-linear
|
|
radial symmetric activation functions and a linear output layer
|
|
(`'linear'' here means that each node in the output layer has a linear
|
|
activation function). The layers are normally fully-connected and there
|
|
are no cycles, thus RBFs can be viewed as a type of fully-connected
|
|
FFNN. They are however usually treated as a separate type of NN due the
|
|
unusual activation functions.
|
|
|
|
\hypertarget{multilayer-perceptrons}{%
|
|
\subsection{Multilayer perceptrons}\label{multilayer-perceptrons}}
|
|
|
|
One uses often so-called fully-connected feed-forward neural networks
|
|
with three or more layers (an input layer, one or more hidden layers and
|
|
an output layer) consisting of neurons that have non-linear activation
|
|
functions.
|
|
|
|
Such networks are often called \emph{multilayer perceptrons} (MLPs).
|
|
|
|
\hypertarget{why-multilayer-perceptrons}{%
|
|
\subsection{Why multilayer
|
|
perceptrons?}\label{why-multilayer-perceptrons}}
|
|
|
|
According to the \emph{Universal approximation theorem}, a feed-forward
|
|
neural network with just a single hidden layer containing a finite
|
|
number of neurons can approximate a continuous multidimensional function
|
|
to arbitrary accuracy, assuming the activation function for the hidden
|
|
layer is a \textbf{non-constant, bounded and monotonically-increasing
|
|
continuous function}.
|
|
|
|
Note that the requirements on the activation function only applies to
|
|
the hidden layer, the output nodes are always assumed to be linear, so
|
|
as to not restrict the range of output values.
|
|
|
|
\hypertarget{mathematical-model}{%
|
|
\subsection{Mathematical model}\label{mathematical-model}}
|
|
|
|
The output \(y\) is produced via the activation function \(f\)
|
|
|
|
\[
|
|
y = f\left(\sum_{i=1}^n w_ix_i + b_i\right) = f(z),
|
|
\]
|
|
|
|
This function receives \(x_i\) as inputs. Here the activation
|
|
\(z=(\sum_{i=1}^n w_ix_i+b_i)\). In an FFNN of such neurons, the
|
|
\emph{inputs} \(x_i\) are the \emph{outputs} of the neurons in the
|
|
preceding layer. Furthermore, an MLP is fully-connected, which means
|
|
that each neuron receives a weighted sum of the outputs of \emph{all}
|
|
neurons in the previous layer.
|
|
|
|
\hypertarget{mathematical-model}{%
|
|
\subsection{Mathematical model}\label{mathematical-model}}
|
|
|
|
First, for each node \(i\) in the first hidden layer, we calculate a
|
|
weighted sum \(z_i^1\) of the input coordinates \(x_j\),
|
|
|
|
\hypertarget{_auto1}{}
|
|
|
|
\[
|
|
\begin{equation} z_i^1 = \sum_{j=1}^{M} w_{ij}^1 x_j + b_i^1
|
|
\label{_auto1} \tag{2}
|
|
\end{equation}
|
|
\]
|
|
|
|
Here \(b_i\) is the so-called bias which is normally needed in case of
|
|
zero activation weights or inputs. How to fix the biases and the weights
|
|
will be discussed below. The value of \(z_i^1\) is the argument to the
|
|
activation function \(f_i\) of each node \(i\), The variable \(M\)
|
|
stands for all possible inputs to a given node \(i\) in the first layer.
|
|
We define the output \(y_i^1\) of all neurons in layer 1 as
|
|
|
|
\hypertarget{outputLayer1}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
y_i^1 = f(z_i^1) = f\left(\sum_{j=1}^M w_{ij}^1 x_j + b_i^1\right)
|
|
\label{outputLayer1} \tag{3}
|
|
\end{equation}
|
|
\]
|
|
|
|
where we assume that all nodes in the same layer have identical
|
|
activation functions, hence the notation \(f\). In general, we could
|
|
assume in the more general case that different layers have different
|
|
activation functions. In this case we would identify these functions
|
|
with a superscript \(l\) for the \(l\)-th layer,
|
|
|
|
\hypertarget{generalLayer}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
y_i^l = f^l(u_i^l) = f^l\left(\sum_{j=1}^{N_{l-1}} w_{ij}^l y_j^{l-1} + b_i^l\right)
|
|
\label{generalLayer} \tag{4}
|
|
\end{equation}
|
|
\]
|
|
|
|
where \(N_l\) is the number of nodes in layer \(l\). When the output of
|
|
all the nodes in the first hidden layer are computed, the values of the
|
|
subsequent layer can be calculated and so forth until the output is
|
|
obtained.
|
|
|
|
\hypertarget{mathematical-model}{%
|
|
\subsection{Mathematical model}\label{mathematical-model}}
|
|
|
|
The output of neuron \(i\) in layer 2 is thus,
|
|
|
|
\hypertarget{_auto2}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
y_i^2 = f^2\left(\sum_{j=1}^N w_{ij}^2 y_j^1 + b_i^2\right)
|
|
\label{_auto2} \tag{5}
|
|
\end{equation}
|
|
\]
|
|
|
|
\hypertarget{outputLayer2}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
= f^2\left[\sum_{j=1}^N w_{ij}^2f^1\left(\sum_{k=1}^M w_{jk}^1 x_k + b_j^1\right) + b_i^2\right]
|
|
\label{outputLayer2} \tag{6}
|
|
\end{equation}
|
|
\]
|
|
|
|
where we have substituted \(y_k^1\) with the inputs \(x_k\). Finally,
|
|
the ANN output reads
|
|
|
|
\hypertarget{_auto3}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
y_i^3 = f^3\left(\sum_{j=1}^N w_{ij}^3 y_j^2 + b_i^3\right)
|
|
\label{_auto3} \tag{7}
|
|
\end{equation}
|
|
\]
|
|
|
|
\hypertarget{_auto4}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
= f_3\left[\sum_{j} w_{ij}^3 f^2\left(\sum_{k} w_{jk}^2 f^1\left(\sum_{m} w_{km}^1 x_m + b_k^1\right) + b_j^2\right)
|
|
+ b_1^3\right]
|
|
\label{_auto4} \tag{8}
|
|
\end{equation}
|
|
\]
|
|
|
|
\hypertarget{mathematical-model}{%
|
|
\subsection{Mathematical model}\label{mathematical-model}}
|
|
|
|
We can generalize this expression to an MLP with \(l\) hidden layers.
|
|
The complete functional form is,
|
|
|
|
\hypertarget{completeNN}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
y^{l+1}_i = f^{l+1}\left[\!\sum_{j=1}^{N_l} w_{ij}^3 f^l\left(\sum_{k=1}^{N_{l-1}}w_{jk}^{l-1}\left(\dots f^1\left(\sum_{n=1}^{N_0} w_{mn}^1 x_n+ b_m^1\right)\dots\right)+b_k^2\right)+b_1^3\right]
|
|
\label{completeNN} \tag{9}
|
|
\end{equation}
|
|
\]
|
|
|
|
which illustrates a basic property of MLPs: The only independent
|
|
variables are the input values \(x_n\).
|
|
|
|
\hypertarget{mathematical-model}{%
|
|
\subsection{Mathematical model}\label{mathematical-model}}
|
|
|
|
This confirms that an MLP, despite its quite convoluted mathematical
|
|
form, is nothing more than an analytic function, specifically a mapping
|
|
of real-valued vectors
|
|
\(\hat{x} \in \mathbb{R}^n \rightarrow \hat{y} \in \mathbb{R}^m\).
|
|
|
|
Furthermore, the flexibility and universality of an MLP can be
|
|
illustrated by realizing that the expression is essentially a nested sum
|
|
of scaled activation functions of the form
|
|
|
|
\hypertarget{_auto5}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
f(x) = c_1 f(c_2 x + c_3) + c_4
|
|
\label{_auto5} \tag{10}
|
|
\end{equation}
|
|
\]
|
|
|
|
where the parameters \(c_i\) are weights and biases. By adjusting these
|
|
parameters, the activation functions can be shifted up and down or left
|
|
and right, change slope or be rescaled which is the key to the
|
|
flexibility of a neural network.
|
|
|
|
\hypertarget{matrix-vector-notation}{%
|
|
\subsubsection{Matrix-vector notation}\label{matrix-vector-notation}}
|
|
|
|
We can introduce a more convenient notation for the activations in an A
|
|
NN.
|
|
|
|
Additionally, we can represent the biases and activations as layer-wise
|
|
column vectors \(\hat{b}_l\) and \(\hat{y}_l\), so that the \(i\)-th
|
|
element of each vector is the bias \(b_i^l\) and activation \(y_i^l\) of
|
|
node \(i\) in layer \(l\) respectively.
|
|
|
|
We have that \(\mathrm{W}_l\) is an \(N_{l-1} \times N_l\) matrix, while
|
|
\(\hat{b}_l\) and \(\hat{y}_l\) are \(N_l \times 1\) column vectors.
|
|
With this notation, the sum becomes a matrix-vector multiplication, and
|
|
we can write the equation for the activations of hidden layer 2
|
|
(assuming three nodes for simplicity) as
|
|
|
|
\hypertarget{_auto6}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
\hat{y}_2 = f_2(\mathrm{W}_2 \hat{y}_{1} + \hat{b}_{2}) =
|
|
f_2\left(\left[\begin{array}{ccc}
|
|
w^2_{11} &w^2_{12} &w^2_{13} \\
|
|
w^2_{21} &w^2_{22} &w^2_{23} \\
|
|
w^2_{31} &w^2_{32} &w^2_{33} \\
|
|
\end{array} \right] \cdot
|
|
\left[\begin{array}{c}
|
|
y^1_1 \\
|
|
y^1_2 \\
|
|
y^1_3 \\
|
|
\end{array}\right] +
|
|
\left[\begin{array}{c}
|
|
b^2_1 \\
|
|
b^2_2 \\
|
|
b^2_3 \\
|
|
\end{array}\right]\right).
|
|
\label{_auto6} \tag{11}
|
|
\end{equation}
|
|
\]
|
|
|
|
\hypertarget{matrix-vector-notation-and-activation}{%
|
|
\subsubsection{Matrix-vector notation and
|
|
activation}\label{matrix-vector-notation-and-activation}}
|
|
|
|
The activation of node \(i\) in layer 2 is
|
|
|
|
\hypertarget{_auto7}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
y^2_i = f_2\Bigr(w^2_{i1}y^1_1 + w^2_{i2}y^1_2 + w^2_{i3}y^1_3 + b^2_i\Bigr) =
|
|
f_2\left(\sum_{j=1}^3 w^2_{ij} y_j^1 + b^2_i\right).
|
|
\label{_auto7} \tag{12}
|
|
\end{equation}
|
|
\]
|
|
|
|
This is not just a convenient and compact notation, but also a useful
|
|
and intuitive way to think about MLPs: The output is calculated by a
|
|
series of matrix-vector multiplications and vector additions that are
|
|
used as input to the activation functions. For each operation
|
|
\(\mathrm{W}_l \hat{y}_{l-1}\) we move forward one layer.
|
|
|
|
\hypertarget{activation-functions}{%
|
|
\subsubsection{Activation functions}\label{activation-functions}}
|
|
|
|
A property that characterizes a neural network, other than its
|
|
connectivity, is the choice of activation function(s). As described in,
|
|
the following restrictions are imposed on an activation function for a
|
|
FFNN to fulfill the universal approximation theorem
|
|
|
|
\begin{itemize}
|
|
\item
|
|
Non-constant
|
|
\item
|
|
Bounded
|
|
\item
|
|
Monotonically-increasing
|
|
\item
|
|
Continuous
|
|
\end{itemize}
|
|
|
|
\hypertarget{activation-functions-logistic-and-hyperbolic-ones}{%
|
|
\subsubsection{Activation functions, Logistic and Hyperbolic
|
|
ones}\label{activation-functions-logistic-and-hyperbolic-ones}}
|
|
|
|
The second requirement excludes all linear functions. Furthermore, in a
|
|
MLP with only linear activation functions, each layer simply performs a
|
|
linear transformation of its inputs.
|
|
|
|
Regardless of the number of layers, the output of the NN will be nothing
|
|
but a linear function of the inputs. Thus we need to introduce some kind
|
|
of non-linearity to the NN to be able to fit non-linear functions
|
|
Typical examples are the logistic \emph{Sigmoid}
|
|
|
|
\[
|
|
f(x) = \frac{1}{1 + e^{-x}},
|
|
\]
|
|
|
|
and the \emph{hyperbolic tangent} function
|
|
|
|
\[
|
|
f(x) = \tanh(x)
|
|
\]
|
|
|
|
\hypertarget{relevance}{%
|
|
\subsubsection{Relevance}\label{relevance}}
|
|
|
|
The \emph{sigmoid} function are more biologically plausible because the
|
|
output of inactive neurons are zero. Such activation function are called
|
|
\emph{one-sided}. However, it has been shown that the hyperbolic tangent
|
|
performs better than the sigmoid for training MLPs. has become the most
|
|
popular for \emph{deep neural networks}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}1}]:} \PY{o}{\PYZpc{}}\PY{k}{matplotlib} inline
|
|
|
|
\PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}The sigmoid function (or the logistic curve) is a }
|
|
\PY{l+s+sd}{function that takes any real number, z, and outputs a number (0,1).}
|
|
\PY{l+s+sd}{It is useful in neural networks for assigning weights on a relative scale.}
|
|
\PY{l+s+sd}{The value z is the weighted sum of parameters involved in the learning algorithm.\PYZdq{}\PYZdq{}\PYZdq{}}
|
|
|
|
\PY{k+kn}{import} \PY{n+nn}{numpy}
|
|
\PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt}
|
|
\PY{k+kn}{import} \PY{n+nn}{math} \PY{k}{as} \PY{n+nn}{mt}
|
|
|
|
\PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{o}{.}\PY{l+m+mi}{1}\PY{p}{)}
|
|
\PY{n}{sigma\PYZus{}fn} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{vectorize}\PY{p}{(}\PY{k}{lambda} \PY{n}{z}\PY{p}{:} \PY{l+m+mi}{1}\PY{o}{/}\PY{p}{(}\PY{l+m+mi}{1}\PY{o}{+}\PY{n}{numpy}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{n}{z}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sigma} \PY{o}{=} \PY{n}{sigma\PYZus{}fn}\PY{p}{(}\PY{n}{z}\PY{p}{)}
|
|
|
|
\PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
|
|
\PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{sigma}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{0.1}\PY{p}{,} \PY{l+m+mf}{1.1}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,}\PY{l+m+mi}{5}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sigmoid function}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}Step Function\PYZdq{}\PYZdq{}\PYZdq{}}
|
|
\PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{o}{.}\PY{l+m+mi}{02}\PY{p}{)}
|
|
\PY{n}{step\PYZus{}fn} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{vectorize}\PY{p}{(}\PY{k}{lambda} \PY{n}{z}\PY{p}{:} \PY{l+m+mf}{1.0} \PY{k}{if} \PY{n}{z} \PY{o}{\PYZgt{}}\PY{o}{=} \PY{l+m+mf}{0.0} \PY{k}{else} \PY{l+m+mf}{0.0}\PY{p}{)}
|
|
\PY{n}{step} \PY{o}{=} \PY{n}{step\PYZus{}fn}\PY{p}{(}\PY{n}{z}\PY{p}{)}
|
|
|
|
\PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
|
|
\PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{step}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{0.5}\PY{p}{,} \PY{l+m+mf}{1.5}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,}\PY{l+m+mi}{5}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{step function}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}Sine Function\PYZdq{}\PYZdq{}\PYZdq{}}
|
|
\PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{,} \PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{,} \PY{l+m+mf}{0.1}\PY{p}{)}
|
|
\PY{n}{t} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{sin}\PY{p}{(}\PY{n}{z}\PY{p}{)}
|
|
|
|
\PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
|
|
\PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{t}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{1.0}\PY{p}{,} \PY{l+m+mf}{1.0}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{,}\PY{l+m+mi}{2}\PY{o}{*}\PY{n}{mt}\PY{o}{.}\PY{n}{pi}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sine function}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{l+s+sd}{\PYZdq{}\PYZdq{}\PYZdq{}Plots a graph of the squashing function used by a rectified linear}
|
|
\PY{l+s+sd}{unit\PYZdq{}\PYZdq{}\PYZdq{}}
|
|
\PY{n}{z} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{2}\PY{p}{,} \PY{l+m+mi}{2}\PY{p}{,} \PY{o}{.}\PY{l+m+mi}{1}\PY{p}{)}
|
|
\PY{n}{zero} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{z}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{y} \PY{o}{=} \PY{n}{numpy}\PY{o}{.}\PY{n}{max}\PY{p}{(}\PY{p}{[}\PY{n}{zero}\PY{p}{,} \PY{n}{z}\PY{p}{]}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
|
|
|
|
\PY{n}{fig} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{figure}\PY{p}{(}\PY{p}{)}
|
|
\PY{n}{ax} \PY{o}{=} \PY{n}{fig}\PY{o}{.}\PY{n}{add\PYZus{}subplot}\PY{p}{(}\PY{l+m+mi}{111}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{plot}\PY{p}{(}\PY{n}{z}\PY{p}{,} \PY{n}{y}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{2.0}\PY{p}{,} \PY{l+m+mf}{2.0}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlim}\PY{p}{(}\PY{p}{[}\PY{o}{\PYZhy{}}\PY{l+m+mf}{2.0}\PY{p}{,} \PY{l+m+mf}{2.0}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{grid}\PY{p}{(}\PY{k+kc}{True}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{z}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{Rectified linear unit}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\hypertarget{the-multilayer-perceptron-mlp}{%
|
|
\subsection{The multilayer perceptron
|
|
(MLP)}\label{the-multilayer-perceptron-mlp}}
|
|
|
|
The multilayer perceptron is a very popular, and easy to implement
|
|
approach, to deep learning. It consists of 1. A neural network with one
|
|
or more layers of nodes between the input and the output nodes.
|
|
|
|
\begin{enumerate}
|
|
\def\labelenumi{\arabic{enumi}.}
|
|
\setcounter{enumi}{1}
|
|
\item
|
|
The multilayer network structure, or architecture, or topology,
|
|
consists of an input layer, one or more hidden layers, and one output
|
|
layer.
|
|
\item
|
|
The input nodes pass values to the first hidden layer, its nodes pass
|
|
the information on to the second and so on till we reach the output
|
|
layer.
|
|
\end{enumerate}
|
|
|
|
As a convention it is normal to call a network with one layer of input
|
|
units, one layer of hidden units and one layer of output units as a
|
|
two-layer network. A network with two layers of hidden units is called a
|
|
three-layer network etc etc.
|
|
|
|
For an MLP network there is no direct connection between the output
|
|
nodes/neurons/units and the input nodes/neurons/units. Hereafter we will
|
|
call the various entities of a layer for nodes. There are also no
|
|
connections within a single layer.
|
|
|
|
The number of input nodes does not need to equal the number of output
|
|
nodes. This applies also to the hidden layers. Each layer may have its
|
|
own number of nodes and activation functions.
|
|
|
|
The hidden layers have their name from the fact that they are not linked
|
|
to observables and as we will see below when we define the so-called
|
|
activation \(\hat{z}\), we can think of this as a basis expansion of the
|
|
original inputs \(\hat{x}\). The difference however between neural
|
|
networks and say linear regression is that now these basis functions
|
|
(which will correspond to the weights in the network) are learned from
|
|
data. This results in an important difference between neural networks
|
|
and deep learning approaches on one side and methods like logistic
|
|
regression or linear regression and their modifications on the other
|
|
side.
|
|
|
|
\hypertarget{from-one-to-many-layers-the-universal-approximation-theorem}{%
|
|
\subsection{From one to many layers, the universal approximation
|
|
theorem}\label{from-one-to-many-layers-the-universal-approximation-theorem}}
|
|
|
|
A neural network with only one layer, what we called the simple
|
|
perceptron, is best suited if we have a standard binary model with clear
|
|
(linear) boundaries between the outcomes. As such it could equally well
|
|
be replaced by standard linear regression or logistic regression.
|
|
Networks with one or more hidden layers approximate systems with more
|
|
complex boundaries.
|
|
|
|
As stated earlier, an important theorem in studies of neural networks,
|
|
restated without proof here, is the
|
|
\href{http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.441.7873\&rep=rep1\&type=pdf}{universal
|
|
approximation theorem}.
|
|
|
|
It states that a feed-forward network with a single hidden layer
|
|
containing a finite number of neurons can approximate continuous
|
|
functions on compact subsets of real functions. The theorem thus states
|
|
that simple neural networks can represent a wide variety of interesting
|
|
functions when given appropriate parameters. It is the multilayer
|
|
feedforward architecture itself which gives neural networks the
|
|
potential of being universal approximators.
|
|
|
|
\hypertarget{deriving-the-back-propagation-code-for-a-multilayer-perceptron-model}{%
|
|
\subsection{Deriving the back propagation code for a multilayer
|
|
perceptron
|
|
model}\label{deriving-the-back-propagation-code-for-a-multilayer-perceptron-model}}
|
|
|
|
\textbf{Note: figures will be inserted later!}
|
|
|
|
As we have seen now in a feed forward network, we can express the final
|
|
output of our network in terms of basic matrix-vector multiplications.
|
|
The unknowwn quantities are our weights \(w_{ij}\) and we need to find
|
|
an algorithm for changing them so that our errors are as small as
|
|
possible. This leads us to the famous
|
|
\href{https://www.nature.com/articles/323533a0}{back propagation
|
|
algorithm}.
|
|
|
|
The questions we want to ask are how do changes in the biases and the
|
|
weights in our network change the cost function and how can we use the
|
|
final output to modify the weights?
|
|
|
|
To derive these equations let us start with a plain regression problem
|
|
and define our cost function as
|
|
|
|
\[
|
|
{\cal C}(\hat{W}) = \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2,
|
|
\]
|
|
|
|
where the \(t_i\)s are our \(n\) targets (the values we want to
|
|
reproduce), while the outputs of the network after having propagated all
|
|
inputs \(\hat{x}\) are given by \(y_i\). Below we will demonstrate how
|
|
the basic equations arising from the back propagation algorithm can be
|
|
modified in order to study classification problems with \(K\) classes.
|
|
|
|
\hypertarget{definitions}{%
|
|
\subsection{Definitions}\label{definitions}}
|
|
|
|
With our definition of the targets \(\hat{t}\), the outputs of the
|
|
network \(\hat{y}\) and the inputs \(\hat{x}\) we define now the
|
|
activation \(z_j^l\) of node/neuron/unit \(j\) of the \(l\)-th layer as
|
|
a function of the bias, the weights which add up from the previous layer
|
|
\(l-1\) and the forward passes/outputs \(\hat{a}^{l-1}\) from the
|
|
previous layer as
|
|
|
|
\[
|
|
z_j^l = \sum_{i=1}^{M_{l-1}}w_{ij}^la_j^{l-1}+b_j^l,
|
|
\]
|
|
|
|
where \(b_k^l\) are the biases from layer \(l\). Here \(M_{l-1}\)
|
|
represents the total number of nodes/neurons/units of layer \(l-1\). The
|
|
figure here illustrates this equation. We can rewrite this in a more
|
|
compact form as the matrix-vector products we discussed earlier,
|
|
|
|
\[
|
|
\hat{z}^l = \left(\hat{W}^l\right)^T\hat{a}^{l-1}+\hat{b}^l.
|
|
\]
|
|
|
|
With the activation values \(\hat{z}^l\) we can in turn define the
|
|
output of layer \(l\) as \(\hat{a}^l = f(\hat{z}^l)\) where \(f\) is our
|
|
activation function. In the examples here we will use the sigmoid
|
|
function discussed in our logistic regression lectures. We will also use
|
|
the same activation function \(f\) for all layers and their nodes. It
|
|
means we have
|
|
|
|
\[
|
|
a_j^l = f(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}.
|
|
\]
|
|
|
|
\hypertarget{derivatives-and-the-chain-rule}{%
|
|
\subsection{Derivatives and the chain
|
|
rule}\label{derivatives-and-the-chain-rule}}
|
|
|
|
From the definition of the activation \(z_j^l\) we have
|
|
|
|
\[
|
|
\frac{\partial z_j^l}{\partial w_{ji}^l} = a_i^{l-1},
|
|
\]
|
|
|
|
and
|
|
|
|
\[
|
|
\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l.
|
|
\]
|
|
|
|
With our definition of the activation function we have that (note that
|
|
this function depends only on \(z_j^l\))
|
|
|
|
\[
|
|
\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)).
|
|
\]
|
|
|
|
\hypertarget{derivative-of-the-cost-function}{%
|
|
\subsection{Derivative of the cost
|
|
function}\label{derivative-of-the-cost-function}}
|
|
|
|
With these definitions we can now compute the derivative of the cost
|
|
function in terms of the weights.
|
|
|
|
Let us specialize to the output layer \(l=L\). Our cost function is
|
|
|
|
\[
|
|
{\cal C}(\hat{W^L}) = \frac{1}{2}\sum_{i=1}^n\left(y_i - t_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - t_i\right)^2,
|
|
\]
|
|
|
|
The derivative of this function with respect to the weights is
|
|
|
|
\[
|
|
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \left(a_j^L - t_j\right)\frac{\partial a_j^L}{\partial w_{jk}^{L}},
|
|
\]
|
|
|
|
The last partial derivative can easily be computed and reads (by
|
|
applying the chain rule)
|
|
|
|
\[
|
|
\frac{\partial a_j^L}{\partial w_{jk}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},
|
|
\]
|
|
|
|
\hypertarget{bringing-it-together-first-back-propagation-equation}{%
|
|
\subsection{Bringing it together, first back propagation
|
|
equation}\label{bringing-it-together-first-back-propagation-equation}}
|
|
|
|
We have thus
|
|
|
|
\[
|
|
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \left(a_j^L - t_j\right)a_j^L(1-a_j^L)a_k^{L-1},
|
|
\]
|
|
|
|
Defining
|
|
|
|
\[
|
|
\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - t_j\right) = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
|
|
\]
|
|
|
|
and using the Hadamard product of two vectors we can write this as
|
|
|
|
\[
|
|
\hat{\delta}^L = f'(\hat{z}^L)\circ\frac{\partial {\cal C}}{\partial (\hat{a}L)}.
|
|
\]
|
|
|
|
This is an important expression. The second term on the right handside
|
|
measures how fast the cost function is changing as a function of the
|
|
\(j\)th output activation. If, for example, the cost function doesn't
|
|
depend much on a particular output node \(j\), then \(\delta_j^L\) will
|
|
be small, which is what we would expect. The first term on the right,
|
|
measures how fast the activation function \(f\) is changing at a given
|
|
activation value \(z_j^L\).
|
|
|
|
Notice that everything in the above equations is easily computed. In
|
|
particular, we compute \(z_j^L\) while computing the behaviour of the
|
|
network, and it is only a small additional overhead to compute
|
|
\(f'(z^L_j)\). The exact form of the derivative with respect to the
|
|
output depends on the form of the cost function. However, provided the
|
|
cost function is known there should be little trouble in calculating
|
|
|
|
\[
|
|
\frac{\partial {\cal C}}{\partial (a_j^L)}
|
|
\]
|
|
|
|
With the definition of \(\delta_j^L\) we have a more compact definition
|
|
of the derivative of the cost function in terms of the weights, namely
|
|
|
|
\[
|
|
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \delta_j^La_k^{L-1}.
|
|
\]
|
|
|
|
\hypertarget{derivatives-in-terms-of-z_jl}{%
|
|
\subsection{\texorpdfstring{Derivatives in terms of
|
|
\(z_j^L\)}{Derivatives in terms of z\_j\^{}L}}\label{derivatives-in-terms-of-z_jl}}
|
|
|
|
It is also easy to see that our previous equation can be written as
|
|
|
|
\[
|
|
\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L},
|
|
\]
|
|
|
|
which can also be interpreted as the partial derivative of the cost
|
|
function with respect to the biases \(b_j^L\), namely
|
|
|
|
\[
|
|
\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L},
|
|
\]
|
|
|
|
That is, the error \(\delta_j^L\) is exactly equal to the rate of change
|
|
of the cost function as a function of the bias. \#\# Bringing it
|
|
together
|
|
|
|
We have now three equations that are essential for the computations of
|
|
the derivatives of the cost function at the output layer. These
|
|
equations are needed to start the algorithm and they are
|
|
|
|
\textbf{The starting equations.}
|
|
|
|
\hypertarget{_auto8}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
\frac{\partial{\cal C}(\hat{W^L})}{\partial w_{jk}^L} = \delta_j^La_k^{L-1},
|
|
\label{_auto8} \tag{13}
|
|
\end{equation}
|
|
\]
|
|
|
|
and
|
|
|
|
\hypertarget{_auto9}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)},
|
|
\label{_auto9} \tag{14}
|
|
\end{equation}
|
|
\]
|
|
|
|
and
|
|
|
|
\hypertarget{_auto10}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L},
|
|
\label{_auto10} \tag{15}
|
|
\end{equation}
|
|
\]
|
|
|
|
An interesting consequence of the above equations is that when the
|
|
activation \(a_k^{L-1}\) is small, the gradient term, that is the
|
|
derivative of the cost function with respect to the weights, will also
|
|
tend to be small. We say then that the weight learns slowly, meaning
|
|
that it changes slowly when we minimize the weights via say gradient
|
|
descent. In this case we say the system learns slowly.
|
|
|
|
Another interesting feature is that is when the activation function,
|
|
represented by the sigmoid function here, is rather flat when we move
|
|
towards its end values \(0\) and \(1\) (see the above Python codes). In
|
|
these cases, the derivatives of the activation function will also be
|
|
close to zero, meaning again that the gradients will be small and the
|
|
network learns slowly again.
|
|
|
|
We need a fourth equation and we are set. We are going to propagate
|
|
backwards in order to the determine the weights and biases. In order to
|
|
do so we need to represent the error in the layer before the final one
|
|
\(L-1\) in terms of the errors in the final output layer.
|
|
|
|
\hypertarget{final-back-propagating-equation}{%
|
|
\subsection{Final back propagating
|
|
equation}\label{final-back-propagating-equation}}
|
|
|
|
We have that (replacing \(L\) with a general layer \(l\))
|
|
|
|
\[
|
|
\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}.
|
|
\]
|
|
|
|
We want to express this in terms of the equations for layer \(l+1\).
|
|
Using the chain rule and summing over all \(k\) entries we have
|
|
|
|
\[
|
|
\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}},
|
|
\]
|
|
|
|
and recalling that
|
|
|
|
\[
|
|
z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_j^{l}+b_j^{l+1},
|
|
\]
|
|
|
|
we obtain
|
|
|
|
\[
|
|
\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),
|
|
\]
|
|
|
|
This is our final equation.
|
|
|
|
We are now ready to set up the algorithm for back propagation and
|
|
learning the weights and biases.
|
|
|
|
\hypertarget{setting-up-the-back-propagation-algorithm}{%
|
|
\subsection{Setting up the Back propagation
|
|
algorithm}\label{setting-up-the-back-propagation-algorithm}}
|
|
|
|
The four equations provide us with a way of computing the gradient of
|
|
the cost function. Let us write this out in the form of an algorithm.
|
|
|
|
First, we set up the input data \(\hat{x}\) and the activations
|
|
\(\hat{z}_1\) of the input layer and compute the activation function and
|
|
the pertinent outputs \(\hat{a}^1\).
|
|
|
|
Secondly, we perform then the feed forward till we reach the output
|
|
layer and compute all \(\hat{z}_l\) of the input layer and compute the
|
|
activation function and the pertinent outputs \(\hat{a}^l\) for
|
|
\(l=2,3,\dots,L\).
|
|
|
|
Thereafter we compute the ouput error \(\hat{\delta}^L\) by computing
|
|
all
|
|
|
|
\[
|
|
\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}.
|
|
\]
|
|
|
|
Then we compute the back propagate error for each \(l=L-1,L-2,\dots,2\)
|
|
as
|
|
|
|
\[
|
|
\delta_j^l =\sum_k \sum_k \delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).
|
|
\]
|
|
|
|
Finally, we update the weights and the biases using gradient descent for
|
|
each \(l=L-1,L-2,dots,2\) and update the weights and biases according to
|
|
the rules
|
|
|
|
\[
|
|
w_{jk}^l\leftarrow = w_{jk}^l- \eta \delta_j^la_k^{l-1},
|
|
\]
|
|
|
|
\[
|
|
b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^L},
|
|
\]
|
|
|
|
The parameter \(\eta\) is the learning parameter discussed in connection
|
|
with the gradient descent methods. Here it is convenient to use
|
|
stochastic gradient descent (see the examples below) with mini-batches
|
|
with an outer loop that steps through multiple epochs of training.
|
|
|
|
\hypertarget{setting-up-a-multi-layer-perceptron-model-for-classification}{%
|
|
\subsection{Setting up a Multi-layer perceptron model for
|
|
classification}\label{setting-up-a-multi-layer-perceptron-model-for-classification}}
|
|
|
|
We are now gong to develop an example based on the MNIST data base. This
|
|
is a classification problem and we need to use our cross-entropy
|
|
function we discussed in connection with logistic regression. The
|
|
cross-entropy defines our cost function for the classificaton problems
|
|
with neural networks.
|
|
|
|
In binary classification with two classes \((0, 1)\) we define the
|
|
logistic/sigmoid function as the probability that a particular input is
|
|
in class \(0\) or \(1\). This is possible because the logistic function
|
|
takes any input from the real numbers and inputs a number between 0 and
|
|
1, and can therefore be interpreted as a probability. It also has other
|
|
nice properties, such as a derivative that is simple to calculate.
|
|
|
|
For an input \(\boldsymbol{a}\) from the hidden layer, the probability
|
|
that the input \(\boldsymbol{x}\) is in class 0 or 1 is just. We let
|
|
\(\theta\) represent the unknown weights and biases to be adjusted by
|
|
our equations). The variable \(x\) represents our activation values
|
|
\(z\). We have
|
|
|
|
\[
|
|
P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp (- \boldsymbol{x}} ,
|
|
\]
|
|
|
|
and
|
|
|
|
\[
|
|
P(y = 1 \mid \boldsymbol{x}, \boldsymbol{\theta}) = 1 - P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) ,
|
|
\]
|
|
|
|
where \(y \in \{0, 1\}\) and \(\boldsymbol{\theta}\) represents the
|
|
weights and biases of our network.
|
|
|
|
\hypertarget{defining-the-cost-function}{%
|
|
\subsection{Defining the cost
|
|
function}\label{defining-the-cost-function}}
|
|
|
|
Our cost function is given as (see the Logistic regression lectures)
|
|
|
|
\[
|
|
\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n
|
|
y_i \ln[P(y_i = 0)] + (1 - y_i) \ln [1 - P(y_i = 0)] = \sum_{i=1}^n \mathcal{L}_i(\boldsymbol{\theta}) .
|
|
\]
|
|
|
|
This last equality means that we can interpret our \emph{cost} function
|
|
as a sum over the \emph{loss} function for each point in the dataset
|
|
\(\mathcal{L}_i(\boldsymbol{\theta})\).\\
|
|
The negative sign is just so that we can think about our algorithm as
|
|
minimizing a positive number, rather than maximizing a negative number.
|
|
|
|
In \emph{multiclass} classification it is common to treat each integer
|
|
label as a so called \emph{one-hot} vector:
|
|
|
|
\(y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,\)
|
|
and
|
|
|
|
\(y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,\)
|
|
|
|
i.e.~a binary bit string of length \(C\), where \(C = 10\) is the number
|
|
of classes in the MNIST dataset (numbers from \(0\) to \(9\))..
|
|
|
|
If \(\boldsymbol{x}_i\) is the \(i\)-th input (image), \(y_{ic}\) refers
|
|
to the \(c\)-th component of the \(i\)-th output vector
|
|
\(\boldsymbol{y}_i\).\\
|
|
The probability of \(\boldsymbol{x}_i\) being in class \(c\) will be
|
|
given by the softmax function:
|
|
|
|
\[
|
|
P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) = \frac{\exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_c)}}
|
|
{\sum_{c'=0}^{C-1} \exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_{c'})}} ,
|
|
\]
|
|
|
|
which reduces to the logistic function in the binary case.\\
|
|
The likelihood of this \(C\)-class classifier is now given as:
|
|
|
|
\[
|
|
P(\mathcal{D} \mid \boldsymbol{\theta}) = \prod_{i=1}^n \prod_{c=0}^{C-1} [P(y_{ic} = 1)]^{y_{ic}} .
|
|
\]
|
|
|
|
Again we take the negative log-likelihood to define our cost function:
|
|
|
|
\[
|
|
\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n \sum_{c=0}^{C-1}
|
|
y_{ic} \ln[P(y_{ic} = 1)] = \sum_{i=1}^n
|
|
\mathcal{L}_i(\boldsymbol{\theta}) .
|
|
\]
|
|
|
|
The back propagation equations need now only a small change, namely the
|
|
definition of a new cost function. We are thus ready to use the same
|
|
equations as before! We leave it as an exercise in project 2 to derive
|
|
these equations.
|
|
|
|
\hypertarget{developing-a-code-for-doing-neural-networks-with-back-propagation}{%
|
|
\subsection{Developing a code for doing neural networks with back
|
|
propagation}\label{developing-a-code-for-doing-neural-networks-with-back-propagation}}
|
|
|
|
One can identify a set of key steps when using neural networks to solve
|
|
supervised learning problems:
|
|
|
|
\begin{enumerate}
|
|
\def\labelenumi{\arabic{enumi}.}
|
|
\item
|
|
Collect and pre-process data
|
|
\item
|
|
Define model and architecture
|
|
\item
|
|
Choose cost function and optimizer
|
|
\item
|
|
Train the model
|
|
\item
|
|
Evaluate model performance on test data
|
|
\item
|
|
Adjust hyperparameters (if necessary, network architecture)
|
|
\end{enumerate}
|
|
|
|
\hypertarget{collect-and-pre-process-data}{%
|
|
\subsection{Collect and pre-process
|
|
data}\label{collect-and-pre-process-data}}
|
|
|
|
Here we will be using the MNIST dataset, which is readily available
|
|
through the \textbf{scikit-learn} package. You may also find it for
|
|
example \href{http://yann.lecun.com/exdb/mnist/}{here}.\\
|
|
The \emph{MNIST} (Modified National Institute of Standards and
|
|
Technology) database is a large database of handwritten digits that is
|
|
commonly used for training various image processing systems.\\
|
|
The MNIST dataset consists of 70 000 images of size 28x28 pixels, each
|
|
labeled from 0 to 9.\\
|
|
The scikit-learn dataset we will use consists of a selection of 1797
|
|
images of size \(8\times 8\) collected and processed from this database.
|
|
|
|
To feed data into a feed-forward neural network we need to represent the
|
|
inputs as a feature matrix \(X = (n_{inputs}, n_{features})\). Each row
|
|
represents an \emph{input}, in this case a handwritten digit, and each
|
|
column represents a \emph{feature}, in this case a pixel. The correct
|
|
answers, also known as \emph{labels} or \emph{targets} are represented
|
|
as a 1D array of integers \(Y = (n_{inputs}) = (5, 3, 1, 8,...)\).
|
|
|
|
As an example, say we want to build a neural network using supervised
|
|
learning to predict Body-Mass Index (BMI) from measurements of height
|
|
(in m)\\
|
|
and weight (in kg). If we have measurements of 5 people the feature
|
|
matrix could be for example:
|
|
|
|
\[ X = \begin{bmatrix}
|
|
1.85 & 81\\
|
|
1.71 & 65\\
|
|
1.95 & 103\\
|
|
1.55 & 42\\
|
|
1.63 & 56
|
|
\end{bmatrix} ,\]
|
|
|
|
and the targets would be:
|
|
|
|
\[ Y = (23.7, 22.2, 27.1, 17.5, 21.1) \]
|
|
|
|
Since each input image is a 2D matrix, we need to flatten the image
|
|
(i.e. ``unravel'' the 2D matrix into a 1D array) to turn the data into a
|
|
feature matrix. This means we lose all spatial information in the image,
|
|
such as locality and translational invariance. More complicated
|
|
architectures such as Convolutional Neural Networks can take advantage
|
|
of such information, and are most commonly applied when analyzing
|
|
images.
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}2}]:} \PY{c+c1}{\PYZsh{} import necessary packages}
|
|
\PY{k+kn}{import} \PY{n+nn}{numpy} \PY{k}{as} \PY{n+nn}{np}
|
|
\PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt}
|
|
\PY{k+kn}{from} \PY{n+nn}{sklearn} \PY{k}{import} \PY{n}{datasets}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} ensure the same random numbers appear every time}
|
|
\PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{seed}\PY{p}{(}\PY{l+m+mi}{0}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} display images in notebook}
|
|
\PY{o}{\PYZpc{}}\PY{k}{matplotlib} inline
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{rcParams}\PY{p}{[}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{figure.figsize}\PY{l+s+s1}{\PYZsq{}}\PY{p}{]} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{12}\PY{p}{,}\PY{l+m+mi}{12}\PY{p}{)}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} download MNIST dataset}
|
|
\PY{n}{digits} \PY{o}{=} \PY{n}{datasets}\PY{o}{.}\PY{n}{load\PYZus{}digits}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} define inputs and labels}
|
|
\PY{n}{inputs} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{images}
|
|
\PY{n}{labels} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{target}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{inputs = (n\PYZus{}inputs, pixel\PYZus{}width, pixel\PYZus{}height) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{labels = (n\PYZus{}inputs) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{labels}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} flatten the image}
|
|
\PY{c+c1}{\PYZsh{} the value \PYZhy{}1 means dimension is inferred from the remaining dimensions: 8x8 = 64}
|
|
\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{inputs}\PY{p}{)}
|
|
\PY{n}{inputs} \PY{o}{=} \PY{n}{inputs}\PY{o}{.}\PY{n}{reshape}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{o}{\PYZhy{}}\PY{l+m+mi}{1}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{X = (n\PYZus{}inputs, n\PYZus{}features) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} choose some random images to display}
|
|
\PY{n}{indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
|
|
\PY{n}{random\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}\PY{n}{indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{l+m+mi}{5}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{image} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{digits}\PY{o}{.}\PY{n}{images}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{]}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{subplot}\PY{p}{(}\PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{n}{i}\PY{o}{+}\PY{l+m+mi}{1}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{axis}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{off}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{imshow}\PY{p}{(}\PY{n}{image}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{n}{plt}\PY{o}{.}\PY{n}{cm}\PY{o}{.}\PY{n}{gray\PYZus{}r}\PY{p}{,} \PY{n}{interpolation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{nearest}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Label: }\PY{l+s+si}{\PYZpc{}d}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{digits}\PY{o}{.}\PY{n}{target}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
inputs = (n\_inputs, pixel\_width, pixel\_height) = (1797, 8, 8)
|
|
labels = (n\_inputs) = (1797,)
|
|
X = (n\_inputs, n\_features) = (1797, 64)
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{center}
|
|
\adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_98_1.png}
|
|
\end{center}
|
|
{ \hspace*{\fill} \\}
|
|
|
|
\hypertarget{train-and-test-datasets}{%
|
|
\subsection{Train and test datasets}\label{train-and-test-datasets}}
|
|
|
|
Performing analysis before partitioning the dataset is a major error,
|
|
that can lead to incorrect conclusions.
|
|
|
|
We will reserve \(80 \%\) of our dataset for training and \(20 \%\) for
|
|
testing.
|
|
|
|
It is important that the train and test datasets are drawn randomly from
|
|
our dataset, to ensure no bias in the sampling.\\
|
|
Say you are taking measurements of weather data to predict the weather
|
|
in the coming 5 days. You don't want to train your model on measurements
|
|
taken from the hours 00.00 to 12.00, and then test it on data collected
|
|
from 12.00 to 24.00.
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}3}]:} \PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{model\PYZus{}selection} \PY{k}{import} \PY{n}{train\PYZus{}test\PYZus{}split}
|
|
|
|
\PY{c+c1}{\PYZsh{} one\PYZhy{}liner from scikit\PYZhy{}learn library}
|
|
\PY{n}{train\PYZus{}size} \PY{o}{=} \PY{l+m+mf}{0.8}
|
|
\PY{n}{test\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n}{train\PYZus{}size}
|
|
\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{train\PYZus{}test\PYZus{}split}\PY{p}{(}\PY{n}{inputs}\PY{p}{,} \PY{n}{labels}\PY{p}{,} \PY{n}{train\PYZus{}size}\PY{o}{=}\PY{n}{train\PYZus{}size}\PY{p}{,}
|
|
\PY{n}{test\PYZus{}size}\PY{o}{=}\PY{n}{test\PYZus{}size}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} equivalently in numpy}
|
|
\PY{k}{def} \PY{n+nf}{train\PYZus{}test\PYZus{}split\PYZus{}numpy}\PY{p}{(}\PY{n}{inputs}\PY{p}{,} \PY{n}{labels}\PY{p}{,} \PY{n}{train\PYZus{}size}\PY{p}{,} \PY{n}{test\PYZus{}size}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{inputs}\PY{p}{)}
|
|
\PY{n}{inputs\PYZus{}shuffled} \PY{o}{=} \PY{n}{inputs}\PY{o}{.}\PY{n}{copy}\PY{p}{(}\PY{p}{)}
|
|
\PY{n}{labels\PYZus{}shuffled} \PY{o}{=} \PY{n}{labels}\PY{o}{.}\PY{n}{copy}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{shuffle}\PY{p}{(}\PY{n}{inputs\PYZus{}shuffled}\PY{p}{)}
|
|
\PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{shuffle}\PY{p}{(}\PY{n}{labels\PYZus{}shuffled}\PY{p}{)}
|
|
|
|
\PY{n}{train\PYZus{}end} \PY{o}{=} \PY{n+nb}{int}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{o}{*}\PY{n}{train\PYZus{}size}\PY{p}{)}
|
|
\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test} \PY{o}{=} \PY{n}{inputs\PYZus{}shuffled}\PY{p}{[}\PY{p}{:}\PY{n}{train\PYZus{}end}\PY{p}{]}\PY{p}{,} \PY{n}{inputs\PYZus{}shuffled}\PY{p}{[}\PY{n}{train\PYZus{}end}\PY{p}{:}\PY{p}{]}
|
|
\PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{labels\PYZus{}shuffled}\PY{p}{[}\PY{p}{:}\PY{n}{train\PYZus{}end}\PY{p}{]}\PY{p}{,} \PY{n}{labels\PYZus{}shuffled}\PY{p}{[}\PY{n}{train\PYZus{}end}\PY{p}{:}\PY{p}{]}
|
|
|
|
\PY{k}{return} \PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test}
|
|
|
|
\PY{c+c1}{\PYZsh{}X\PYZus{}train, X\PYZus{}test, Y\PYZus{}train, Y\PYZus{}test = train\PYZus{}test\PYZus{}split\PYZus{}numpy(inputs, labels, train\PYZus{}size, test\PYZus{}size)}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Number of training images: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Number of test images: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Number of training images: 1437
|
|
Number of test images: 360
|
|
|
|
\end{Verbatim}
|
|
|
|
\hypertarget{define-model-and-architecture}{%
|
|
\subsection{Define model and
|
|
architecture}\label{define-model-and-architecture}}
|
|
|
|
Our simple feed-forward neural network will consist of an \emph{input}
|
|
layer, a single \emph{hidden} layer and an \emph{output} layer. The
|
|
activation \(y\) of each neuron is a weighted sum of inputs, passed
|
|
through an activation function:
|
|
|
|
\[ z = \sum_{i=1}^n w_i a_i ,\]
|
|
|
|
\[ y = f(z) ,\]
|
|
|
|
where \(f\) is the activation function, \(a_i\) represents input from
|
|
neuron \(i\) in the preceding layer and \(w_i\) is the weight to input
|
|
\(i\).\\
|
|
The activation of the neurons in the input layer is just the features
|
|
(e.g.~a pixel value).
|
|
|
|
The simplest activation function for a neuron is the \emph{Heaviside}
|
|
function:
|
|
|
|
\[ f(z) =
|
|
\begin{cases}
|
|
1, & z > 0\\
|
|
0, & \text{otherwise}
|
|
\end{cases}
|
|
\]
|
|
|
|
A feed-forward neural network with this activation is known as a
|
|
\emph{perceptron}.\\
|
|
For a binary classifier (i.e.~two classes, 0 or 1, dog or not-dog) we
|
|
can also use this in our output layer.\\
|
|
This activation can be generalized to \(k\) classes (using e.g.~the
|
|
\emph{one-against-all} strategy), and we call these architectures
|
|
\emph{multiclass perceptrons}.
|
|
|
|
However, it is now common to use the terms Single Layer Perceptron (SLP)
|
|
(1 hidden layer) and\\
|
|
Multilayer Perceptron (MLP) (2 or more hidden layers) to refer to
|
|
feed-forward neural networks with any activation function.
|
|
|
|
Typical choices for activation functions include the sigmoid function,
|
|
hyperbolic tangent, and Rectified Linear Unit (ReLU).\\
|
|
We will be using the sigmoid function \(\sigma(x)\):
|
|
|
|
\[ f(x) = \sigma(x) = \frac{1}{1 + e^{-x}} ,\]
|
|
|
|
which is inspired by probability theory (see logistic regression) and
|
|
was most commonly used until about 2011.
|
|
|
|
\hypertarget{layers}{%
|
|
\subsection{Layers}\label{layers}}
|
|
|
|
\begin{itemize}
|
|
\tightlist
|
|
\item
|
|
Input
|
|
\end{itemize}
|
|
|
|
Since each input image has 8x8 = 64 pixels or features, we have an input
|
|
layer of 64 neurons.
|
|
|
|
\begin{itemize}
|
|
\tightlist
|
|
\item
|
|
Hidden layer
|
|
\end{itemize}
|
|
|
|
We will use 50 neurons in the hidden layer receiving input from the
|
|
neurons in the input layer.\\
|
|
Since each neuron in the hidden layer is connected to the 64 inputs we
|
|
have 64x50 = 3200 weights to the hidden layer.
|
|
|
|
\begin{itemize}
|
|
\tightlist
|
|
\item
|
|
Output
|
|
\end{itemize}
|
|
|
|
If we were building a binary classifier, it would be sufficient with a
|
|
single neuron in the output layer, which could output 0 or 1 according
|
|
to the Heaviside function. This would be an example of a \emph{hard}
|
|
classifier, meaning it outputs the class of the input directly. However,
|
|
if we are dealing with noisy data it is often beneficial to use a
|
|
\emph{soft} classifier, which outputs the probability of being in class
|
|
0 or 1.
|
|
|
|
For a soft binary classifier, we could use a single neuron and interpret
|
|
the output as either being the probability of being in class 0 or the
|
|
probability of being in class 1. Alternatively we could use 2 neurons,
|
|
and interpret each neuron as the probability of being in each class.
|
|
|
|
Since we are doing multiclass classification, with 10 categories, it is
|
|
natural to use 10 neurons in the output layer. We number the neurons
|
|
\(j = 0,1,...,9\). The activation of each output neuron \(j\) will be
|
|
according to the \emph{softmax} function:
|
|
|
|
\[ P(\text{class $j$} \mid \text{input $\boldsymbol{a}$}) = \frac{\exp{(\boldsymbol{a}^T \boldsymbol{w}_j)}}
|
|
{\sum_{c=0}^{9} \exp{(\boldsymbol{a}^T \boldsymbol{w}_c)}} ,\]
|
|
|
|
i.e.~each neuron \(j\) outputs the probability of being in class \(j\)
|
|
given an input from the hidden layer \(\boldsymbol{a}\), with
|
|
\(\boldsymbol{w}_j\) the weights of neuron \(j\) to the inputs.\\
|
|
The denominator is a normalization factor to ensure the outputs
|
|
(probabilities) sum up to 1.\\
|
|
The exponent is just the weighted sum of inputs as before:
|
|
|
|
\[ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.\]
|
|
|
|
Since each neuron in the output layer is connected to the 50 inputs from
|
|
the hidden layer we have 50x10 = 500 weights to the output layer.
|
|
|
|
\hypertarget{weights-and-biases}{%
|
|
\subsection{Weights and biases}\label{weights-and-biases}}
|
|
|
|
Typically weights are initialized with small values distributed around
|
|
zero, drawn from a uniform or normal distribution. Setting all weights
|
|
to zero means all neurons give the same output, making the network
|
|
useless.
|
|
|
|
Adding a bias value to the weighted sum of inputs allows the neural
|
|
network to represent a greater range of values. Without it, any input
|
|
with the value 0 will be mapped to zero (before being passed through the
|
|
activation). The bias unit has an output of 1, and a weight to each
|
|
neuron \(j\), \(b_j\):
|
|
|
|
\[ z_j = \sum_{i=1}^n w_ {ij} a_i + 1\cdot b_j.\]
|
|
|
|
The bias weights \(\boldsymbol{b}\) are often initialized to zero, but a
|
|
small value like \(0.01\) ensures all neurons have some output which can
|
|
be backpropagated in the first training cycle.
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}5}]:} \PY{c+c1}{\PYZsh{} building our neural network}
|
|
|
|
\PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{n}{n\PYZus{}features} \PY{o}{=} \PY{n}{X\PYZus{}train}\PY{o}{.}\PY{n}{shape}
|
|
\PY{n}{n\PYZus{}hidden\PYZus{}neurons} \PY{o}{=} \PY{l+m+mi}{50}
|
|
\PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{l+m+mi}{10}
|
|
|
|
\PY{c+c1}{\PYZsh{} we make the weights normally distributed using numpy.random.randn}
|
|
|
|
\PY{c+c1}{\PYZsh{} weights and bias in the hidden layer}
|
|
\PY{n}{hidden\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n}{n\PYZus{}features}\PY{p}{,} \PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)}
|
|
\PY{n}{hidden\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
|
|
|
|
\PY{c+c1}{\PYZsh{} weights and bias in the output layer}
|
|
\PY{n}{output\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{)}
|
|
\PY{n}{output\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n}{n\PYZus{}categories}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
|
|
\end{Verbatim}
|
|
|
|
|
|
\hypertarget{feed-forward-pass}{%
|
|
\subsection{Feed-forward pass}\label{feed-forward-pass}}
|
|
|
|
Denote \(F\) the number of features, \(H\) the number of hidden neurons
|
|
and \(C\) the number of categories.\\
|
|
For each input image we calculate a weighted sum of input features
|
|
(pixel values) to each neuron \(j\) in the hidden layer \(l\):
|
|
|
|
\[ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},\]
|
|
|
|
this is then passed through our activation function
|
|
|
|
\[ a_{j}^{l} = f(z_{j}^{l}) .\]
|
|
|
|
We calculate a weighted sum of inputs (activations in the hidden layer)
|
|
to each neuron \(j\) in the output layer:
|
|
|
|
\[ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.\]
|
|
|
|
Finally we calculate the output of neuron \(j\) in the output layer
|
|
using the softmax function:
|
|
|
|
\[ a_{j}^{L} = \frac{\exp{(z_j^{L})}}
|
|
{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .\]
|
|
|
|
\hypertarget{matrix-multiplication}{%
|
|
\subsection{Matrix multiplication}\label{matrix-multiplication}}
|
|
|
|
Since our data has the dimensions \(X = (n_{inputs}, n_{features})\) and
|
|
our weights to the hidden layer have the dimensions\\
|
|
\(W_{hidden} = (n_{features}, n_{hidden})\), we can easily feed the
|
|
network all our training data in one go by taking the matrix product
|
|
|
|
\[ X W^{h} = (n_{inputs}, n_{hidden}),\]
|
|
|
|
and obtain a matrix that holds the weighted sum of inputs to the hidden
|
|
layer for each input image and each hidden neuron.\\
|
|
We also add the bias to obtain a matrix of weighted sums to the hidden
|
|
layer \(Z^{h}\):
|
|
|
|
\[ \hat{z}^{l} = \hat{X} \hat{W}^{l} + \hat{b}^{l} ,\]
|
|
|
|
meaning the same bias (1D array with size equal number of hidden
|
|
neurons) is added to each input image.\\
|
|
This is then passed through the activation:
|
|
|
|
\[ \hat{a}^{l} = f(\hat{z}^l) .\]
|
|
|
|
This is fed to the output layer:
|
|
|
|
\[ \hat{z}^{L} = \hat{a}^{L} \hat{W}^{L} + \hat{b}^{L} .\]
|
|
|
|
Finally we receive our output values for each image and each category by
|
|
passing it through the softmax function:
|
|
|
|
\[ output = softmax (\hat{z}^{L}) = (n_{inputs}, n_{categories}) .\]
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}6}]:} \PY{c+c1}{\PYZsh{} setup the feed\PYZhy{}forward pass, subscript h = hidden layer}
|
|
|
|
\PY{k}{def} \PY{n+nf}{sigmoid}\PY{p}{(}\PY{n}{x}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{return} \PY{l+m+mi}{1}\PY{o}{/}\PY{p}{(}\PY{l+m+mi}{1} \PY{o}{+} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{n}{x}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{feed\PYZus{}forward}\PY{p}{(}\PY{n}{X}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} weighted sum of inputs to the hidden layer}
|
|
\PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{hidden\PYZus{}bias}
|
|
\PY{c+c1}{\PYZsh{} activation in the hidden layer}
|
|
\PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n}{z\PYZus{}h}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} weighted sum of inputs to the output layer}
|
|
\PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{output\PYZus{}bias}
|
|
\PY{c+c1}{\PYZsh{} softmax output}
|
|
\PY{c+c1}{\PYZsh{} axis 0 holds each input and axis 1 the probabilities of each category}
|
|
\PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n}{z\PYZus{}o}\PY{p}{)}
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
|
|
|
|
\PY{k}{return} \PY{n}{probabilities}
|
|
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n}{feed\PYZus{}forward}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{probabilities = (n\PYZus{}inputs, n\PYZus{}categories) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{probabilities}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{probability that image 0 is in category 0,1,2,...,9 = }\PY{l+s+se}{\PYZbs{}n}\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{probabilities}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{probabilities sum up to: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{probabilities}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} we obtain a prediction by taking the class with the highest likelihood}
|
|
\PY{k}{def} \PY{n+nf}{predict}\PY{p}{(}\PY{n}{X}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n}{feed\PYZus{}forward}\PY{p}{(}\PY{n}{X}\PY{p}{)}
|
|
\PY{k}{return} \PY{n}{np}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n}{probabilities}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
|
|
|
|
\PY{n}{predictions} \PY{o}{=} \PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{predictions = (n\PYZus{}inputs) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{predictions}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{prediction for image 0: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{predictions}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{correct label for image 0: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}\PY{p}{)}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
probabilities = (n\_inputs, n\_categories) = (1437, 10)
|
|
probability that image 0 is in category 0,1,2,{\ldots},9 =
|
|
[3.89940599e-05 1.79115580e-01 1.47286800e-02 7.96733555e-01
|
|
3.28982767e-04 1.49752254e-07 9.19699482e-05 4.42365585e-03
|
|
3.57722690e-06 4.53485505e-03]
|
|
probabilities sum up to: 1.0000000000000002
|
|
|
|
predictions = (n\_inputs) = (1437,)
|
|
prediction for image 0: 3
|
|
correct label for image 0: 6
|
|
|
|
\end{Verbatim}
|
|
|
|
\hypertarget{choose-cost-function-and-optimizer}{%
|
|
\subsection{Choose cost function and
|
|
optimizer}\label{choose-cost-function-and-optimizer}}
|
|
|
|
To measure how well our neural network is doing we need to introduce a
|
|
cost function.\\
|
|
We will call the function that gives the error of a single sample output
|
|
the \emph{loss} function, and the function that gives the total error of
|
|
our network across all samples the \emph{cost} function. A typical
|
|
choice for multiclass classification is the \emph{cross-entropy} loss,
|
|
also known as the negative log likelihood.
|
|
|
|
In \emph{multiclass} classification it is common to treat each integer
|
|
label as a so called \emph{one-hot} vector:
|
|
|
|
\[ y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,\]
|
|
|
|
\[ y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,\]
|
|
|
|
i.e.~a binary bit string of length \(C\), where \(C = 10\) is the number
|
|
of classes in the MNIST dataset.
|
|
|
|
Let \(y_{ic}\) denote the \(c\)-th component of the \(i\)-th one-hot
|
|
vector.\\
|
|
We define the cost function \(\mathcal{C}\) as a sum over the
|
|
cross-entropy loss for each point \(\boldsymbol{x}_i\) in the dataset:
|
|
|
|
\hypertarget{_auto11}{}
|
|
|
|
\[
|
|
\begin{equation} \mathcal{C}(\boldsymbol{\theta}) = \sum_{i=1}^N \mathcal{L}_i (\boldsymbol{\theta})
|
|
\label{_auto11} \tag{16}
|
|
\end{equation}
|
|
\]
|
|
|
|
\hypertarget{_auto12}{}
|
|
|
|
\[
|
|
\begin{equation}
|
|
= -\sum_{i=1}^N \sum_{c=0}^{C-1} y_{ic} \log P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) .
|
|
\label{_auto12} \tag{17}
|
|
\end{equation}
|
|
\]
|
|
|
|
In the one-hot representation only one of the terms in the loss function
|
|
is non-zero, namely the probability of the correct category \(c'\)\\
|
|
(i.e.~the category \(c'\) such that \(y_{ic'} = 1\)). This means that
|
|
the cross entropy loss only punishes you for how wrong you got the
|
|
correct label. The probability of category \(c\) is given by the softmax
|
|
function. The vector \(\boldsymbol{\theta}\) represents the parameters
|
|
of our network, i.e.~all the weights and biases.
|
|
|
|
A full derivation is given in the appendix at the end.
|
|
|
|
\hypertarget{optimizing-the-cost-function}{%
|
|
\subsection{Optimizing the cost
|
|
function}\label{optimizing-the-cost-function}}
|
|
|
|
The network is trained by finding the weights and biases that minimize
|
|
the cost function. One of the most widely used classes of methods is
|
|
\emph{gradient descent} and its generalizations. The idea behind
|
|
gradient descent is simply to adjust the weights in the direction where
|
|
the gradient of the cost function is large and negative. This ensures we
|
|
flow toward a \emph{local} minimum of the cost function.\\
|
|
Each parameter \(\theta\) is iteratively adjusted according to the rule
|
|
|
|
\[ \theta_{i+1} = \theta_i - \eta \nabla \mathcal{C}(\theta_i) ,\]
|
|
|
|
where \(\eta\) is known as the \emph{learning rate}, which controls how
|
|
big a step we take towards the minimum.\\
|
|
This update can be repeated for any number of iterations, or until we
|
|
are satisfied with the result.
|
|
|
|
A simple and effective improvement is a variant called \emph{Batch
|
|
Gradient Descent}.\\
|
|
Instead of calculating the gradient on the whole dataset, we calculate
|
|
an approximation of the gradient on a subset of the data called a
|
|
\emph{minibatch}.\\
|
|
If there are \(N\) data points and we have a minibatch size of \(M\),
|
|
the total number of batches is \(N/M\).\\
|
|
We denote each minibatch \(B_k\), with \(k = 1, 2,...,N/M\). The
|
|
gradient then becomes:
|
|
|
|
\[ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad
|
|
\frac{1}{M} \sum_{i \in B_k} \nabla \mathcal{L}_i(\theta) ,\]
|
|
|
|
i.e.~instead of averaging the loss over the entire dataset, we average
|
|
over a minibatch.
|
|
|
|
This has two important benefits:\\
|
|
1. Introducing stochasticity decreases the chance that the algorithm
|
|
becomes stuck in a local minima.
|
|
|
|
\begin{enumerate}
|
|
\def\labelenumi{\arabic{enumi}.}
|
|
\setcounter{enumi}{1}
|
|
\tightlist
|
|
\item
|
|
It significantly speeds up the calculation, since we do not have to
|
|
use the entire dataset to calculate the gradient.
|
|
\end{enumerate}
|
|
|
|
\hypertarget{regularization}{%
|
|
\subsection{Regularization}\label{regularization}}
|
|
|
|
It is common to add an extra term to the cost function, proportional to
|
|
the size of the weights. This is equivalent to constraining the size of
|
|
the weights, so that they do not grow out of control. Constraining the
|
|
size of the weights means that the weights cannot grow arbitrarily large
|
|
to fit the training data, and in this way reduces \emph{overfitting}.
|
|
|
|
We will measure the size of the weights using the so called
|
|
\emph{L2-norm}, meaning our cost function becomes:
|
|
|
|
\[ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad
|
|
\frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) + \lambda \lvert \lvert \boldsymbol{w} \rvert \rvert_2^2
|
|
= \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}(\theta) + \lambda \sum_{ij} w_{ij}^2,\]
|
|
|
|
i.e.~we sum up all the weights squared. The factor \(\lambda\) is known
|
|
as a regularization parameter.
|
|
|
|
In order to train the model, we need to calculate the derivative of the
|
|
cost function with respect to every bias and weight in the network. In
|
|
total our network has \((64 + 1)\times 50=3250\) weights in the hidden
|
|
layer and \((50 + 1)\times 10=510\) weights to the output layer (\(+1\)
|
|
for the bias), and the gradient must be calculated for every parameter.
|
|
We use the \emph{backpropagation} algorithm discussed above. This is a
|
|
clever use of the chain rule that allows us to calculate the gradient
|
|
efficently.
|
|
|
|
\hypertarget{matrix-multiplication}{%
|
|
\subsection{Matrix multiplication}\label{matrix-multiplication}}
|
|
|
|
To more efficently train our network these equations are implemented
|
|
using matrix operations.\\
|
|
The error in the output layer is calculated simply as
|
|
|
|
\[ \delta_L = \hat{y} - y = (n_{inputs}, n_{categories}) .\]
|
|
|
|
The gradient for the output weights is calculated as
|
|
|
|
\[ \nabla W_{L} = \hat{a}^T \delta_L = (n_{hidden}, n_{categories}) ,\]
|
|
|
|
where \(\hat{a} = (n_{inputs}, n_{hidden})\). This simply means that we
|
|
are summing up the gradients for each input.\\
|
|
Since we are going backwards we have to transpose the activation matrix.
|
|
|
|
The gradient with respect to the output bias is then
|
|
|
|
\[ \nabla \hat{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .\]
|
|
|
|
The error in the hidden layer is
|
|
|
|
\[ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,\]
|
|
|
|
where \(f'(a_{h})\) is the derivative of the activation in the hidden
|
|
layer. The matrix products mean that we are summing up the products for
|
|
each neuron in the output layer. The symbol \(\circ\) denotes the
|
|
\emph{Hadamard product}, meaning element-wise multiplication.
|
|
|
|
This again gives us the gradients in the hidden layer:
|
|
|
|
\[ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,\]
|
|
|
|
\[ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .\]
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}7}]:} \PY{c+c1}{\PYZsh{} to categorical turns our integer vector into a onehot representation}
|
|
\PY{c+c1}{\PYZsh{}from keras.utils import to\PYZus{}categorical}
|
|
|
|
\PY{c+c1}{\PYZsh{} calculate the accuracy score of our model}
|
|
\PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{metrics} \PY{k}{import} \PY{n}{accuracy\PYZus{}score}
|
|
|
|
\PY{c+c1}{\PYZsh{} one\PYZhy{}hot in numpy}
|
|
\PY{k}{def} \PY{n+nf}{to\PYZus{}categorical\PYZus{}numpy}\PY{p}{(}\PY{n}{integer\PYZus{}vector}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{integer\PYZus{}vector}\PY{p}{)}
|
|
\PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{max}\PY{p}{(}\PY{n}{integer\PYZus{}vector}\PY{p}{)} \PY{o}{+} \PY{l+m+mi}{1}
|
|
\PY{n}{onehot\PYZus{}vector} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{onehot\PYZus{}vector}\PY{p}{[}\PY{n+nb}{range}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{)}\PY{p}{,} \PY{n}{integer\PYZus{}vector}\PY{p}{]} \PY{o}{=} \PY{l+m+mi}{1}
|
|
|
|
\PY{k}{return} \PY{n}{onehot\PYZus{}vector}
|
|
|
|
\PY{c+c1}{\PYZsh{}Y\PYZus{}train\PYZus{}onehot, Y\PYZus{}test\PYZus{}onehot = to\PYZus{}categorical(Y\PYZus{}train), to\PYZus{}categorical(Y\PYZus{}test)}
|
|
\PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{,} \PY{n}{Y\PYZus{}test\PYZus{}onehot} \PY{o}{=} \PY{n}{to\PYZus{}categorical\PYZus{}numpy}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{,} \PY{n}{to\PYZus{}categorical\PYZus{}numpy}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{feed\PYZus{}forward\PYZus{}train}\PY{p}{(}\PY{n}{X}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} weighted sum of inputs to the hidden layer}
|
|
\PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{hidden\PYZus{}bias}
|
|
\PY{c+c1}{\PYZsh{} activation in the hidden layer}
|
|
\PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n}{z\PYZus{}h}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} weighted sum of inputs to the output layer}
|
|
\PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n}{output\PYZus{}bias}
|
|
\PY{c+c1}{\PYZsh{} softmax output}
|
|
\PY{c+c1}{\PYZsh{} axis 0 holds each input and axis 1 the probabilities of each category}
|
|
\PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n}{z\PYZus{}o}\PY{p}{)}
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} for backpropagation need activations in hidden and output layers}
|
|
\PY{k}{return} \PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{probabilities}
|
|
|
|
\PY{k}{def} \PY{n+nf}{backpropagation}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n}{Y}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n}{probabilities} \PY{o}{=} \PY{n}{feed\PYZus{}forward\PYZus{}train}\PY{p}{(}\PY{n}{X}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} error in the output layer}
|
|
\PY{n}{error\PYZus{}output} \PY{o}{=} \PY{n}{probabilities} \PY{o}{\PYZhy{}} \PY{n}{Y}
|
|
\PY{c+c1}{\PYZsh{} error in the hidden layer}
|
|
\PY{n}{error\PYZus{}hidden} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n}{output\PYZus{}weights}\PY{o}{.}\PY{n}{T}\PY{p}{)} \PY{o}{*} \PY{n}{a\PYZus{}h} \PY{o}{*} \PY{p}{(}\PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n}{a\PYZus{}h}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} gradients for the output layer}
|
|
\PY{n}{output\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}output}\PY{p}{)}
|
|
\PY{n}{output\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} gradient for the hidden layer}
|
|
\PY{n}{hidden\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}hidden}\PY{p}{)}
|
|
\PY{n}{hidden\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}hidden}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
|
|
|
|
\PY{k}{return} \PY{n}{output\PYZus{}weights\PYZus{}gradient}\PY{p}{,} \PY{n}{output\PYZus{}bias\PYZus{}gradient}\PY{p}{,} \PY{n}{hidden\PYZus{}weights\PYZus{}gradient}\PY{p}{,} \PY{n}{hidden\PYZus{}bias\PYZus{}gradient}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Old accuracy on training data: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{n}{eta} \PY{o}{=} \PY{l+m+mf}{0.01}
|
|
\PY{n}{lmbd} \PY{o}{=} \PY{l+m+mf}{0.01}
|
|
\PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{l+m+mi}{1000}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} calculate gradients}
|
|
\PY{n}{dWo}\PY{p}{,} \PY{n}{dBo}\PY{p}{,} \PY{n}{dWh}\PY{p}{,} \PY{n}{dBh} \PY{o}{=} \PY{n}{backpropagation}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} regularization term gradients}
|
|
\PY{n}{dWo} \PY{o}{+}\PY{o}{=} \PY{n}{lmbd} \PY{o}{*} \PY{n}{output\PYZus{}weights}
|
|
\PY{n}{dWh} \PY{o}{+}\PY{o}{=} \PY{n}{lmbd} \PY{o}{*} \PY{n}{hidden\PYZus{}weights}
|
|
|
|
\PY{c+c1}{\PYZsh{} update weights and biases}
|
|
\PY{n}{output\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dWo}
|
|
\PY{n}{output\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dBo}
|
|
\PY{n}{hidden\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dWh}
|
|
\PY{n}{hidden\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n}{eta} \PY{o}{*} \PY{n}{dBh}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{New accuracy on training data: }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Old accuracy on training data: 0.16423103688239388
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:4: RuntimeWarning: overflow encountered in exp
|
|
after removing the cwd from sys.path.
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
New accuracy on training data: 0.10090466249130133
|
|
|
|
\end{Verbatim}
|
|
|
|
\hypertarget{improving-performance}{%
|
|
\subsection{Improving performance}\label{improving-performance}}
|
|
|
|
As we can see the network does not seem to be learning at all. It seems
|
|
to be just guessing the label for each image.\\
|
|
In order to obtain a network that does something useful, we will have to
|
|
do a bit more work.
|
|
|
|
The choice of \emph{hyperparameters} such as learning rate and
|
|
regularization parameter is hugely influential for the performance of
|
|
the network. Typically a \emph{grid-search} is performed, wherein we
|
|
test different hyperparameters separated by orders of magnitude. For
|
|
example we could test the learning rates
|
|
\(\eta = 10^{-6}, 10^{-5},...,10^{-1}\) with different regularization
|
|
parameters \(\lambda = 10^{-6},...,10^{-0}\).
|
|
|
|
Next, we haven't implemented minibatching yet, which introduces
|
|
stochasticity and is though to act as an important regularizer on the
|
|
weights. We call a feed-forward + backward pass with a minibatch an
|
|
\emph{iteration}, and a full training period going through the entire
|
|
dataset (\(n/M\) batches) an \emph{epoch}.
|
|
|
|
If this does not improve network performance, you may want to consider
|
|
altering the network architecture, adding more neurons or hidden
|
|
layers.\\
|
|
Andrew Ng goes through some of these considerations in this
|
|
\href{https://youtu.be/F1ka6a13S9I}{video}. You can find a summary of
|
|
the video
|
|
\href{https://kevinzakka.github.io/2016/09/26/applying-deep-learning/}{here}.
|
|
|
|
\hypertarget{full-object-oriented-implementation}{%
|
|
\subsection{Full object-oriented
|
|
implementation}\label{full-object-oriented-implementation}}
|
|
|
|
It is very natural to think of the network as an object, with specific
|
|
instances of the network being realizations of this object with
|
|
different hyperparameters. An implementation using Python classes
|
|
provides a clean structure and interface, and the full implementation of
|
|
our neural network is given below.
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}8}]:} \PY{k}{class} \PY{n+nc}{NeuralNetwork}\PY{p}{:}
|
|
\PY{k}{def} \PY{n+nf}{\PYZus{}\PYZus{}init\PYZus{}\PYZus{}}\PY{p}{(}
|
|
\PY{n+nb+bp}{self}\PY{p}{,}
|
|
\PY{n}{X\PYZus{}data}\PY{p}{,}
|
|
\PY{n}{Y\PYZus{}data}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{o}{=}\PY{l+m+mi}{50}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{l+m+mi}{10}\PY{p}{,}
|
|
\PY{n}{epochs}\PY{o}{=}\PY{l+m+mi}{10}\PY{p}{,}
|
|
\PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{l+m+mi}{100}\PY{p}{,}
|
|
\PY{n}{eta}\PY{o}{=}\PY{l+m+mf}{0.1}\PY{p}{,}
|
|
\PY{n}{lmbd}\PY{o}{=}\PY{l+m+mf}{0.0}\PY{p}{,}
|
|
|
|
\PY{p}{)}\PY{p}{:}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data\PYZus{}full} \PY{o}{=} \PY{n}{X\PYZus{}data}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data\PYZus{}full} \PY{o}{=} \PY{n}{Y\PYZus{}data}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n}{X\PYZus{}data}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features} \PY{o}{=} \PY{n}{X\PYZus{}data}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons} \PY{o}{=} \PY{n}{n\PYZus{}hidden\PYZus{}neurons}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{n}{n\PYZus{}categories}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs} \PY{o}{=} \PY{n}{epochs}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{n}{batch\PYZus{}size}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{/}\PY{o}{/} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{=} \PY{n}{eta}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{=} \PY{n}{lmbd}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}biases\PYZus{}and\PYZus{}weights}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{create\PYZus{}biases\PYZus{}and\PYZus{}weights}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{randn}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{)} \PY{o}{+} \PY{l+m+mf}{0.01}
|
|
|
|
\PY{k}{def} \PY{n+nf}{feed\PYZus{}forward}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} feed\PYZhy{}forward for training}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}h}\PY{p}{)}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias}
|
|
|
|
\PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}o}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{feed\PYZus{}forward\PYZus{}out}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{X}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} feed\PYZhy{}forward for output}
|
|
\PY{n}{z\PYZus{}h} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{X}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias}
|
|
\PY{n}{a\PYZus{}h} \PY{o}{=} \PY{n}{sigmoid}\PY{p}{(}\PY{n}{z\PYZus{}h}\PY{p}{)}
|
|
|
|
\PY{n}{z\PYZus{}o} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}h}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}\PY{p}{)} \PY{o}{+} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias}
|
|
|
|
\PY{n}{exp\PYZus{}term} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{exp}\PY{p}{(}\PY{n}{z\PYZus{}o}\PY{p}{)}
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n}{exp\PYZus{}term} \PY{o}{/} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{exp\PYZus{}term}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{,} \PY{n}{keepdims}\PY{o}{=}\PY{k+kc}{True}\PY{p}{)}
|
|
\PY{k}{return} \PY{n}{probabilities}
|
|
|
|
\PY{k}{def} \PY{n+nf}{backpropagation}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{error\PYZus{}output} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{probabilities} \PY{o}{\PYZhy{}} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data}
|
|
\PY{n}{error\PYZus{}hidden} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}\PY{o}{.}\PY{n}{T}\PY{p}{)} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h} \PY{o}{*} \PY{p}{(}\PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h}\PY{p}{)}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{a\PYZus{}h}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}output}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}output}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data}\PY{o}{.}\PY{n}{T}\PY{p}{,} \PY{n}{error\PYZus{}hidden}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias\PYZus{}gradient} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{error\PYZus{}hidden}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
|
|
|
|
\PY{k}{if} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{\PYZgt{}} \PY{l+m+mf}{0.0}\PY{p}{:}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights\PYZus{}gradient} \PY{o}{+}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights\PYZus{}gradient} \PY{o}{+}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}weights\PYZus{}gradient}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{output\PYZus{}bias\PYZus{}gradient}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}weights\PYZus{}gradient}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias} \PY{o}{\PYZhy{}}\PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{*} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{hidden\PYZus{}bias\PYZus{}gradient}
|
|
|
|
\PY{k}{def} \PY{n+nf}{predict}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{X}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{feed\PYZus{}forward\PYZus{}out}\PY{p}{(}\PY{n}{X}\PY{p}{)}
|
|
\PY{k}{return} \PY{n}{np}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n}{probabilities}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{predict\PYZus{}probabilities}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{X}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{feed\PYZus{}forward\PYZus{}out}\PY{p}{(}\PY{n}{X}\PY{p}{)}
|
|
\PY{k}{return} \PY{n}{probabilities}
|
|
|
|
\PY{k}{def} \PY{n+nf}{train}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{data\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} pick datapoints with replacement}
|
|
\PY{n}{chosen\PYZus{}datapoints} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}
|
|
\PY{n}{data\PYZus{}indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{replace}\PY{o}{=}\PY{k+kc}{False}
|
|
\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} minibatch training data}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}data\PYZus{}full}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}data\PYZus{}full}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{feed\PYZus{}forward}\PY{p}{(}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{backpropagation}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\hypertarget{evaluate-model-performance-on-test-data}{%
|
|
\subsection{Evaluate model performance on test
|
|
data}\label{evaluate-model-performance-on-test-data}}
|
|
|
|
To measure the performance of our network we evaluate how well it does
|
|
it data it has never seen before, i.e.~the test data.\\
|
|
We measure the performance of the network using the \emph{accuracy}
|
|
score.\\
|
|
The accuracy is as you would expect just the number of images correctly
|
|
labeled divided by the total number of images. A perfect classifier will
|
|
have an accuracy score of \(1\).
|
|
|
|
\[ \text{Accuracy} = \frac{\sum_{i=1}^n I(\hat{y}_i = y_i)}{n} ,\]
|
|
|
|
where \(I\) is the indicator function, \(1\) if \(\hat{y}_i = y_i\) and
|
|
\(0\) otherwise.
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}9}]:} \PY{n}{epochs} \PY{o}{=} \PY{l+m+mi}{100}
|
|
\PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{100}
|
|
|
|
\PY{n}{dnn} \PY{o}{=} \PY{n}{NeuralNetwork}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{,} \PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{,} \PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{o}{=}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{n}{n\PYZus{}categories}\PY{p}{)}
|
|
\PY{n}{dnn}\PY{o}{.}\PY{n}{train}\PY{p}{(}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}predict} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} accuracy score from scikit library}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Accuracy score on test set: }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}predict}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} equivalent in numpy}
|
|
\PY{k}{def} \PY{n+nf}{accuracy\PYZus{}score\PYZus{}numpy}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}pred}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{return} \PY{n}{np}\PY{o}{.}\PY{n}{sum}\PY{p}{(}\PY{n}{Y\PYZus{}test} \PY{o}{==} \PY{n}{Y\PYZus{}pred}\PY{p}{)} \PY{o}{/} \PY{n+nb}{len}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{}print(\PYZdq{}Accuracy score on test set: \PYZdq{}, accuracy\PYZus{}score\PYZus{}numpy(Y\PYZus{}test, test\PYZus{}predict))}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Accuracy score on test set: 0.9305555555555556
|
|
|
|
\end{Verbatim}
|
|
|
|
\hypertarget{adjust-hyperparameters}{%
|
|
\subsection{Adjust hyperparameters}\label{adjust-hyperparameters}}
|
|
|
|
We now perform a grid search to find the optimal hyperparameters for the
|
|
network.\\
|
|
Note that we are only using 1 layer with 50 neurons, and human
|
|
performance is estimated to be around \(98\%\) (\(2\%\) error rate).
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}10}]:} \PY{n}{eta\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
|
|
\PY{n}{lmbd\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
|
|
\PY{c+c1}{\PYZsh{} store the models for later use}
|
|
\PY{n}{DNN\PYZus{}numpy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} grid search}
|
|
\PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{dnn} \PY{o}{=} \PY{n}{NeuralNetwork}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train\PYZus{}onehot}\PY{p}{,} \PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{,} \PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{o}{=}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{n}{n\PYZus{}categories}\PY{p}{)}
|
|
\PY{n}{dnn}\PY{o}{.}\PY{n}{train}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{DNN\PYZus{}numpy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{dnn}
|
|
|
|
\PY{n}{test\PYZus{}predict} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Accuracy score on test set: }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}predict}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Learning rate = 1e-05
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.18888888888888888
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.18333333333333332
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.20277777777777778
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.2
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.12222222222222222
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.18888888888888888
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.1527777777777778
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.5916666666666667
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.5583333333333333
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.5361111111111111
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.5777777777777777
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.6055555555555555
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.6416666666666667
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.8111111111111111
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.8833333333333333
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.9
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.8666666666666667
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.875
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.8666666666666667
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.9416666666666667
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.9416666666666667
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.9472222222222222
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.9333333333333333
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.9416666666666667
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.9361111111111111
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.9527777777777777
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.9333333333333333
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.23055555555555557
|
|
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:4: RuntimeWarning: overflow encountered in exp
|
|
after removing the cwd from sys.path.
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Learning rate = 0.1
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.10555555555555556
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.10555555555555556
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.11666666666666667
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.08611111111111111
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.10555555555555556
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.125
|
|
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:44: RuntimeWarning: overflow encountered in exp
|
|
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:45: RuntimeWarning: invalid value encountered in true\_divide
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Learning rate = 1.0
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.10555555555555556
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.07777777777777778
|
|
|
|
|
|
\end{Verbatim}
|
|
|
|
\hypertarget{visualization}{%
|
|
\subsection{Visualization}\label{visualization}}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}11}]:} \PY{c+c1}{\PYZsh{} visual representation of grid search}
|
|
\PY{c+c1}{\PYZsh{} uses seaborn heatmap, you can also do this with matplotlib imshow}
|
|
\PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
|
|
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{dnn} \PY{o}{=} \PY{n}{DNN\PYZus{}numpy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
|
|
|
|
\PY{n}{train\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
|
|
|
|
\PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{train\PYZus{}pred}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}pred}\PY{p}{)}
|
|
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
/usr/local/lib/python3.7/site-packages/ipykernel\_launcher.py:4: RuntimeWarning: overflow encountered in exp
|
|
after removing the cwd from sys.path.
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{center}
|
|
\adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_117_1.png}
|
|
\end{center}
|
|
{ \hspace*{\fill} \\}
|
|
|
|
\begin{center}
|
|
\adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_117_2.png}
|
|
\end{center}
|
|
{ \hspace*{\fill} \\}
|
|
|
|
\hypertarget{scikit-learn-implementation}{%
|
|
\subsection{scikit-learn
|
|
implementation}\label{scikit-learn-implementation}}
|
|
|
|
\textbf{scikit-learn} focuses more on traditional machine learning
|
|
methods, such as regression, clustering, decision trees, etc. As such,
|
|
it has only two types of neural networks: Multi Layer Perceptron
|
|
outputting continuous values, \emph{MPLRegressor}, and Multi Layer
|
|
Perceptron outputting labels, \emph{MLPClassifier}. We will see how
|
|
simple it is to use these classes.
|
|
|
|
\textbf{scikit-learn} implements a few improvements from our neural
|
|
network, such as early stopping, a varying learning rate, different
|
|
optimization methods, etc. We would therefore expect a better
|
|
performance overall.
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}12}]:} \PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{neural\PYZus{}network} \PY{k}{import} \PY{n}{MLPClassifier}
|
|
\PY{c+c1}{\PYZsh{} store models for later use}
|
|
\PY{n}{DNN\PYZus{}scikit} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{dnn} \PY{o}{=} \PY{n}{MLPClassifier}\PY{p}{(}\PY{n}{hidden\PYZus{}layer\PYZus{}sizes}\PY{o}{=}\PY{p}{(}\PY{n}{n\PYZus{}hidden\PYZus{}neurons}\PY{p}{)}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{logistic}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,}
|
|
\PY{n}{alpha}\PY{o}{=}\PY{n}{lmbd}\PY{p}{,} \PY{n}{learning\PYZus{}rate\PYZus{}init}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{max\PYZus{}iter}\PY{o}{=}\PY{n}{epochs}\PY{p}{)}
|
|
\PY{n}{dnn}\PY{o}{.}\PY{n}{fit}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}
|
|
|
|
\PY{n}{DNN\PYZus{}scikit}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{dnn}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Accuracy score on test set: }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{dnn}\PY{o}{.}\PY{n}{score}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
/usr/local/lib/python3.7/site-packages/sklearn/neural\_network/multilayer\_perceptron.py:564: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
|
|
\% self.max\_iter, ConvergenceWarning)
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Learning rate = 1e-05
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.20833333333333334
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.16944444444444445
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.18055555555555555
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.17222222222222222
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.23333333333333334
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.2916666666666667
|
|
|
|
Learning rate = 1e-05
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.1527777777777778
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.8666666666666667
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.8416666666666667
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.9277777777777778
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.8861111111111111
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.8805555555555555
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.875
|
|
|
|
Learning rate = 0.0001
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.875
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.9833333333333333
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.9833333333333333
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.9805555555555555
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.9861111111111112
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.9805555555555555
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.9805555555555555
|
|
|
|
Learning rate = 0.001
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.9527777777777777
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.9861111111111112
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.9916666666666667
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.9861111111111112
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.9833333333333333
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.9944444444444445
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.975
|
|
|
|
Learning rate = 0.01
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.9416666666666667
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.9416666666666667
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.75
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.8972222222222223
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.8944444444444445
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.9277777777777778
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.8972222222222223
|
|
|
|
Learning rate = 0.1
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.7388888888888889
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.08611111111111111
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.10555555555555556
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.11388888888888889
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.1527777777777778
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.15
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.08888888888888889
|
|
|
|
Learning rate = 1.0
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.15
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 1e-05
|
|
Accuracy score on test set: 0.1
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.0001
|
|
Accuracy score on test set: 0.11944444444444445
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.001
|
|
Accuracy score on test set: 0.18055555555555555
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.01
|
|
Accuracy score on test set: 0.10555555555555556
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 0.1
|
|
Accuracy score on test set: 0.08333333333333333
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 1.0
|
|
Accuracy score on test set: 0.09166666666666666
|
|
|
|
Learning rate = 10.0
|
|
Lambda = 10.0
|
|
Accuracy score on test set: 0.08055555555555556
|
|
|
|
|
|
\end{Verbatim}
|
|
|
|
\hypertarget{visualization}{%
|
|
\subsection{Visualization}\label{visualization}}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}13}]:} \PY{c+c1}{\PYZsh{} optional}
|
|
\PY{c+c1}{\PYZsh{} visual representation of grid search}
|
|
\PY{c+c1}{\PYZsh{} uses seaborn heatmap, could probably do this in matplotlib}
|
|
\PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
|
|
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{dnn} \PY{o}{=} \PY{n}{DNN\PYZus{}scikit}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
|
|
|
|
\PY{n}{train\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}pred} \PY{o}{=} \PY{n}{dnn}\PY{o}{.}\PY{n}{predict}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{)}
|
|
|
|
\PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{train\PYZus{}pred}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{accuracy\PYZus{}score}\PY{p}{(}\PY{n}{Y\PYZus{}test}\PY{p}{,} \PY{n}{test\PYZus{}pred}\PY{p}{)}
|
|
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{center}
|
|
\adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_121_0.png}
|
|
\end{center}
|
|
{ \hspace*{\fill} \\}
|
|
|
|
\begin{center}
|
|
\adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_121_1.png}
|
|
\end{center}
|
|
{ \hspace*{\fill} \\}
|
|
|
|
\hypertarget{building-neural-networks-in-tensorflow-and-keras}{%
|
|
\subsection{Building neural networks in Tensorflow and
|
|
Keras}\label{building-neural-networks-in-tensorflow-and-keras}}
|
|
|
|
Now we want to build on the experience gained from our neural network
|
|
implementation in NumPy and scikit-learn and use it to construct a
|
|
neural network in Tensorflow. Once we have constructed a neural network
|
|
in NumPy and Tensorflow, building one in Keras is really quite trivial,
|
|
though the performance may suffer.
|
|
|
|
In our previous example we used only one hidden layer, and in this we
|
|
will use two. From this it should be quite clear how to build one using
|
|
an arbitrary number of hidden layers, using data structures such as
|
|
Python lists or NumPy arrays.
|
|
|
|
\hypertarget{tensorflow}{%
|
|
\subsection{Tensorflow}\label{tensorflow}}
|
|
|
|
Tensorflow is an open source library machine learning library developed
|
|
by the Google Brain team for internal use. It was released under the
|
|
Apache 2.0 open source license in November 9, 2015.
|
|
|
|
Tensorflow is a computational framework that allows you to construct
|
|
machine learning models at different levels of abstraction, from
|
|
high-level, object-oriented APIs like Keras, down to the C++ kernels
|
|
that Tensorflow is built upon. The higher levels of abstraction are
|
|
simpler to use, but less flexible, and our choice of implementation
|
|
should reflect the problems we are trying to solve.
|
|
|
|
\href{https://www.tensorflow.org/guide/graphs}{Tensorflow uses}
|
|
so-called graphs to represent your computation in terms of the
|
|
dependencies between individual operations, such that you first build a
|
|
Tensorflow \emph{graph} to represent your model, and then create a
|
|
Tensorflow \emph{session} to run the graph.
|
|
|
|
In this guide we will analyze the same data as we did in our NumPy and
|
|
scikit-learn tutorial, gathered from the MNIST database of images. We
|
|
will give an introduction to the lower level Python Application Program
|
|
Interfaces (APIs), and see how we use them to build our graph. Then we
|
|
will build (effectively) the same graph in Keras, to see just how simple
|
|
solving a machine learning problem can be.
|
|
|
|
To install tensorflow on Unix/Linux systems, use pip as
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}13}]:} \PY{n}{pip3} \PY{n}{install} \PY{n}{tensorflow}
|
|
\end{Verbatim}
|
|
|
|
|
|
and/or if you use \textbf{anaconda}, just write (or install from the
|
|
graphical user interface)
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}14}]:} \PY{n}{conda} \PY{n}{install} \PY{n}{tensorflow}
|
|
\end{Verbatim}
|
|
|
|
|
|
\hypertarget{collect-and-pre-process-data}{%
|
|
\subsection{Collect and pre-process
|
|
data}\label{collect-and-pre-process-data}}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}14}]:} \PY{c+c1}{\PYZsh{} import necessary packages}
|
|
\PY{k+kn}{import} \PY{n+nn}{numpy} \PY{k}{as} \PY{n+nn}{np}
|
|
\PY{k+kn}{import} \PY{n+nn}{matplotlib}\PY{n+nn}{.}\PY{n+nn}{pyplot} \PY{k}{as} \PY{n+nn}{plt}
|
|
\PY{k+kn}{from} \PY{n+nn}{sklearn} \PY{k}{import} \PY{n}{datasets}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} ensure the same random numbers appear every time}
|
|
\PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{seed}\PY{p}{(}\PY{l+m+mi}{0}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} display images in notebook}
|
|
\PY{o}{\PYZpc{}}\PY{k}{matplotlib} inline
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{rcParams}\PY{p}{[}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{figure.figsize}\PY{l+s+s1}{\PYZsq{}}\PY{p}{]} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{12}\PY{p}{,}\PY{l+m+mi}{12}\PY{p}{)}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} download MNIST dataset}
|
|
\PY{n}{digits} \PY{o}{=} \PY{n}{datasets}\PY{o}{.}\PY{n}{load\PYZus{}digits}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} define inputs and labels}
|
|
\PY{n}{inputs} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{images}
|
|
\PY{n}{labels} \PY{o}{=} \PY{n}{digits}\PY{o}{.}\PY{n}{target}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{inputs = (n\PYZus{}inputs, pixel\PYZus{}width, pixel\PYZus{}height) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{labels = (n\PYZus{}inputs) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{labels}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} flatten the image}
|
|
\PY{c+c1}{\PYZsh{} the value \PYZhy{}1 means dimension is inferred from the remaining dimensions: 8x8 = 64}
|
|
\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n+nb}{len}\PY{p}{(}\PY{n}{inputs}\PY{p}{)}
|
|
\PY{n}{inputs} \PY{o}{=} \PY{n}{inputs}\PY{o}{.}\PY{n}{reshape}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{,} \PY{o}{\PYZhy{}}\PY{l+m+mi}{1}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{X = (n\PYZus{}inputs, n\PYZus{}features) = }\PY{l+s+s2}{\PYZdq{}} \PY{o}{+} \PY{n+nb}{str}\PY{p}{(}\PY{n}{inputs}\PY{o}{.}\PY{n}{shape}\PY{p}{)}\PY{p}{)}
|
|
|
|
|
|
\PY{c+c1}{\PYZsh{} choose some random images to display}
|
|
\PY{n}{indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
|
|
\PY{n}{random\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}\PY{n}{indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{l+m+mi}{5}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{image} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{digits}\PY{o}{.}\PY{n}{images}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{]}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{subplot}\PY{p}{(}\PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{5}\PY{p}{,} \PY{n}{i}\PY{o}{+}\PY{l+m+mi}{1}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{axis}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{off}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{imshow}\PY{p}{(}\PY{n}{image}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{n}{plt}\PY{o}{.}\PY{n}{cm}\PY{o}{.}\PY{n}{gray\PYZus{}r}\PY{p}{,} \PY{n}{interpolation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{nearest}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Label: }\PY{l+s+si}{\PYZpc{}d}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{digits}\PY{o}{.}\PY{n}{target}\PY{p}{[}\PY{n}{random\PYZus{}indices}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{]}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
inputs = (n\_inputs, pixel\_width, pixel\_height) = (1797, 8, 8)
|
|
labels = (n\_inputs) = (1797,)
|
|
X = (n\_inputs, n\_features) = (1797, 64)
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{center}
|
|
\adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{output_127_1.png}
|
|
\end{center}
|
|
{ \hspace*{\fill} \\}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}15}]:} \PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{utils} \PY{k}{import} \PY{n}{to\PYZus{}categorical}
|
|
\PY{k+kn}{from} \PY{n+nn}{sklearn}\PY{n+nn}{.}\PY{n+nn}{model\PYZus{}selection} \PY{k}{import} \PY{n}{train\PYZus{}test\PYZus{}split}
|
|
|
|
\PY{c+c1}{\PYZsh{} one\PYZhy{}hot representation of labels}
|
|
\PY{n}{labels} \PY{o}{=} \PY{n}{to\PYZus{}categorical}\PY{p}{(}\PY{n}{labels}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} split into train and test data}
|
|
\PY{n}{train\PYZus{}size} \PY{o}{=} \PY{l+m+mf}{0.8}
|
|
\PY{n}{test\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{1} \PY{o}{\PYZhy{}} \PY{n}{train\PYZus{}size}
|
|
\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{train\PYZus{}test\PYZus{}split}\PY{p}{(}\PY{n}{inputs}\PY{p}{,} \PY{n}{labels}\PY{p}{,} \PY{n}{train\PYZus{}size}\PY{o}{=}\PY{n}{train\PYZus{}size}\PY{p}{,}
|
|
\PY{n}{test\PYZus{}size}\PY{o}{=}\PY{n}{test\PYZus{}size}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
Using TensorFlow backend.
|
|
|
|
\end{Verbatim}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
|
|
---------------------------------------------------------------------------
|
|
|
|
ModuleNotFoundError Traceback (most recent call last)
|
|
|
|
<ipython-input-15-b532db97c8ce> in <module>()
|
|
----> 1 from keras.utils import to\_categorical
|
|
2 from sklearn.model\_selection import train\_test\_split
|
|
3
|
|
4 \# one-hot representation of labels
|
|
5 labels = to\_categorical(labels)
|
|
|
|
|
|
/usr/local/lib/python3.7/site-packages/keras/\_\_init\_\_.py in <module>()
|
|
1 from \_\_future\_\_ import absolute\_import
|
|
2
|
|
----> 3 from . import utils
|
|
4 from . import activations
|
|
5 from . import applications
|
|
|
|
|
|
/usr/local/lib/python3.7/site-packages/keras/utils/\_\_init\_\_.py in <module>()
|
|
4 from . import data\_utils
|
|
5 from . import io\_utils
|
|
----> 6 from . import conv\_utils
|
|
7
|
|
8 \# Globally-importable utils.
|
|
|
|
|
|
/usr/local/lib/python3.7/site-packages/keras/utils/conv\_utils.py in <module>()
|
|
7 from six.moves import range
|
|
8 import numpy as np
|
|
----> 9 from .. import backend as K
|
|
10
|
|
11
|
|
|
|
|
|
/usr/local/lib/python3.7/site-packages/keras/backend/\_\_init\_\_.py in <module>()
|
|
87 elif \_BACKEND == 'tensorflow':
|
|
88 sys.stderr.write('Using TensorFlow backend.\textbackslash{}n')
|
|
---> 89 from .tensorflow\_backend import *
|
|
90 else:
|
|
91 \# Try and load external backend.
|
|
|
|
|
|
/usr/local/lib/python3.7/site-packages/keras/backend/tensorflow\_backend.py in <module>()
|
|
3 from \_\_future\_\_ import print\_function
|
|
4
|
|
----> 5 import tensorflow as tf
|
|
6 from tensorflow.python.framework import ops as tf\_ops
|
|
7 from tensorflow.python.training import moving\_averages
|
|
|
|
|
|
ModuleNotFoundError: No module named 'tensorflow'
|
|
|
|
\end{Verbatim}
|
|
|
|
\hypertarget{using-tensorflow-backend}{%
|
|
\subsection{Using TensorFlow backend}\label{using-tensorflow-backend}}
|
|
|
|
\begin{enumerate}
|
|
\def\labelenumi{\arabic{enumi}.}
|
|
\item
|
|
Define model and architecture
|
|
\item
|
|
Choose cost function and optimizer
|
|
\end{enumerate}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}17}]:} \PY{k+kn}{import} \PY{n+nn}{tensorflow} \PY{k}{as} \PY{n+nn}{tf}
|
|
|
|
\PY{k}{class} \PY{n+nc}{NeuralNetworkTensorflow}\PY{p}{:}
|
|
\PY{k}{def} \PY{n+nf}{\PYZus{}\PYZus{}init\PYZus{}\PYZus{}}\PY{p}{(}
|
|
\PY{n+nb+bp}{self}\PY{p}{,}
|
|
\PY{n}{X\PYZus{}train}\PY{p}{,}
|
|
\PY{n}{Y\PYZus{}train}\PY{p}{,}
|
|
\PY{n}{X\PYZus{}test}\PY{p}{,}
|
|
\PY{n}{Y\PYZus{}test}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{o}{=}\PY{l+m+mi}{100}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{o}{=}\PY{l+m+mi}{50}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}categories}\PY{o}{=}\PY{l+m+mi}{2}\PY{p}{,}
|
|
\PY{n}{epochs}\PY{o}{=}\PY{l+m+mi}{10}\PY{p}{,}
|
|
\PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{l+m+mi}{100}\PY{p}{,}
|
|
\PY{n}{eta}\PY{o}{=}\PY{l+m+mf}{0.1}\PY{p}{,}
|
|
\PY{n}{lmbd}\PY{o}{=}\PY{l+m+mf}{0.0}\PY{p}{,}
|
|
\PY{p}{)}\PY{p}{:}
|
|
|
|
\PY{c+c1}{\PYZsh{} keep track of number of steps}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{global\PYZus{}step} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{Variable}\PY{p}{(}\PY{l+m+mi}{0}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{int32}\PY{p}{,} \PY{n}{trainable}\PY{o}{=}\PY{k+kc}{False}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{global\PYZus{}step}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}train} \PY{o}{=} \PY{n}{X\PYZus{}train}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}train} \PY{o}{=} \PY{n}{Y\PYZus{}train}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}test} \PY{o}{=} \PY{n}{X\PYZus{}test}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}test} \PY{o}{=} \PY{n}{Y\PYZus{}test}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{=} \PY{n}{X\PYZus{}train}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{0}\PY{p}{]}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features} \PY{o}{=} \PY{n}{X\PYZus{}train}\PY{o}{.}\PY{n}{shape}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1} \PY{o}{=} \PY{n}{n\PYZus{}neurons\PYZus{}layer1}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2} \PY{o}{=} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{n}{n\PYZus{}categories}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs} \PY{o}{=} \PY{n}{epochs}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{n}{batch\PYZus{}size}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs} \PY{o}{/}\PY{o}{/} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta} \PY{o}{=} \PY{n}{eta}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd} \PY{o}{=} \PY{n}{lmbd}
|
|
|
|
\PY{c+c1}{\PYZsh{} build network piece by piece}
|
|
\PY{c+c1}{\PYZsh{} name scopes (with) are used to enforce creation of new variables}
|
|
\PY{c+c1}{\PYZsh{} https://www.tensorflow.org/guide/variables}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}placeholders}\PY{p}{(}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}DNN}\PY{p}{(}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}loss}\PY{p}{(}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}optimiser}\PY{p}{(}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{create\PYZus{}accuracy}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{create\PYZus{}placeholders}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} placeholders are fine here, but \PYZdq{}Datasets\PYZdq{} are the preferred method}
|
|
\PY{c+c1}{\PYZsh{} of streaming data into a model}
|
|
\PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{data}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{placeholder}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{,} \PY{n}{shape}\PY{o}{=}\PY{p}{(}\PY{k+kc}{None}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features}\PY{p}{)}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{X\PYZus{}data}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{placeholder}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{,} \PY{n}{shape}\PY{o}{=}\PY{p}{(}\PY{k+kc}{None}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{)}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{Y\PYZus{}data}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{create\PYZus{}DNN}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{DNN}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
|
|
\PY{c+c1}{\PYZsh{} the weights are stored to calculate regularization loss later}
|
|
|
|
\PY{c+c1}{\PYZsh{} Fully connected layer 1}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc1} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{weight\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}features}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc1}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
|
|
\PY{n}{b\PYZus{}fc1} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{bias\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc1}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
|
|
\PY{n}{a\PYZus{}fc1} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{sigmoid}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc1}\PY{p}{)} \PY{o}{+} \PY{n}{b\PYZus{}fc1}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} Fully connected layer 2}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc2} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{weight\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc2}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
|
|
\PY{n}{b\PYZus{}fc2} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{bias\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{fc2}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
|
|
\PY{n}{a\PYZus{}fc2} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{sigmoid}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}fc1}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc2}\PY{p}{)} \PY{o}{+} \PY{n}{b\PYZus{}fc2}\PY{p}{)}
|
|
|
|
\PY{c+c1}{\PYZsh{} Output layer}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}out} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{weight\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{out}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
|
|
\PY{n}{b\PYZus{}out} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{bias\PYZus{}variable}\PY{p}{(}\PY{p}{[}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}categories}\PY{p}{]}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{out}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}out} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{matmul}\PY{p}{(}\PY{n}{a\PYZus{}fc2}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}out}\PY{p}{)} \PY{o}{+} \PY{n}{b\PYZus{}out}
|
|
|
|
\PY{k}{def} \PY{n+nf}{create\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{loss}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{softmax\PYZus{}loss} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{reduce\PYZus{}mean}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{softmax\PYZus{}cross\PYZus{}entropy\PYZus{}with\PYZus{}logits\PYZus{}v2}\PY{p}{(}\PY{n}{labels}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y}\PY{p}{,} \PY{n}{logits}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}out}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{n}{regularizer\PYZus{}loss\PYZus{}fc1} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{l2\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc1}\PY{p}{)}
|
|
\PY{n}{regularizer\PYZus{}loss\PYZus{}fc2} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{l2\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}fc2}\PY{p}{)}
|
|
\PY{n}{regularizer\PYZus{}loss\PYZus{}out} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{l2\PYZus{}loss}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{W\PYZus{}out}\PY{p}{)}
|
|
\PY{n}{regularizer\PYZus{}loss} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{lmbd}\PY{o}{*}\PY{p}{(}\PY{n}{regularizer\PYZus{}loss\PYZus{}fc1} \PY{o}{+} \PY{n}{regularizer\PYZus{}loss\PYZus{}fc2} \PY{o}{+} \PY{n}{regularizer\PYZus{}loss\PYZus{}out}\PY{p}{)}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{loss} \PY{o}{=} \PY{n}{softmax\PYZus{}loss} \PY{o}{+} \PY{n}{regularizer\PYZus{}loss}
|
|
|
|
\PY{k}{def} \PY{n+nf}{create\PYZus{}accuracy}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{accuracy}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{probabilities} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{nn}\PY{o}{.}\PY{n}{softmax}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{z\PYZus{}out}\PY{p}{)}
|
|
\PY{n}{predictions} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n}{probabilities}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
|
|
\PY{n}{labels} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{argmax}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y}\PY{p}{,} \PY{n}{axis}\PY{o}{=}\PY{l+m+mi}{1}\PY{p}{)}
|
|
|
|
\PY{n}{correct\PYZus{}predictions} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{equal}\PY{p}{(}\PY{n}{predictions}\PY{p}{,} \PY{n}{labels}\PY{p}{)}
|
|
\PY{n}{correct\PYZus{}predictions} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{cast}\PY{p}{(}\PY{n}{correct\PYZus{}predictions}\PY{p}{,} \PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{accuracy} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{reduce\PYZus{}mean}\PY{p}{(}\PY{n}{correct\PYZus{}predictions}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{create\PYZus{}optimiser}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{name\PYZus{}scope}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{optimizer}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{:}
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{optimizer} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{train}\PY{o}{.}\PY{n}{GradientDescentOptimizer}\PY{p}{(}\PY{n}{learning\PYZus{}rate}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{eta}\PY{p}{)}\PY{o}{.}\PY{n}{minimize}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{global\PYZus{}step}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{global\PYZus{}step}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{weight\PYZus{}variable}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{shape}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{initial} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{truncated\PYZus{}normal}\PY{p}{(}\PY{n}{shape}\PY{p}{,} \PY{n}{stddev}\PY{o}{=}\PY{l+m+mf}{0.1}\PY{p}{)}
|
|
\PY{k}{return} \PY{n}{tf}\PY{o}{.}\PY{n}{Variable}\PY{p}{(}\PY{n}{initial}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{n}{name}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{dtype}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{bias\PYZus{}variable}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{,} \PY{n}{shape}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{tf}\PY{o}{.}\PY{n}{float32}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{initial} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{constant}\PY{p}{(}\PY{l+m+mf}{0.1}\PY{p}{,} \PY{n}{shape}\PY{o}{=}\PY{n}{shape}\PY{p}{)}
|
|
\PY{k}{return} \PY{n}{tf}\PY{o}{.}\PY{n}{Variable}\PY{p}{(}\PY{n}{initial}\PY{p}{,} \PY{n}{name}\PY{o}{=}\PY{n}{name}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n}{dtype}\PY{p}{)}
|
|
|
|
\PY{k}{def} \PY{n+nf}{fit}\PY{p}{(}\PY{n+nb+bp}{self}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{data\PYZus{}indices} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{arange}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{n\PYZus{}inputs}\PY{p}{)}
|
|
|
|
\PY{k}{with} \PY{n}{tf}\PY{o}{.}\PY{n}{Session}\PY{p}{(}\PY{p}{)} \PY{k}{as} \PY{n}{sess}\PY{p}{:}
|
|
\PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{global\PYZus{}variables\PYZus{}initializer}\PY{p}{(}\PY{p}{)}\PY{p}{)}
|
|
\PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{epochs}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{iterations}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{chosen\PYZus{}datapoints} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{random}\PY{o}{.}\PY{n}{choice}\PY{p}{(}\PY{n}{data\PYZus{}indices}\PY{p}{,} \PY{n}{size}\PY{o}{=}\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{replace}\PY{o}{=}\PY{k+kc}{False}\PY{p}{)}
|
|
\PY{n}{batch\PYZus{}X}\PY{p}{,} \PY{n}{batch\PYZus{}Y} \PY{o}{=} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}train}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}train}\PY{p}{[}\PY{n}{chosen\PYZus{}datapoints}\PY{p}{]}
|
|
|
|
\PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{p}{[}\PY{n}{DNN}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{DNN}\PY{o}{.}\PY{n}{optimizer}\PY{p}{]}\PY{p}{,}
|
|
\PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n}{batch\PYZus{}X}\PY{p}{,}
|
|
\PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n}{batch\PYZus{}Y}\PY{p}{\PYZcb{}}\PY{p}{)}
|
|
\PY{n}{accuracy} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{n}{DNN}\PY{o}{.}\PY{n}{accuracy}\PY{p}{,}
|
|
\PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n}{batch\PYZus{}X}\PY{p}{,}
|
|
\PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n}{batch\PYZus{}Y}\PY{p}{\PYZcb{}}\PY{p}{)}
|
|
\PY{n}{step} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{n}{DNN}\PY{o}{.}\PY{n}{global\PYZus{}step}\PY{p}{)}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{train\PYZus{}loss}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{p}{[}\PY{n}{DNN}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{DNN}\PY{o}{.}\PY{n}{accuracy}\PY{p}{]}\PY{p}{,}
|
|
\PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}train}\PY{p}{,}
|
|
\PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}train}\PY{p}{\PYZcb{}}\PY{p}{)}
|
|
|
|
\PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{test\PYZus{}loss}\PY{p}{,} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{sess}\PY{o}{.}\PY{n}{run}\PY{p}{(}\PY{p}{[}\PY{n}{DNN}\PY{o}{.}\PY{n}{loss}\PY{p}{,} \PY{n}{DNN}\PY{o}{.}\PY{n}{accuracy}\PY{p}{]}\PY{p}{,}
|
|
\PY{n}{feed\PYZus{}dict}\PY{o}{=}\PY{p}{\PYZob{}}\PY{n}{DNN}\PY{o}{.}\PY{n}{X}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{X\PYZus{}test}\PY{p}{,}
|
|
\PY{n}{DNN}\PY{o}{.}\PY{n}{Y}\PY{p}{:} \PY{n+nb+bp}{self}\PY{o}{.}\PY{n}{Y\PYZus{}test}\PY{p}{\PYZcb{}}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\hypertarget{optimizing-and-using-gradient-descent}{%
|
|
\subsection{Optimizing and using gradient
|
|
descent}\label{optimizing-and-using-gradient-descent}}
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}18}]:} \PY{n}{epochs} \PY{o}{=} \PY{l+m+mi}{100}
|
|
\PY{n}{batch\PYZus{}size} \PY{o}{=} \PY{l+m+mi}{100}
|
|
\PY{n}{n\PYZus{}neurons\PYZus{}layer1} \PY{o}{=} \PY{l+m+mi}{100}
|
|
\PY{n}{n\PYZus{}neurons\PYZus{}layer2} \PY{o}{=} \PY{l+m+mi}{50}
|
|
\PY{n}{n\PYZus{}categories} \PY{o}{=} \PY{l+m+mi}{10}
|
|
\PY{n}{eta\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
|
|
\PY{n}{lmbd\PYZus{}vals} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{logspace}\PY{p}{(}\PY{o}{\PYZhy{}}\PY{l+m+mi}{5}\PY{p}{,} \PY{l+m+mi}{1}\PY{p}{,} \PY{l+m+mi}{7}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}19}]:} \PY{n}{DNN\PYZus{}tf} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{DNN} \PY{o}{=} \PY{n}{NeuralNetworkTensorflow}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{,}
|
|
\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{,}
|
|
\PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{)}
|
|
\PY{n}{DNN}\PY{o}{.}\PY{n}{fit}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{DNN\PYZus{}tf}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test accuracy: }\PY{l+s+si}{\PYZpc{}.3f}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{DNN}\PY{o}{.}\PY{n}{test\PYZus{}accuracy}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}20}]:} \PY{c+c1}{\PYZsh{} optional}
|
|
\PY{c+c1}{\PYZsh{} visual representation of grid search}
|
|
\PY{c+c1}{\PYZsh{} uses seaborn heatmap, could probably do this in matplotlib}
|
|
\PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
|
|
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{DNN} \PY{o}{=} \PY{n}{DNN\PYZus{}tf}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
|
|
|
|
\PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{train\PYZus{}accuracy}
|
|
\PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{test\PYZus{}accuracy}
|
|
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}21}]:} \PY{c+c1}{\PYZsh{} optional}
|
|
\PY{c+c1}{\PYZsh{} we can use log files to visualize our graph in Tensorboard}
|
|
\PY{n}{writer} \PY{o}{=} \PY{n}{tf}\PY{o}{.}\PY{n}{summary}\PY{o}{.}\PY{n}{FileWriter}\PY{p}{(}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{logs/}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}
|
|
\PY{n}{writer}\PY{o}{.}\PY{n}{add\PYZus{}graph}\PY{p}{(}\PY{n}{tf}\PY{o}{.}\PY{n}{get\PYZus{}default\PYZus{}graph}\PY{p}{(}\PY{p}{)}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\hypertarget{using-keras}{%
|
|
\subsection{Using Keras}\label{using-keras}}
|
|
|
|
Keras is a high level
|
|
\href{https://en.wikipedia.org/wiki/Application_programming_interface}{neural
|
|
network} that supports Tensorflow, CTNK and Theano as backends.\\
|
|
If you have Tensorflow installed Keras is available through the
|
|
\emph{tf.keras} module.\\
|
|
If you have Anaconda installed you may run the following command
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}22}]:} \PY{n}{conda} \PY{n}{install} \PY{n}{keras}
|
|
\end{Verbatim}
|
|
|
|
|
|
Alternatively, if you have Tensorflow or one of the other supported
|
|
backends install you may use the pip package manager:
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}23}]:} \PY{n}{pip3} \PY{n}{install} \PY{n}{keras}
|
|
\end{Verbatim}
|
|
|
|
|
|
or look up the \href{https://keras.io/}{instructions here}.
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}24}]:} \PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{models} \PY{k}{import} \PY{n}{Sequential}
|
|
\PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{layers} \PY{k}{import} \PY{n}{Dense}
|
|
\PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{regularizers} \PY{k}{import} \PY{n}{l2}
|
|
\PY{k+kn}{from} \PY{n+nn}{keras}\PY{n+nn}{.}\PY{n+nn}{optimizers} \PY{k}{import} \PY{n}{SGD}
|
|
|
|
\PY{k}{def} \PY{n+nf}{create\PYZus{}neural\PYZus{}network\PYZus{}keras}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{,} \PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{model} \PY{o}{=} \PY{n}{Sequential}\PY{p}{(}\PY{p}{)}
|
|
\PY{n}{model}\PY{o}{.}\PY{n}{add}\PY{p}{(}\PY{n}{Dense}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sigmoid}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{kernel\PYZus{}regularizer}\PY{o}{=}\PY{n}{l2}\PY{p}{(}\PY{n}{lmbd}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{model}\PY{o}{.}\PY{n}{add}\PY{p}{(}\PY{n}{Dense}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{sigmoid}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{kernel\PYZus{}regularizer}\PY{o}{=}\PY{n}{l2}\PY{p}{(}\PY{n}{lmbd}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{model}\PY{o}{.}\PY{n}{add}\PY{p}{(}\PY{n}{Dense}\PY{p}{(}\PY{n}{n\PYZus{}categories}\PY{p}{,} \PY{n}{activation}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{softmax}\PY{l+s+s1}{\PYZsq{}}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{n}{sgd} \PY{o}{=} \PY{n}{SGD}\PY{p}{(}\PY{n}{lr}\PY{o}{=}\PY{n}{eta}\PY{p}{)}
|
|
\PY{n}{model}\PY{o}{.}\PY{n}{compile}\PY{p}{(}\PY{n}{loss}\PY{o}{=}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{categorical\PYZus{}crossentropy}\PY{l+s+s1}{\PYZsq{}}\PY{p}{,} \PY{n}{optimizer}\PY{o}{=}\PY{n}{sgd}\PY{p}{,} \PY{n}{metrics}\PY{o}{=}\PY{p}{[}\PY{l+s+s1}{\PYZsq{}}\PY{l+s+s1}{accuracy}\PY{l+s+s1}{\PYZsq{}}\PY{p}{]}\PY{p}{)}
|
|
|
|
\PY{k}{return} \PY{n}{model}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}25}]:} \PY{n}{DNN\PYZus{}keras} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{,} \PY{n}{dtype}\PY{o}{=}\PY{n+nb}{object}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i}\PY{p}{,} \PY{n}{eta} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j}\PY{p}{,} \PY{n}{lmbd} \PY{o+ow}{in} \PY{n+nb}{enumerate}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{DNN} \PY{o}{=} \PY{n}{create\PYZus{}neural\PYZus{}network\PYZus{}keras}\PY{p}{(}\PY{n}{n\PYZus{}neurons\PYZus{}layer1}\PY{p}{,} \PY{n}{n\PYZus{}neurons\PYZus{}layer2}\PY{p}{,} \PY{n}{n\PYZus{}categories}\PY{p}{,}
|
|
\PY{n}{eta}\PY{o}{=}\PY{n}{eta}\PY{p}{,} \PY{n}{lmbd}\PY{o}{=}\PY{n}{lmbd}\PY{p}{)}
|
|
\PY{n}{DNN}\PY{o}{.}\PY{n}{fit}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{,} \PY{n}{epochs}\PY{o}{=}\PY{n}{epochs}\PY{p}{,} \PY{n}{batch\PYZus{}size}\PY{o}{=}\PY{n}{batch\PYZus{}size}\PY{p}{,} \PY{n}{verbose}\PY{o}{=}\PY{l+m+mi}{0}\PY{p}{)}
|
|
\PY{n}{scores} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{evaluate}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{)}
|
|
|
|
\PY{n}{DNN\PYZus{}keras}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}
|
|
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Learning rate = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{eta}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Lambda = }\PY{l+s+s2}{\PYZdq{}}\PY{p}{,} \PY{n}{lmbd}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test accuracy: }\PY{l+s+si}{\PYZpc{}.3f}\PY{l+s+s2}{\PYZdq{}} \PY{o}{\PYZpc{}} \PY{n}{scores}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}\PY{p}{)}
|
|
\PY{n+nb}{print}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
\begin{Verbatim}[commandchars=\\\{\}]
|
|
{\color{incolor}In [{\color{incolor}26}]:} \PY{c+c1}{\PYZsh{} optional}
|
|
\PY{c+c1}{\PYZsh{} visual representation of grid search}
|
|
\PY{c+c1}{\PYZsh{} uses seaborn heatmap, could probably do this in matplotlib}
|
|
\PY{k+kn}{import} \PY{n+nn}{seaborn} \PY{k}{as} \PY{n+nn}{sns}
|
|
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{set}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{train\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{test\PYZus{}accuracy} \PY{o}{=} \PY{n}{np}\PY{o}{.}\PY{n}{zeros}\PY{p}{(}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{,} \PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{)}
|
|
|
|
\PY{k}{for} \PY{n}{i} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{eta\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{k}{for} \PY{n}{j} \PY{o+ow}{in} \PY{n+nb}{range}\PY{p}{(}\PY{n+nb}{len}\PY{p}{(}\PY{n}{lmbd\PYZus{}vals}\PY{p}{)}\PY{p}{)}\PY{p}{:}
|
|
\PY{n}{DNN} \PY{o}{=} \PY{n}{DNN\PYZus{}keras}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]}
|
|
|
|
\PY{n}{train\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{evaluate}\PY{p}{(}\PY{n}{X\PYZus{}train}\PY{p}{,} \PY{n}{Y\PYZus{}train}\PY{p}{)}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
|
|
\PY{n}{test\PYZus{}accuracy}\PY{p}{[}\PY{n}{i}\PY{p}{]}\PY{p}{[}\PY{n}{j}\PY{p}{]} \PY{o}{=} \PY{n}{DNN}\PY{o}{.}\PY{n}{evaluate}\PY{p}{(}\PY{n}{X\PYZus{}test}\PY{p}{,} \PY{n}{Y\PYZus{}test}\PY{p}{)}\PY{p}{[}\PY{l+m+mi}{1}\PY{p}{]}
|
|
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{train\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Training Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
|
|
\PY{n}{fig}\PY{p}{,} \PY{n}{ax} \PY{o}{=} \PY{n}{plt}\PY{o}{.}\PY{n}{subplots}\PY{p}{(}\PY{n}{figsize} \PY{o}{=} \PY{p}{(}\PY{l+m+mi}{10}\PY{p}{,} \PY{l+m+mi}{10}\PY{p}{)}\PY{p}{)}
|
|
\PY{n}{sns}\PY{o}{.}\PY{n}{heatmap}\PY{p}{(}\PY{n}{test\PYZus{}accuracy}\PY{p}{,} \PY{n}{annot}\PY{o}{=}\PY{k+kc}{True}\PY{p}{,} \PY{n}{ax}\PY{o}{=}\PY{n}{ax}\PY{p}{,} \PY{n}{cmap}\PY{o}{=}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{viridis}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}title}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{Test Accuracy}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}ylabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{eta\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{ax}\PY{o}{.}\PY{n}{set\PYZus{}xlabel}\PY{p}{(}\PY{l+s+s2}{\PYZdq{}}\PY{l+s+s2}{\PYZdl{}}\PY{l+s+s2}{\PYZbs{}}\PY{l+s+s2}{lambda\PYZdl{}}\PY{l+s+s2}{\PYZdq{}}\PY{p}{)}
|
|
\PY{n}{plt}\PY{o}{.}\PY{n}{show}\PY{p}{(}\PY{p}{)}
|
|
\end{Verbatim}
|
|
|
|
|
|
|
|
% Add a bibliography block to the postdoc
|
|
|
|
|
|
|
|
\end{document}
|