diff --git a/doc/Articles/cnn/conv_arithmetic.aux b/doc/Articles/cnn/conv_arithmetic.aux new file mode 100644 index 000000000..75bad5595 --- /dev/null +++ b/doc/Articles/cnn/conv_arithmetic.aux @@ -0,0 +1,163 @@ +\relax +\providecommand\zref@newlabel[2]{} +\providecommand\hyper@newdestlabel[2]{} +\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument} +\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined +\global\let\oldcontentsline\contentsline +\gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}} +\global\let\oldnewlabel\newlabel +\gdef\newlabel#1#2{\newlabelxx{#1}#2} +\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}} +\AtEndDocument{\ifx\hyper@anchor\@undefined +\let\contentsline\oldcontentsline +\let\newlabel\oldnewlabel +\fi} +\fi} +\global\let\hyper@last\relax +\gdef\HyperFirstAtBeginDocument#1{#1} +\providecommand\HyField@AuxAddToFields[1]{} +\providecommand\HyField@AuxAddToCoFields[2]{} +\citation{le1997reading} +\citation{krizhevsky2012imagenet} +\citation{zeiler2011adaptive,zeiler2014visualizing,long2015fully,radford2015unsupervised,visin15,im2016generating} +\citation{bergstra2010theano,bastien2012theano} +\citation{collobert2011torch7} +\citation{abaditensorflow} +\citation{jia2014caffe} +\@writefile{toc}{\contentsline {chapter}{\numberline {1}Introduction}{5}{chapter.1}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\citation{Goodfellow-et-al-2016-Book} +\@writefile{toc}{\contentsline {section}{\numberline {1.1}Discrete convolutions}{6}{section.1.1}} +\@writefile{lof}{\contentsline {figure}{\numberline {1.1}{\ignorespaces Computing the output values of a discrete convolution.\relax }}{7}{figure.caption.3}} +\providecommand*\caption@xref[2]{\@setref\relax\@undefined{#1}} +\newlabel{fig:numerical_no_padding_no_strides}{{1.1}{7}{Computing the output values of a discrete convolution.\relax }{figure.caption.3}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {1.2}{\ignorespaces Computing the output values of a discrete convolution for $N = 2$, $i_1 = i_2 = 5$, $k_1 = k_2 = 3$, $s_1 = s_2 = 2$, and $p_1 = p_2 = 1$.\relax }}{7}{figure.caption.4}} +\newlabel{fig:numerical_padding_strides}{{1.2}{7}{Computing the output values of a discrete convolution for $N = 2$, $i_1 = i_2 = 5$, $k_1 = k_2 = 3$, $s_1 = s_2 = 2$, and $p_1 = p_2 = 1$.\relax }{figure.caption.4}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {1.3}{\ignorespaces A convolution mapping from two input feature maps to three output feature maps using a $3 \times 2 \times 3 \times 3$ collection of kernels $\mathbf {w}$. In the left pathway, input feature map 1 is convolved with kernel $\mathbf {w}_{1,1}$ and input feature map 2 is convolved with kernel $\mathbf {w}_{1,2}$, and the results are summed together elementwise to form the first output feature map. The same is repeated for the middle and right pathways to form the second and third feature maps, and all three output feature maps are grouped together to form the output.\relax }}{9}{figure.caption.5}} +\newlabel{fig:full_picture}{{1.3}{9}{A convolution mapping from two input feature maps to three output feature maps using a $3 \times 2 \times 3 \times 3$ collection of kernels $\mathbf {w}$. In the left pathway, input feature map 1 is convolved with kernel $\mathbf {w}_{1,1}$ and input feature map 2 is convolved with kernel $\mathbf {w}_{1,2}$, and the results are summed together elementwise to form the first output feature map. The same is repeated for the middle and right pathways to form the second and third feature maps, and all three output feature maps are grouped together to form the output.\relax }{figure.caption.5}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {1.4}{\ignorespaces An alternative way of viewing strides. Instead of translating the $3 \times 3$ kernel by increments of $s = 2$ (left), the kernel is translated by increments of $1$ and only one in $s = 2$ output elements is retained (right).\relax }}{9}{figure.caption.6}} +\newlabel{fig:strides_subsampling}{{1.4}{9}{An alternative way of viewing strides. Instead of translating the $3 \times 3$ kernel by increments of $s = 2$ (left), the kernel is translated by increments of $1$ and only one in $s = 2$ output elements is retained (right).\relax }{figure.caption.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {1.2}Pooling}{10}{section.1.2}} +\@writefile{lof}{\contentsline {figure}{\numberline {1.5}{\ignorespaces Computing the output values of a $3 \times 3$ average pooling operation on a $5 \times 5$ input using $1 \times 1$ strides.\relax }}{11}{figure.caption.7}} +\newlabel{fig:numerical_average_pooling}{{1.5}{11}{Computing the output values of a $3 \times 3$ average pooling operation on a $5 \times 5$ input using $1 \times 1$ strides.\relax }{figure.caption.7}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {1.6}{\ignorespaces Computing the output values of a $3 \times 3$ max pooling operation on a $5 \times 5$ input using $1 \times 1$ strides.\relax }}{11}{figure.caption.8}} +\newlabel{fig:numerical_max_pooling}{{1.6}{11}{Computing the output values of a $3 \times 3$ max pooling operation on a $5 \times 5$ input using $1 \times 1$ strides.\relax }{figure.caption.8}{}} +\@writefile{toc}{\contentsline {chapter}{\numberline {2}Convolution arithmetic}{12}{chapter.2}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{toc}{\contentsline {section}{\numberline {2.1}No zero padding, unit strides}{12}{section.2.1}} +\newlabel{rel:no_padding_no_strides}{{1}{13}{}{relationship.1}{}} +\pgfsyspdfmark {pgfid4}{9782558}{40989860} +\@writefile{toc}{\contentsline {section}{\numberline {2.2}Zero padding, unit strides}{13}{section.2.2}} +\newlabel{rel:arbitrary_padding_no_strides}{{2}{13}{}{relationship.2}{}} +\pgfsyspdfmark {pgfid5}{9782558}{31382350} +\@writefile{toc}{\contentsline {subsection}{\numberline {2.2.1}Half (same) padding}{13}{subsection.2.2.1}} +\newlabel{rel:same_padding_no_strides}{{3}{13}{}{relationship.3}{}} +\pgfsyspdfmark {pgfid6}{9782558}{18692419} +\@writefile{toc}{\contentsline {subsection}{\numberline {2.2.2}Full padding}{13}{subsection.2.2.2}} +\newlabel{rel:full_padding_no_strides}{{4}{13}{}{relationship.4}{}} +\pgfsyspdfmark {pgfid7}{9782558}{7844659} +\@writefile{lof}{\contentsline {figure}{\numberline {2.1}{\ignorespaces (No padding, unit strides) Convolving a $3 \times 3$ kernel over a $4 \times 4$ input using unit strides (i.e., $i = 4$, $k = 3$, $s = 1$ and $p = 0$).\relax }}{14}{figure.caption.9}} +\newlabel{fig:no_padding_no_strides}{{2.1}{14}{(No padding, unit strides) Convolving a $3 \times 3$ kernel over a $4 \times 4$ input using unit strides (i.e., $i = 4$, $k = 3$, $s = 1$ and $p = 0$).\relax }{figure.caption.9}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {2.2}{\ignorespaces (Arbitrary padding, unit strides) Convolving a $4 \times 4$ kernel over a $5 \times 5$ input padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i = 5$, $k = 4$, $s = 1$ and $p = 2$).\relax }}{14}{figure.caption.10}} +\newlabel{fig:arbitrary_padding_no_strides}{{2.2}{14}{(Arbitrary padding, unit strides) Convolving a $4 \times 4$ kernel over a $5 \times 5$ input padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i = 5$, $k = 4$, $s = 1$ and $p = 2$).\relax }{figure.caption.10}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {2.3}{\ignorespaces (Half padding, unit strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 1$).\relax }}{14}{figure.caption.11}} +\newlabel{fig:same_padding_no_strides}{{2.3}{14}{(Half padding, unit strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 1$).\relax }{figure.caption.11}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {2.4}{\ignorespaces (Full padding, unit strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using full padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 2$).\relax }}{14}{figure.caption.12}} +\newlabel{fig:full_padding_no_strides}{{2.4}{14}{(Full padding, unit strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using full padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 2$).\relax }{figure.caption.12}{}} +\@writefile{toc}{\contentsline {section}{\numberline {2.3}No zero padding, non-unit strides}{15}{section.2.3}} +\newlabel{rel:no_padding_strides}{{5}{15}{}{relationship.5}{}} +\pgfsyspdfmark {pgfid8}{9782558}{25170765} +\@writefile{toc}{\contentsline {section}{\numberline {2.4}Zero padding, non-unit strides}{15}{section.2.4}} +\newlabel{rel:padding_strides}{{6}{15}{}{relationship.6}{}} +\pgfsyspdfmark {pgfid9}{9782558}{12981054} +\@writefile{lof}{\contentsline {figure}{\numberline {2.5}{\ignorespaces (No zero padding, arbitrary strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 0$).\relax }}{17}{figure.caption.13}} +\newlabel{fig:no_padding_strides}{{2.5}{17}{(No zero padding, arbitrary strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 0$).\relax }{figure.caption.13}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {2.6}{\ignorespaces (Arbitrary padding and strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 1$).\relax }}{17}{figure.caption.14}} +\newlabel{fig:padding_strides}{{2.6}{17}{(Arbitrary padding and strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 1$).\relax }{figure.caption.14}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {2.7}{\ignorespaces (Arbitrary padding and strides) Convolving a $3 \times 3$ kernel over a $6 \times 6$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 6$, $k = 3$, $s = 2$ and $p = 1$). In this case, the bottom row and right column of the zero padded input are not covered by the kernel.\relax }}{17}{figure.caption.15}} +\newlabel{fig:padding_strides_odd}{{2.7}{17}{(Arbitrary padding and strides) Convolving a $3 \times 3$ kernel over a $6 \times 6$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 6$, $k = 3$, $s = 2$ and $p = 1$). In this case, the bottom row and right column of the zero padded input are not covered by the kernel.\relax }{figure.caption.15}{}} +\newlabel{fig:no_padding_no_strides_explained}{{2.8a}{17}{The kernel has to slide two steps to the right to touch the right side of the input (and equivalently downwards). Adding one to account for the initial kernel position, the output size is $3 \times 3$.\relax }{figure.caption.16}{}} +\newlabel{sub@fig:no_padding_no_strides_explained}{{a}{17}{The kernel has to slide two steps to the right to touch the right side of the input (and equivalently downwards). Adding one to account for the initial kernel position, the output size is $3 \times 3$.\relax }{figure.caption.16}{}} +\newlabel{fig:no_padding_strides_explained}{{2.8b}{17}{The kernel has to slide one step of size two to the right to touch the right side of the input (and equivalently downwards). Adding one to account for the initial kernel position, the output size is $2 \times 2$.\relax }{figure.caption.16}{}} +\newlabel{sub@fig:no_padding_strides_explained}{{b}{17}{The kernel has to slide one step of size two to the right to touch the right side of the input (and equivalently downwards). Adding one to account for the initial kernel position, the output size is $2 \times 2$.\relax }{figure.caption.16}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {2.8}{\ignorespaces Counting kernel positions.\relax }}{17}{figure.caption.16}} +\citation{boureau-cvpr-10,boureau-icml-10,boureau-iccv-11,ICML2011Saxe_551} +\@writefile{toc}{\contentsline {chapter}{\numberline {3}Pooling arithmetic}{18}{chapter.3}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\newlabel{rel:pooling}{{7}{18}{}{relationship.7}{}} +\pgfsyspdfmark {pgfid12}{9782558}{18616645} +\@writefile{toc}{\contentsline {chapter}{\numberline {4}Transposed convolution arithmetic}{19}{chapter.4}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{toc}{\contentsline {section}{\numberline {4.1}Convolution as a matrix operation}{20}{section.4.1}} +\@writefile{toc}{\contentsline {section}{\numberline {4.2}Transposed convolution}{20}{section.4.2}} +\@writefile{toc}{\contentsline {section}{\numberline {4.3}No zero padding, unit strides, transposed}{21}{section.4.3}} +\newlabel{rel:no_padding_no_strides_transposed}{{8}{22}{}{relationship.8}{}} +\pgfsyspdfmark {pgfid13}{9782558}{39416996} +\@writefile{toc}{\contentsline {section}{\numberline {4.4}Zero padding, unit strides, transposed}{22}{section.4.4}} +\newlabel{rel:arbitrary_padding_no_strides_transposed}{{9}{22}{}{relationship.9}{}} +\pgfsyspdfmark {pgfid14}{9782558}{25985771} +\@writefile{toc}{\contentsline {subsection}{\numberline {4.4.1}Half (same) padding, transposed}{22}{subsection.4.4.1}} +\newlabel{rel:half_padding_no_strides_transposed}{{10}{22}{}{relationship.10}{}} +\pgfsyspdfmark {pgfid15}{9782558}{13476983} +\@writefile{toc}{\contentsline {subsection}{\numberline {4.4.2}Full padding, transposed}{22}{subsection.4.4.2}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.1}{\ignorespaces The transpose of convolving a $3 \times 3$ kernel over a $4 \times 4$ input using unit strides (i.e., $i = 4$, $k = 3$, $s = 1$ and $p = 0$). It is equivalent to convolving a $3 \times 3$ kernel over a $2 \times 2$ input padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i' = 2$, $k' = k$, $s' = 1$ and $p' = 2$).\relax }}{23}{figure.caption.17}} +\newlabel{fig:no_padding_no_strides_transposed}{{4.1}{23}{The transpose of convolving a $3 \times 3$ kernel over a $4 \times 4$ input using unit strides (i.e., $i = 4$, $k = 3$, $s = 1$ and $p = 0$). It is equivalent to convolving a $3 \times 3$ kernel over a $2 \times 2$ input padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i' = 2$, $k' = k$, $s' = 1$ and $p' = 2$).\relax }{figure.caption.17}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.2}{\ignorespaces The transpose of convolving a $4 \times 4$ kernel over a $5 \times 5$ input padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i = 5$, $k = 4$, $s = 1$ and $p = 2$). It is equivalent to convolving a $4 \times 4$ kernel over a $6 \times 6$ input padded with a $1 \times 1$ border of zeros using unit strides (i.e., $i' = 6$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }}{23}{figure.caption.18}} +\newlabel{fig:arbitrary_padding_no_strides_transposed}{{4.2}{23}{The transpose of convolving a $4 \times 4$ kernel over a $5 \times 5$ input padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i = 5$, $k = 4$, $s = 1$ and $p = 2$). It is equivalent to convolving a $4 \times 4$ kernel over a $6 \times 6$ input padded with a $1 \times 1$ border of zeros using unit strides (i.e., $i' = 6$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }{figure.caption.18}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.3}{\ignorespaces The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 1$). It is equivalent to convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half padding and unit strides (i.e., $i' = 5$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }}{23}{figure.caption.19}} +\newlabel{fig:same_padding_no_strides_transposed}{{4.3}{23}{The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 1$). It is equivalent to convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half padding and unit strides (i.e., $i' = 5$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }{figure.caption.19}{}} +\newlabel{rel:full_padding_no_strides_transposed}{{11}{24}{}{relationship.11}{}} +\pgfsyspdfmark {pgfid16}{9782558}{38404829} +\@writefile{toc}{\contentsline {section}{\numberline {4.5}No zero padding, non-unit strides, transposed}{24}{section.4.5}} +\newlabel{rel:no_padding_strides_transposed}{{12}{24}{}{relationship.12}{}} +\pgfsyspdfmark {pgfid17}{9782558}{17976964} +\@writefile{toc}{\contentsline {section}{\numberline {4.6}Zero padding, non-unit strides, transposed}{24}{section.4.6}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.4}{\ignorespaces The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input using full padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 2$). It is equivalent to convolving a $3 \times 3$ kernel over a $7 \times 7$ input using unit strides (i.e., $i' = 7$, $k' = k$, $s' = 1$ and $p' = 0$).\relax }}{25}{figure.caption.20}} +\newlabel{fig:full_padding_no_strides_transposed}{{4.4}{25}{The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input using full padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 2$). It is equivalent to convolving a $3 \times 3$ kernel over a $7 \times 7$ input using unit strides (i.e., $i' = 7$, $k' = k$, $s' = 1$ and $p' = 0$).\relax }{figure.caption.20}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.5}{\ignorespaces The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 0$). It is equivalent to convolving a $3 \times 3$ kernel over a $2 \times 2$ input (with $1$ zero inserted between inputs) padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i' = 2$, $\mathaccentV {tilde}07E{i}' = 3$, $k' = k$, $s' = 1$ and $p' = 2$).\relax }}{25}{figure.caption.21}} +\newlabel{fig:no_padding_strides_transposed}{{4.5}{25}{The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 0$). It is equivalent to convolving a $3 \times 3$ kernel over a $2 \times 2$ input (with $1$ zero inserted between inputs) padded with a $2 \times 2$ border of zeros using unit strides (i.e., $i' = 2$, $\tilde {i}' = 3$, $k' = k$, $s' = 1$ and $p' = 2$).\relax }{figure.caption.21}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {4.6}{\ignorespaces The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 1$). It is equivalent to convolving a $3 \times 3$ kernel over a $3 \times 3$ input (with $1$ zero inserted between inputs) padded with a $1 \times 1$ border of zeros using unit strides (i.e., $i' = 3$, $\mathaccentV {tilde}07E{i}' = 5$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }}{25}{figure.caption.22}} +\newlabel{fig:padding_strides_transposed}{{4.6}{25}{The transpose of convolving a $3 \times 3$ kernel over a $5 \times 5$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 1$). It is equivalent to convolving a $3 \times 3$ kernel over a $3 \times 3$ input (with $1$ zero inserted between inputs) padded with a $1 \times 1$ border of zeros using unit strides (i.e., $i' = 3$, $\tilde {i}' = 5$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }{figure.caption.22}{}} +\newlabel{rel:padding_strides_transposed}{{13}{26}{}{relationship.13}{}} +\pgfsyspdfmark {pgfid18}{9782558}{37677014} +\newlabel{rel:padding_strides_transposed_odd}{{14}{26}{}{relationship.14}{}} +\pgfsyspdfmark {pgfid19}{9782558}{26483833} +\@writefile{lof}{\contentsline {figure}{\numberline {4.7}{\ignorespaces The transpose of convolving a $3 \times 3$ kernel over a $6 \times 6$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 6$, $k = 3$, $s = 2$ and $p = 1$). It is equivalent to convolving a $3 \times 3$ kernel over a $2 \times 2$ input (with $1$ zero inserted between inputs) padded with a $1 \times 1$ border of zeros (with an additional border of size $1$ added to the bottom and right edges) using unit strides (i.e., $i' = 3$, $\mathaccentV {tilde}07E{i}' = 5$, $a = 1$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }}{27}{figure.caption.23}} +\newlabel{fig:padding_strides_odd_transposed}{{4.7}{27}{The transpose of convolving a $3 \times 3$ kernel over a $6 \times 6$ input padded with a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 6$, $k = 3$, $s = 2$ and $p = 1$). It is equivalent to convolving a $3 \times 3$ kernel over a $2 \times 2$ input (with $1$ zero inserted between inputs) padded with a $1 \times 1$ border of zeros (with an additional border of size $1$ added to the bottom and right edges) using unit strides (i.e., $i' = 3$, $\tilde {i}' = 5$, $a = 1$, $k' = k$, $s' = 1$ and $p' = 1$).\relax }{figure.caption.23}{}} +\citation{chen2014semantic,yu2015multi} +\citation{oord2016wavenet} +\bibdata{bibliography} +\@writefile{toc}{\contentsline {chapter}{\numberline {5}Miscellaneous convolutions}{28}{chapter.5}} +\@writefile{lof}{\addvspace {10\p@ }} +\@writefile{lot}{\addvspace {10\p@ }} +\@writefile{toc}{\contentsline {section}{\numberline {5.1}Dilated convolutions}{28}{section.5.1}} +\newlabel{rel:dilation}{{15}{28}{}{relationship.15}{}} +\pgfsyspdfmark {pgfid20}{9782558}{7844659} +\@writefile{lof}{\contentsline {figure}{\numberline {5.1}{\ignorespaces (Dilated convolution) Convolving a $3 \times 3$ kernel over a $7 \times 7$ input with a dilation factor of 2 (i.e., $i = 7$, $k = 3$, $d = 2$, $s = 1$ and $p = 0$).\relax }}{29}{figure.caption.24}} +\newlabel{fig:dilation}{{5.1}{29}{(Dilated convolution) Convolving a $3 \times 3$ kernel over a $7 \times 7$ input with a dilation factor of 2 (i.e., $i = 7$, $k = 3$, $d = 2$, $s = 1$ and $p = 0$).\relax }{figure.caption.24}{}} +\bibcite{abaditensorflow}{{1}{2015}{{Abadi {\em et~al.}}}{{Abadi, Agarwal, Barham, Brevdo, Chen, Citro, Corrado, Davis, Dean, Devin, {\em et~al.}}}} +\bibcite{bastien2012theano}{{2}{2012}{{Bastien {\em et~al.}}}{{Bastien, Lamblin, Pascanu, Bergstra, Goodfellow, Bergeron, Bouchard, Warde-Farley, and Bengio}}} +\bibcite{bergstra2010theano}{{3}{2010}{{Bergstra {\em et~al.}}}{{Bergstra, Breuleux, Bastien, Lamblin, Pascanu, Desjardins, Turian, Warde-Farley, and Bengio}}} +\bibcite{boureau-cvpr-10}{{4}{2010a}{{Boureau {\em et~al.}}}{{Boureau, Bach, LeCun, and Ponce}}} +\bibcite{boureau-icml-10}{{5}{2010b}{{Boureau {\em et~al.}}}{{Boureau, Ponce, and LeCun}}} +\bibcite{boureau-iccv-11}{{6}{2011}{{Boureau {\em et~al.}}}{{Boureau, {Le Roux}, Bach, Ponce, and LeCun}}} +\bibcite{chen2014semantic}{{7}{2014}{{Chen {\em et~al.}}}{{Chen, Papandreou, Kokkinos, Murphy, and Yuille}}} +\bibcite{collobert2011torch7}{{8}{2011}{{Collobert {\em et~al.}}}{{Collobert, Kavukcuoglu, and Farabet}}} +\bibcite{Goodfellow-et-al-2016-Book}{{9}{2016}{{Goodfellow {\em et~al.}}}{{Goodfellow, Bengio, and Courville}}} +\bibcite{im2016generating}{{10}{2016}{{Im {\em et~al.}}}{{Im, Kim, Jiang, and Memisevic}}} +\bibcite{jia2014caffe}{{11}{2014}{{Jia {\em et~al.}}}{{Jia, Shelhamer, Donahue, Karayev, Long, Girshick, Guadarrama, and Darrell}}} +\bibcite{krizhevsky2012imagenet}{{12}{2012}{{Krizhevsky {\em et~al.}}}{{Krizhevsky, Sutskever, and Hinton}}} +\bibcite{le1997reading}{{13}{1997}{{Le~Cun {\em et~al.}}}{{Le~Cun, Bottou, and Bengio}}} +\bibcite{long2015fully}{{14}{2015}{{Long {\em et~al.}}}{{Long, Shelhamer, and Darrell}}} +\bibcite{oord2016wavenet}{{15}{2016}{{Oord {\em et~al.}}}{{Oord, Dieleman, Zen, Simonyan, Vinyals, Graves, Kalchbrenner, Senior, and Kavukcuoglu}}} +\bibcite{radford2015unsupervised}{{16}{2015}{{Radford {\em et~al.}}}{{Radford, Metz, and Chintala}}} +\bibcite{ICML2011Saxe_551}{{17}{2011}{{Saxe {\em et~al.}}}{{Saxe, Koh, Chen, Bhand, Suresh, and Ng}}} +\bibcite{visin15}{{18}{2015}{{Visin {\em et~al.}}}{{Visin, Kastner, Courville, Bengio, Matteucci, and Cho}}} +\bibcite{yu2015multi}{{19}{2015}{{Yu and Koltun}}{{Yu and Koltun}}} +\bibcite{zeiler2014visualizing}{{20}{2014}{{Zeiler and Fergus}}{{Zeiler and Fergus}}} +\bibcite{zeiler2011adaptive}{{21}{2011}{{Zeiler {\em et~al.}}}{{Zeiler, Taylor, and Fergus}}} +\bibstyle{natbib} diff --git a/doc/Articles/cnn/conv_arithmetic.log b/doc/Articles/cnn/conv_arithmetic.log new file mode 100644 index 000000000..1645a2a15 --- /dev/null +++ b/doc/Articles/cnn/conv_arithmetic.log @@ -0,0 +1,2236 @@ +This is pdfTeX, Version 3.14159265-2.6-1.40.16 (TeX Live 2015) (preloaded format=pdflatex 2015.5.24) 8 OCT 2020 22:22 +entering extended mode + restricted \write18 enabled. + %&-line parsing enabled. +**conv_arithmetic.tex +(./conv_arithmetic.tex +LaTeX2e <2015/01/01> +Babel <3.9l> and hyphenation patterns for 79 languages loaded. +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/report.cls +Document Class: report 2014/09/29 v1.4h Standard LaTeX document class +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/size10.clo +File: size10.clo 2014/09/29 v1.4h Standard LaTeX file (size option) +) +\c@part=\count79 +\c@chapter=\count80 +\c@section=\count81 +\c@subsection=\count82 +\c@subsubsection=\count83 +\c@paragraph=\count84 +\c@subparagraph=\count85 +\c@figure=\count86 +\c@table=\count87 +\abovecaptionskip=\skip41 +\belowcaptionskip=\skip42 +\bibindent=\dimen102 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsmath/amsmath.sty +Package: amsmath 2013/01/14 v2.14 AMS math features +\@mathmargin=\skip43 + +For additional information on amsmath, use the `?' option. +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsmath/amstext.sty +Package: amstext 2000/06/29 v2.01 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsmath/amsgen.sty +File: amsgen.sty 1999/11/30 v2.0 +\@emptytoks=\toks14 +\ex@=\dimen103 +)) +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsmath/amsbsy.sty +Package: amsbsy 1999/11/29 v1.2d +\pmbraise@=\dimen104 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsmath/amsopn.sty +Package: amsopn 1999/12/14 v2.01 operator names +) +\inf@bad=\count88 +LaTeX Info: Redefining \frac on input line 210. +\uproot@=\count89 +\leftroot@=\count90 +LaTeX Info: Redefining \overline on input line 306. +\classnum@=\count91 +\DOTSCASE@=\count92 +LaTeX Info: Redefining \ldots on input line 378. +LaTeX Info: Redefining \dots on input line 381. +LaTeX Info: Redefining \cdots on input line 466. +\Mathstrutbox@=\box26 +\strutbox@=\box27 +\big@size=\dimen105 +LaTeX Font Info: Redeclaring font encoding OML on input line 566. +LaTeX Font Info: Redeclaring font encoding OMS on input line 567. +\macc@depth=\count93 +\c@MaxMatrixCols=\count94 +\dotsspace@=\muskip10 +\c@parentequation=\count95 +\dspbrk@lvl=\count96 +\tag@help=\toks15 +\row@=\count97 +\column@=\count98 +\maxfields@=\count99 +\andhelp@=\toks16 +\eqnshift@=\dimen106 +\alignsep@=\dimen107 +\tagshift@=\dimen108 +\tagwidth@=\dimen109 +\totwidth@=\dimen110 +\lineht@=\dimen111 +\@envbody=\toks17 +\multlinegap=\skip44 +\multlinetaggap=\skip45 +\mathdisplay@stack=\toks18 +LaTeX Info: Redefining \[ on input line 2665. +LaTeX Info: Redefining \] on input line 2666. +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsfonts/amsfonts.sty +Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support +\symAMSa=\mathgroup4 +\symAMSb=\mathgroup5 +LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' +(Font) U/euf/m/n --> U/euf/b/n on input line 106. +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/amscls/amsthm.sty +Package: amsthm 2015/03/04 v2.20.2 +\thm@style=\toks19 +\thm@bodyfont=\toks20 +\thm@headfont=\toks21 +\thm@notefont=\toks22 +\thm@headpunct=\toks23 +\thm@preskip=\skip46 +\thm@postskip=\skip47 +\thm@headsep=\skip48 +\dth@everypar=\toks24 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsfonts/amssymb.sty +Package: amssymb 2013/01/14 v3.01 AMS font symbols +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/preprint/authblk.sty +Package: authblk 2001/02/27 1.3 (PWD) +\affilsep=\skip49 +\@affilsep=\skip50 +\c@Maxaffil=\count100 +\c@authors=\count101 +\c@affil=\count102 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/fontenc.sty +Package: fontenc 2005/09/27 v1.99g Standard LaTeX package + +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/t1enc.def +File: t1enc.def 2005/09/27 v1.99g Standard LaTeX file +LaTeX Font Info: Redeclaring font encoding T1 on input line 48. +)) +(/usr/local/texlive/2015/texmf-dist/tex/latex/graphics/graphicx.sty +Package: graphicx 2014/10/28 v1.0g Enhanced LaTeX Graphics (DPC,SPQR) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/graphics/keyval.sty +Package: keyval 2014/10/28 v1.15 key=value parser (DPC) +\KV@toks@=\toks25 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/graphics/graphics.sty +Package: graphics 2014/10/28 v1.0p Standard LaTeX Graphics (DPC,SPQR) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/graphics/trig.sty +Package: trig 1999/03/16 v1.09 sin cos tan (DPC) +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/latexconfig/graphics.cfg +File: graphics.cfg 2010/04/23 v1.9 graphics configuration of TeX Live +) +Package graphics Info: Driver file: pdftex.def on input line 94. + +(/usr/local/texlive/2015/texmf-dist/tex/latex/pdftex-def/pdftex.def +File: pdftex.def 2011/05/27 v0.06d Graphics/color for pdfTeX + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/infwarerr.sty +Package: infwarerr 2010/04/08 v1.3 Providing info/warning/error messages (HO) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/ltxcmds.sty +Package: ltxcmds 2011/11/09 v1.22 LaTeX kernel commands for general use (HO) +) +\Gread@gobject=\count103 +)) +\Gin@req@height=\dimen112 +\Gin@req@width=\dimen113 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/inputenc.sty +Package: inputenc 2015/03/17 v1.2c Input encoding file +\inpenc@prehook=\toks26 +\inpenc@posthook=\toks27 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/utf8.def +File: utf8.def 2014/09/29 v1.1m UTF-8 support for inputenc +Now handling font encoding OML ... +... no UTF-8 mapping file for font encoding OML +Now handling font encoding T1 ... +... processing UTF-8 mapping file for font encoding T1 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/t1enc.dfu +File: t1enc.dfu 2014/09/29 v1.1m UTF-8 support for inputenc + defining Unicode char U+00A1 (decimal 161) + defining Unicode char U+00A3 (decimal 163) + defining Unicode char U+00AB (decimal 171) + defining Unicode char U+00BB (decimal 187) + defining Unicode char U+00BF (decimal 191) + defining Unicode char U+00C0 (decimal 192) + defining Unicode char U+00C1 (decimal 193) + defining Unicode char U+00C2 (decimal 194) + defining Unicode char U+00C3 (decimal 195) + defining Unicode char U+00C4 (decimal 196) + defining Unicode char U+00C5 (decimal 197) + defining Unicode char U+00C6 (decimal 198) + defining Unicode char U+00C7 (decimal 199) + defining Unicode char U+00C8 (decimal 200) + defining Unicode char U+00C9 (decimal 201) + defining Unicode char U+00CA (decimal 202) + defining Unicode char U+00CB (decimal 203) + defining Unicode char U+00CC (decimal 204) + defining Unicode char U+00CD (decimal 205) + defining Unicode char U+00CE (decimal 206) + defining Unicode char U+00CF (decimal 207) + defining Unicode char U+00D0 (decimal 208) + defining Unicode char U+00D1 (decimal 209) + defining Unicode char U+00D2 (decimal 210) + defining Unicode char U+00D3 (decimal 211) + defining Unicode char U+00D4 (decimal 212) + defining Unicode char U+00D5 (decimal 213) + defining Unicode char U+00D6 (decimal 214) + defining Unicode char U+00D8 (decimal 216) + defining Unicode char U+00D9 (decimal 217) + defining Unicode char U+00DA (decimal 218) + defining Unicode char U+00DB (decimal 219) + defining Unicode char U+00DC (decimal 220) + defining Unicode char U+00DD (decimal 221) + defining Unicode char U+00DE (decimal 222) + defining Unicode char U+00DF (decimal 223) + defining Unicode char U+00E0 (decimal 224) + defining Unicode char U+00E1 (decimal 225) + defining Unicode char U+00E2 (decimal 226) + defining Unicode char U+00E3 (decimal 227) + defining Unicode char U+00E4 (decimal 228) + defining Unicode char U+00E5 (decimal 229) + defining Unicode char U+00E6 (decimal 230) + defining Unicode char U+00E7 (decimal 231) + defining Unicode char U+00E8 (decimal 232) + defining Unicode char U+00E9 (decimal 233) + defining Unicode char U+00EA (decimal 234) + defining Unicode char U+00EB (decimal 235) + defining Unicode char U+00EC (decimal 236) + defining Unicode char U+00ED (decimal 237) + defining Unicode char U+00EE (decimal 238) + defining Unicode char U+00EF (decimal 239) + defining Unicode char U+00F0 (decimal 240) + defining Unicode char U+00F1 (decimal 241) + defining Unicode char U+00F2 (decimal 242) + defining Unicode char U+00F3 (decimal 243) + defining Unicode char U+00F4 (decimal 244) + defining Unicode char U+00F5 (decimal 245) + defining Unicode char U+00F6 (decimal 246) + defining Unicode char U+00F8 (decimal 248) + defining Unicode char U+00F9 (decimal 249) + defining Unicode char U+00FA (decimal 250) + defining Unicode char U+00FB (decimal 251) + defining Unicode char U+00FC (decimal 252) + defining Unicode char U+00FD (decimal 253) + defining Unicode char U+00FE (decimal 254) + defining Unicode char U+00FF (decimal 255) + defining Unicode char U+0102 (decimal 258) + defining Unicode char U+0103 (decimal 259) + defining Unicode char U+0104 (decimal 260) + defining Unicode char U+0105 (decimal 261) + defining Unicode char U+0106 (decimal 262) + defining Unicode char U+0107 (decimal 263) + defining Unicode char U+010C (decimal 268) + defining Unicode char U+010D (decimal 269) + defining Unicode char U+010E (decimal 270) + defining Unicode char U+010F (decimal 271) + defining Unicode char U+0110 (decimal 272) + defining Unicode char U+0111 (decimal 273) + defining Unicode char U+0118 (decimal 280) + defining Unicode char U+0119 (decimal 281) + defining Unicode char U+011A (decimal 282) + defining Unicode char U+011B (decimal 283) + defining Unicode char U+011E (decimal 286) + defining Unicode char U+011F (decimal 287) + defining Unicode char U+0130 (decimal 304) + defining Unicode char U+0131 (decimal 305) + defining Unicode char U+0132 (decimal 306) + defining Unicode char U+0133 (decimal 307) + defining Unicode char U+0139 (decimal 313) + defining Unicode char U+013A (decimal 314) + defining Unicode char U+013D (decimal 317) + defining Unicode char U+013E (decimal 318) + defining Unicode char U+0141 (decimal 321) + defining Unicode char U+0142 (decimal 322) + defining Unicode char U+0143 (decimal 323) + defining Unicode char U+0144 (decimal 324) + defining Unicode char U+0147 (decimal 327) + defining Unicode char U+0148 (decimal 328) + defining Unicode char U+014A (decimal 330) + defining Unicode char U+014B (decimal 331) + defining Unicode char U+0150 (decimal 336) + defining Unicode char U+0151 (decimal 337) + defining Unicode char U+0152 (decimal 338) + defining Unicode char U+0153 (decimal 339) + defining Unicode char U+0154 (decimal 340) + defining Unicode char U+0155 (decimal 341) + defining Unicode char U+0158 (decimal 344) + defining Unicode char U+0159 (decimal 345) + defining Unicode char U+015A (decimal 346) + defining Unicode char U+015B (decimal 347) + defining Unicode char U+015E (decimal 350) + defining Unicode char U+015F (decimal 351) + defining Unicode char U+0160 (decimal 352) + defining Unicode char U+0161 (decimal 353) + defining Unicode char U+0162 (decimal 354) + defining Unicode char U+0163 (decimal 355) + defining Unicode char U+0164 (decimal 356) + defining Unicode char U+0165 (decimal 357) + defining Unicode char U+016E (decimal 366) + defining Unicode char U+016F (decimal 367) + defining Unicode char U+0170 (decimal 368) + defining Unicode char U+0171 (decimal 369) + defining Unicode char U+0178 (decimal 376) + defining Unicode char U+0179 (decimal 377) + defining Unicode char U+017A (decimal 378) + defining Unicode char U+017B (decimal 379) + defining Unicode char U+017C (decimal 380) + defining Unicode char U+017D (decimal 381) + defining Unicode char U+017E (decimal 382) + defining Unicode char U+200C (decimal 8204) + defining Unicode char U+2013 (decimal 8211) + defining Unicode char U+2014 (decimal 8212) + defining Unicode char U+2018 (decimal 8216) + defining Unicode char U+2019 (decimal 8217) + defining Unicode char U+201A (decimal 8218) + defining Unicode char U+201C (decimal 8220) + defining Unicode char U+201D (decimal 8221) + defining Unicode char U+201E (decimal 8222) + defining Unicode char U+2030 (decimal 8240) + defining Unicode char U+2031 (decimal 8241) + defining Unicode char U+2039 (decimal 8249) + defining Unicode char U+203A (decimal 8250) + defining Unicode char U+2423 (decimal 9251) +) +Now handling font encoding OT1 ... +... processing UTF-8 mapping file for font encoding OT1 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/ot1enc.dfu +File: ot1enc.dfu 2014/09/29 v1.1m UTF-8 support for inputenc + defining Unicode char U+00A1 (decimal 161) + defining Unicode char U+00A3 (decimal 163) + defining Unicode char U+00B8 (decimal 184) + defining Unicode char U+00BF (decimal 191) + defining Unicode char U+00C5 (decimal 197) + defining Unicode char U+00C6 (decimal 198) + defining Unicode char U+00D8 (decimal 216) + defining Unicode char U+00DF (decimal 223) + defining Unicode char U+00E6 (decimal 230) + defining Unicode char U+00EC (decimal 236) + defining Unicode char U+00ED (decimal 237) + defining Unicode char U+00EE (decimal 238) + defining Unicode char U+00EF (decimal 239) + defining Unicode char U+00F8 (decimal 248) + defining Unicode char U+0131 (decimal 305) + defining Unicode char U+0141 (decimal 321) + defining Unicode char U+0142 (decimal 322) + defining Unicode char U+0152 (decimal 338) + defining Unicode char U+0153 (decimal 339) + defining Unicode char U+2013 (decimal 8211) + defining Unicode char U+2014 (decimal 8212) + defining Unicode char U+2018 (decimal 8216) + defining Unicode char U+2019 (decimal 8217) + defining Unicode char U+201C (decimal 8220) + defining Unicode char U+201D (decimal 8221) +) +Now handling font encoding OMS ... +... processing UTF-8 mapping file for font encoding OMS + +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/omsenc.dfu +File: omsenc.dfu 2014/09/29 v1.1m UTF-8 support for inputenc + defining Unicode char U+00A7 (decimal 167) + defining Unicode char U+00B6 (decimal 182) + defining Unicode char U+00B7 (decimal 183) + defining Unicode char U+2020 (decimal 8224) + defining Unicode char U+2021 (decimal 8225) + defining Unicode char U+2022 (decimal 8226) +) +Now handling font encoding OMX ... +... no UTF-8 mapping file for font encoding OMX +Now handling font encoding U ... +... no UTF-8 mapping file for font encoding U + defining Unicode char U+00A9 (decimal 169) + defining Unicode char U+00AA (decimal 170) + defining Unicode char U+00AE (decimal 174) + defining Unicode char U+00BA (decimal 186) + defining Unicode char U+02C6 (decimal 710) + defining Unicode char U+02DC (decimal 732) + defining Unicode char U+200C (decimal 8204) + defining Unicode char U+2026 (decimal 8230) + defining Unicode char U+2122 (decimal 8482) + defining Unicode char U+2423 (decimal 9251) +)) +(/usr/local/texlive/2015/texmf-dist/tex/latex/mdframed/mdframed.sty +Package: mdframed 2013/07/01 1.9b: mdframed + +(/usr/local/texlive/2015/texmf-dist/tex/latex/oberdiek/kvoptions.sty +Package: kvoptions 2011/06/30 v3.11 Key value format for package options (HO) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/kvsetkeys.sty +Package: kvsetkeys 2012/04/25 v1.16 Key value parser (HO) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/etexcmds.sty +Package: etexcmds 2011/02/16 v1.5 Avoid name clashes with e-TeX commands (HO) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/ifluatex.sty +Package: ifluatex 2010/03/01 v1.3 Provides the ifluatex switch (HO) +Package ifluatex Info: LuaTeX not detected. +) +Package etexcmds Info: Could not find \expanded. +(etexcmds) That can mean that you are not using pdfTeX 1.50 or +(etexcmds) that some package has redefined \expanded. +(etexcmds) In the latter case, load this package earlier. +))) +(/usr/local/texlive/2015/texmf-dist/tex/latex/l3packages/xparse/xparse.sty +(/usr/local/texlive/2015/texmf-dist/tex/latex/l3kernel/expl3.sty +Package: expl3 2015/03/01 v5547 L3 programming layer (loader) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/l3kernel/expl3-code.tex +Package: expl3 2015/03/01 v5547 L3 programming layer (code) +L3 Module: l3bootstrap 2015/02/28 v5542 L3 Bootstrap code +L3 Module: l3names 2015/02/24 v5535 L3 Namespace for primitives +L3 Module: l3basics 2015/01/27 v5500 L3 Basic definitions +L3 Module: l3expan 2014/11/27 v5472 L3 Argument expansion +L3 Module: l3tl 2015/01/27 v5500 L3 Token lists +L3 Module: l3str 2015/03/01 v5545 L3 Strings +L3 Module: l3seq 2014/08/23 v5354 L3 Sequences and stacks +L3 Module: l3int 2015/02/21 v5529 L3 Integers +\c_max_int=\count104 +\l_tmpa_int=\count105 +\l_tmpb_int=\count106 +\g_tmpa_int=\count107 +\g_tmpb_int=\count108 +L3 Module: l3quark 2014/08/23 v5354 L3 Quarks +L3 Module: l3prg 2014/08/23 v5354 L3 Control structures +\g__prg_map_int=\count109 +L3 Module: l3clist 2014/08/23 v5354 L3 Comma separated lists +L3 Module: l3token 2014/09/15 v5422 L3 Experimental token manipulation +L3 Module: l3prop 2014/08/23 v5354 L3 Property lists +L3 Module: l3msg 2015/02/26 v5537 L3 Messages +L3 Module: l3file 2014/08/24 v5369 L3 File and I/O operations +\l_iow_line_count_int=\count110 +\l__iow_target_count_int=\count111 +\l__iow_current_line_int=\count112 +\l__iow_current_word_int=\count113 +\l__iow_current_indentation_int=\count114 +L3 Module: l3skip 2014/08/23 v5354 L3 Dimensions and skips +\c_zero_dim=\dimen114 +\c_max_dim=\dimen115 +\l_tmpa_dim=\dimen116 +\l_tmpb_dim=\dimen117 +\g_tmpa_dim=\dimen118 +\g_tmpb_dim=\dimen119 +\c_zero_skip=\skip51 +\c_max_skip=\skip52 +\l_tmpa_skip=\skip53 +\l_tmpb_skip=\skip54 +\g_tmpa_skip=\skip55 +\g_tmpb_skip=\skip56 +\c_zero_muskip=\muskip11 +\c_max_muskip=\muskip12 +\l_tmpa_muskip=\muskip13 +\l_tmpb_muskip=\muskip14 +\g_tmpa_muskip=\muskip15 +\g_tmpb_muskip=\muskip16 +L3 Module: l3keys 2015/01/27 v5500 L3 Key-value interfaces +\g__keyval_level_int=\count115 +\l_keys_choice_int=\count116 +L3 Module: l3fp 2014/08/22 v5336 L3 Floating points +\c__fp_leading_shift_int=\count117 +\c__fp_middle_shift_int=\count118 +\c__fp_trailing_shift_int=\count119 +\c__fp_big_leading_shift_int=\count120 +\c__fp_big_middle_shift_int=\count121 +\c__fp_big_trailing_shift_int=\count122 +\c__fp_Bigg_leading_shift_int=\count123 +\c__fp_Bigg_middle_shift_int=\count124 +\c__fp_Bigg_trailing_shift_int=\count125 +L3 Module: l3box 2014/08/23 v5354 L3 Experimental boxes +\c_empty_box=\box28 +\l_tmpa_box=\box29 +\l_tmpb_box=\box30 +\g_tmpa_box=\box31 +\g_tmpb_box=\box32 +L3 Module: l3coffins 2014/08/23 v5354 L3 Coffin code layer +\l__coffin_internal_box=\box33 +\l__coffin_internal_dim=\dimen120 +\l__coffin_offset_x_dim=\dimen121 +\l__coffin_offset_y_dim=\dimen122 +\l__coffin_x_dim=\dimen123 +\l__coffin_y_dim=\dimen124 +\l__coffin_x_prime_dim=\dimen125 +\l__coffin_y_prime_dim=\dimen126 +\c_empty_coffin=\box34 +\l__coffin_aligned_coffin=\box35 +\l__coffin_aligned_internal_coffin=\box36 +\l_tmpa_coffin=\box37 +\l_tmpb_coffin=\box38 +\l__coffin_display_coffin=\box39 +\l__coffin_display_coord_coffin=\box40 +\l__coffin_display_pole_coffin=\box41 +\l__coffin_display_offset_dim=\dimen127 +\l__coffin_display_x_dim=\dimen128 +\l__coffin_display_y_dim=\dimen129 +L3 Module: l3color 2014/08/23 v5354 L3 Experimental color support +L3 Module: l3candidates 2015/03/01 v5544 L3 Experimental additions to l3kernel +\l__box_top_dim=\dimen130 +\l__box_bottom_dim=\dimen131 +\l__box_left_dim=\dimen132 +\l__box_right_dim=\dimen133 +\l__box_top_new_dim=\dimen134 +\l__box_bottom_new_dim=\dimen135 +\l__box_left_new_dim=\dimen136 +\l__box_right_new_dim=\dimen137 +\l__box_internal_box=\box42 +\l__coffin_bounding_shift_dim=\dimen138 +\l__coffin_left_corner_dim=\dimen139 +\l__coffin_right_corner_dim=\dimen140 +\l__coffin_bottom_corner_dim=\dimen141 +\l__coffin_top_corner_dim=\dimen142 +\l__coffin_scaled_total_height_dim=\dimen143 +\l__coffin_scaled_width_dim=\dimen144 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/l3kernel/l3unicode-data.def +File: l3unicode-data.def 2015/03/01 v5544 L3 Unicode data +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/l3kernel/l3pdfmode.def +File: l3pdfmode.def 2015/03/01 v5544 L3 Experimental driver: PDF mode +\l__driver_color_stack_int=\count126 +)) +Package: xparse 2014/11/25 v5471 L3 Experimental document command parser +\l__xparse_current_arg_int=\count127 +\l__xparse_m_args_int=\count128 +\l__xparse_mandatory_args_int=\count129 +\l__xparse_processor_int=\count130 +\l__xparse_v_nesting_int=\count131 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/etoolbox/etoolbox.sty +Package: etoolbox 2015/05/04 v2.2 e-TeX tools for LaTeX (JAW) +\etb@tempcnta=\count132 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/oberdiek/zref-abspage.sty +Package: zref-abspage 2012/04/04 v2.24 Module abspage for zref (HO) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/oberdiek/zref-base.sty +Package: zref-base 2012/04/04 v2.24 Module base for zref (HO) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/kvdefinekeys.sty +Package: kvdefinekeys 2011/04/07 v1.3 Define keys (HO) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/pdftexcmds.sty +Package: pdftexcmds 2011/11/29 v0.20 Utility functions of pdfTeX for LuaTeX (HO +) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/ifpdf.sty +Package: ifpdf 2011/01/30 v2.3 Provides the ifpdf switch (HO) +Package ifpdf Info: pdfTeX in PDF mode is detected. +) +Package pdftexcmds Info: LuaTeX not detected. +Package pdftexcmds Info: \pdf@primitive is available. +Package pdftexcmds Info: \pdf@ifprimitive is available. +Package pdftexcmds Info: \pdfdraftmode found. +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/oberdiek/auxhook.sty +Package: auxhook 2011/03/04 v1.3 Hooks for auxiliary files (HO) +) +Package zref Info: New property list: main on input line 759. +Package zref Info: New property: default on input line 760. +Package zref Info: New property: page on input line 761. +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/atbegshi.sty +Package: atbegshi 2011/10/05 v1.16 At begin shipout hook (HO) +) +\c@abspage=\count133 +Package zref Info: New property: abspage on input line 62. +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/needspace/needspace.sty +Package: needspace 2010/09/12 v1.3d reserve vertical space +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/graphics/color.sty +Package: color 2014/10/28 v1.1a Standard LaTeX Color (DPC) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/latexconfig/color.cfg +File: color.cfg 2007/01/18 v1.5 color configuration of teTeX/TeXLive +) +Package color Info: Driver file: pdftex.def on input line 142. +) +\mdf@templength=\skip57 +\c@mdf@globalstyle@cnt=\count134 +\mdf@skipabove@length=\skip58 +\mdf@skipbelow@length=\skip59 +\mdf@leftmargin@length=\skip60 +\mdf@rightmargin@length=\skip61 +\mdf@innerleftmargin@length=\skip62 +\mdf@innerrightmargin@length=\skip63 +\mdf@innertopmargin@length=\skip64 +\mdf@innerbottommargin@length=\skip65 +\mdf@splittopskip@length=\skip66 +\mdf@splitbottomskip@length=\skip67 +\mdf@outermargin@length=\skip68 +\mdf@innermargin@length=\skip69 +\mdf@linewidth@length=\skip70 +\mdf@innerlinewidth@length=\skip71 +\mdf@middlelinewidth@length=\skip72 +\mdf@outerlinewidth@length=\skip73 +\mdf@roundcorner@length=\skip74 +\mdf@footenotedistance@length=\skip75 +\mdf@userdefinedwidth@length=\skip76 +\mdf@needspace@length=\skip77 +\mdf@frametitleaboveskip@length=\skip78 +\mdf@frametitlebelowskip@length=\skip79 +\mdf@frametitlerulewidth@length=\skip80 +\mdf@frametitleleftmargin@length=\skip81 +\mdf@frametitlerightmargin@length=\skip82 +\mdf@shadowsize@length=\skip83 +\mdf@extratopheight@length=\skip84 +\mdf@subtitleabovelinewidth@length=\skip85 +\mdf@subtitlebelowlinewidth@length=\skip86 +\mdf@subtitleaboveskip@length=\skip87 +\mdf@subtitlebelowskip@length=\skip88 +\mdf@subtitleinneraboveskip@length=\skip89 +\mdf@subtitleinnerbelowskip@length=\skip90 +\mdf@subsubtitleabovelinewidth@length=\skip91 +\mdf@subsubtitlebelowlinewidth@length=\skip92 +\mdf@subsubtitleaboveskip@length=\skip93 +\mdf@subsubtitlebelowskip@length=\skip94 +\mdf@subsubtitleinneraboveskip@length=\skip95 +\mdf@subsubtitleinnerbelowskip@length=\skip96 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/frontendlayer/tikz.sty +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/basiclayer/pgf.sty +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/utilities/pgfrcs.sty +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgfutil-common.te +x +\pgfutil@everybye=\toks28 +\pgfutil@tempdima=\dimen145 +\pgfutil@tempdimb=\dimen146 + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgfutil-common-li +sts.tex)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgfutil-latex.def +\pgfutil@abb=\box43 +(/usr/local/texlive/2015/texmf-dist/tex/latex/ms/everyshi.sty +Package: everyshi 2001/05/15 v3.00 EveryShipout Package (MS) +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgfrcs.code.tex +Package: pgfrcs 2013/12/20 v3.0.0 (rcs-revision 1.28) +)) +Package: pgf 2013/12/18 v3.0.0 (rcs-revision 1.14) +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/basiclayer/pgfcore.sty +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/systemlayer/pgfsys.sty +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/systemlayer/pgfsys.code.tex +Package: pgfsys 2013/11/30 v3.0.0 (rcs-revision 1.47) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex +\pgfkeys@pathtoks=\toks29 +\pgfkeys@temptoks=\toks30 + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgfkeysfiltered.c +ode.tex +\pgfkeys@tmptoks=\toks31 +)) +\pgf@x=\dimen147 +\pgf@y=\dimen148 +\pgf@xa=\dimen149 +\pgf@ya=\dimen150 +\pgf@xb=\dimen151 +\pgf@yb=\dimen152 +\pgf@xc=\dimen153 +\pgf@yc=\dimen154 +\w@pgf@writea=\write3 +\r@pgf@reada=\read1 +\c@pgf@counta=\count135 +\c@pgf@countb=\count136 +\c@pgf@countc=\count137 +\c@pgf@countd=\count138 +\t@pgf@toka=\toks32 +\t@pgf@tokb=\toks33 +\t@pgf@tokc=\toks34 + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/systemlayer/pgf.cfg +File: pgf.cfg 2008/05/14 (rcs-revision 1.7) +) +Driver file for pgf: pgfsys-pdftex.def + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-pdftex.d +ef +File: pgfsys-pdftex.def 2013/07/18 (rcs-revision 1.33) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/systemlayer/pgfsys-common-p +df.def +File: pgfsys-common-pdf.def 2013/10/10 (rcs-revision 1.13) +))) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/systemlayer/pgfsyssoftpath. +code.tex +File: pgfsyssoftpath.code.tex 2013/09/09 (rcs-revision 1.9) +\pgfsyssoftpath@smallbuffer@items=\count139 +\pgfsyssoftpath@bigbuffer@items=\count140 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/systemlayer/pgfsysprotocol. +code.tex +File: pgfsysprotocol.code.tex 2006/10/16 (rcs-revision 1.4) +)) (/usr/local/texlive/2015/texmf-dist/tex/latex/xcolor/xcolor.sty +Package: xcolor 2007/01/21 v2.11 LaTeX color extensions (UK) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/latexconfig/color.cfg +File: color.cfg 2007/01/18 v1.5 color configuration of teTeX/TeXLive +) +Package xcolor Info: Driver file: pdftex.def on input line 225. +LaTeX Info: Redefining \color on input line 702. +Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1337. +Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1341. +Package xcolor Info: Model `RGB' extended on input line 1353. +Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1355. +Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1356. +Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1357. +Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1358. +Package xcolor Info: Model `Gray' substituted by `gray' on input line 1359. +Package xcolor Info: Model `wave' substituted by `hsb' on input line 1360. +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcore.code.tex +Package: pgfcore 2010/04/11 v3.0.0 (rcs-revision 1.7) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathcalc.code.tex +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathutil.code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathparser.code.tex +\pgfmath@dimen=\dimen155 +\pgfmath@count=\count141 +\pgfmath@box=\box44 +\pgfmath@toks=\toks35 +\pgfmath@stack@operand=\toks36 +\pgfmath@stack@operation=\toks37 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.code. +tex +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.basic +.code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.trigo +nometric.code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.rando +m.code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.compa +rison.code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.base. +code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.round +.code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.misc. +code.tex) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfunctions.integ +erarithmetics.code.tex))) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmathfloat.code.tex +\c@pgfmathroundto@lastzeros=\count142 +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepoints.co +de.tex +File: pgfcorepoints.code.tex 2013/10/07 (rcs-revision 1.27) +\pgf@picminx=\dimen156 +\pgf@picmaxx=\dimen157 +\pgf@picminy=\dimen158 +\pgf@picmaxy=\dimen159 +\pgf@pathminx=\dimen160 +\pgf@pathmaxx=\dimen161 +\pgf@pathminy=\dimen162 +\pgf@pathmaxy=\dimen163 +\pgf@xx=\dimen164 +\pgf@xy=\dimen165 +\pgf@yx=\dimen166 +\pgf@yy=\dimen167 +\pgf@zx=\dimen168 +\pgf@zy=\dimen169 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathconst +ruct.code.tex +File: pgfcorepathconstruct.code.tex 2013/10/07 (rcs-revision 1.29) +\pgf@path@lastx=\dimen170 +\pgf@path@lasty=\dimen171 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathusage +.code.tex +File: pgfcorepathusage.code.tex 2013/12/13 (rcs-revision 1.23) +\pgf@shorten@end@additional=\dimen172 +\pgf@shorten@start@additional=\dimen173 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorescopes.co +de.tex +File: pgfcorescopes.code.tex 2013/10/09 (rcs-revision 1.44) +\pgfpic=\box45 +\pgf@hbox=\box46 +\pgf@layerbox@main=\box47 +\pgf@picture@serial@count=\count143 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcoregraphicst +ate.code.tex +File: pgfcoregraphicstate.code.tex 2013/09/19 (rcs-revision 1.11) +\pgflinewidth=\dimen174 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretransform +ations.code.tex +File: pgfcoretransformations.code.tex 2013/10/10 (rcs-revision 1.17) +\pgf@pt@x=\dimen175 +\pgf@pt@y=\dimen176 +\pgf@pt@temp=\dimen177 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorequick.cod +e.tex +File: pgfcorequick.code.tex 2008/10/09 (rcs-revision 1.3) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreobjects.c +ode.tex +File: pgfcoreobjects.code.tex 2006/10/11 (rcs-revision 1.2) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepathproce +ssing.code.tex +File: pgfcorepathprocessing.code.tex 2013/09/09 (rcs-revision 1.9) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorearrows.co +de.tex +File: pgfcorearrows.code.tex 2013/11/07 (rcs-revision 1.40) +\pgfarrowsep=\dimen178 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreshade.cod +e.tex +File: pgfcoreshade.code.tex 2013/07/15 (rcs-revision 1.15) +\pgf@max=\dimen179 +\pgf@sys@shading@range@num=\count144 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreimage.cod +e.tex +File: pgfcoreimage.code.tex 2013/07/15 (rcs-revision 1.18) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcoreexternal. +code.tex +File: pgfcoreexternal.code.tex 2013/07/15 (rcs-revision 1.20) +\pgfexternal@startupbox=\box48 +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorelayers.co +de.tex +File: pgfcorelayers.code.tex 2013/07/18 (rcs-revision 1.7) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcoretranspare +ncy.code.tex +File: pgfcoretransparency.code.tex 2013/09/30 (rcs-revision 1.5) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/basiclayer/pgfcorepatterns. +code.tex +File: pgfcorepatterns.code.tex 2013/11/07 (rcs-revision 1.5) +))) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/modules/pgfmoduleshapes.cod +e.tex +File: pgfmoduleshapes.code.tex 2013/10/31 (rcs-revision 1.34) +\pgfnodeparttextbox=\box49 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/modules/pgfmoduleplot.code. +tex +File: pgfmoduleplot.code.tex 2013/07/31 (rcs-revision 1.12) +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version +-0-65.sty +Package: pgfcomp-version-0-65 2007/07/03 v3.0.0 (rcs-revision 1.7) +\pgf@nodesepstart=\dimen180 +\pgf@nodesepend=\dimen181 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/compatibility/pgfcomp-version +-1-18.sty +Package: pgfcomp-version-1-18 2007/07/23 v3.0.0 (rcs-revision 1.1) +)) +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/utilities/pgffor.sty +(/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/utilities/pgfkeys.sty +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgfkeys.code.tex) +) (/usr/local/texlive/2015/texmf-dist/tex/latex/pgf/math/pgfmath.sty +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/utilities/pgffor.code.tex +Package: pgffor 2013/12/13 v3.0.0 (rcs-revision 1.25) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/math/pgfmath.code.tex) +\pgffor@iter=\dimen182 +\pgffor@skip=\dimen183 +\pgffor@stack=\toks38 +\pgffor@toks=\toks39 +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/tikz.cod +e.tex +Package: tikz 2013/12/13 v3.0.0 (rcs-revision 1.142) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/libraries/pgflibraryplothan +dlers.code.tex +File: pgflibraryplothandlers.code.tex 2013/08/31 v3.0.0 (rcs-revision 1.20) +\pgf@plot@mark@count=\count145 +\pgfplotmarksize=\dimen184 +) +\tikz@lastx=\dimen185 +\tikz@lasty=\dimen186 +\tikz@lastxsaved=\dimen187 +\tikz@lastysaved=\dimen188 +\tikzleveldistance=\dimen189 +\tikzsiblingdistance=\dimen190 +\tikz@figbox=\box50 +\tikz@figbox@bg=\box51 +\tikz@tempbox=\box52 +\tikz@tempbox@bg=\box53 +\tikztreelevel=\count146 +\tikznumberofchildren=\count147 +\tikznumberofcurrentchild=\count148 +\tikz@fig@count=\count149 + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/modules/pgfmodulematrix.cod +e.tex +File: pgfmodulematrix.code.tex 2013/09/17 (rcs-revision 1.8) +\pgfmatrixcurrentrow=\count150 +\pgfmatrixcurrentcolumn=\count151 +\pgf@matrix@numberofcolumns=\count152 +) +\tikz@expandcount=\count153 + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarytopaths.code.tex +File: tikzlibrarytopaths.code.tex 2008/06/17 v3.0.0 (rcs-revision 1.2) +))) +(/usr/local/texlive/2015/texmf-dist/tex/latex/mdframed/md-frame-1.mdf +File: md-frame-1.mdf 2013/07/01\ 1.9b: md-frame-1 +) +\mdf@frametitlebox=\box54 +\mdf@footnotebox=\box55 +\mdf@splitbox@one=\box56 +\mdf@splitbox@two=\box57 +\mdf@splitbox@save=\box58 +\mdfsplitboxwidth=\skip97 +\mdfsplitboxtotalwidth=\skip98 +\mdfsplitboxheight=\skip99 +\mdfsplitboxdepth=\skip100 +\mdfsplitboxtotalheight=\skip101 +\mdfframetitleboxwidth=\skip102 +\mdfframetitleboxtotalwidth=\skip103 +\mdfframetitleboxheight=\skip104 +\mdfframetitleboxdepth=\skip105 +\mdfframetitleboxtotalheight=\skip106 +\mdffootnoteboxwidth=\skip107 +\mdffootnoteboxtotalwidth=\skip108 +\mdffootnoteboxheight=\skip109 +\mdffootnoteboxdepth=\skip110 +\mdffootnoteboxtotalheight=\skip111 +\mdftotallinewidth=\skip112 +\mdfboundingboxwidth=\skip113 +\mdfboundingboxtotalwidth=\skip114 +\mdfboundingboxheight=\skip115 +\mdfboundingboxdepth=\skip116 +\mdfboundingboxtotalheight=\skip117 +\mdf@freevspace@length=\skip118 +\mdf@horizontalwidthofbox@length=\skip119 +\mdf@verticalmarginwhole@length=\skip120 +\mdf@horizontalspaceofbox=\skip121 +\mdfsubtitleheight=\skip122 +\mdfsubsubtitleheight=\skip123 +\c@mdfcountframes=\count154 + +****** mdframed patching \endmdf@trivlist + +****** -- success****** + +................................................. +. LaTeX info: "xparse/define-command" +. +. Defining command \newmdtheoremenv with sig. 'O{} m o m o ' on line 601. +................................................. +................................................. +. LaTeX info: "xparse/define-command" +. +. Defining command \mdtheorem with sig. ' O{} m o m o ' on line 701. +................................................. +\mdf@envdepth=\count155 +\c@mdf@env@i=\count156 +\c@mdf@env@ii=\count157 +\c@mdf@zref@counter=\count158 +Package zref Info: New property: mdf@pagevalue on input line 895. +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/natbib/natbib.sty +Package: natbib 2010/09/13 8.31b (PWD, AO) +\bibhang=\skip124 +\bibsep=\skip125 +LaTeX Info: Redefining \cite on input line 694. +\c@NAT@ctr=\count159 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/caption/subcaption.sty +Package: subcaption 2013/02/03 v1.1-62 Sub-captions (AR) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/caption/caption.sty +Package: caption 2013/05/02 v3.3-89 Customizing captions (AR) + +(/usr/local/texlive/2015/texmf-dist/tex/latex/caption/caption3.sty +Package: caption3 2013/05/02 v1.6-88 caption3 kernel (AR) +Package caption3 Info: TeX engine: e-TeX on input line 57. +\captionmargin=\dimen191 +\captionmargin@=\dimen192 +\captionwidth=\dimen193 +\caption@tempdima=\dimen194 +\caption@indent=\dimen195 +\caption@parindent=\dimen196 +\caption@hangindent=\dimen197 +) +\c@ContinuedFloat=\count160 +) +\c@subfigure=\count161 +\c@subtable=\count162 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/epigraph/epigraph.sty +Package: epigraph 2009/09/02 v1.5c typesetting epigraphs +\beforeepigraphskip=\skip126 +\afterepigraphskip=\skip127 +\epigraphwidth=\skip128 +\epigraphrule=\skip129 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/float/float.sty +Package: float 2001/11/08 v1.3d Float enhancements (AL) +\c@float@type=\count163 +\float@exts=\toks40 +\float@box=\box59 +\@float@everytoks=\toks41 +\@floatcapt=\box60 +) +(/usr/local/texlive/2015/texmf-dist/tex/latex/hyperref/hyperref.sty +Package: hyperref 2012/11/06 v6.83m Hypertext links for LaTeX + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/hobsub-hyperref.sty +Package: hobsub-hyperref 2012/05/28 v1.13 Bundle oberdiek, subset hyperref (HO) + + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/hobsub-generic.sty +Package: hobsub-generic 2012/05/28 v1.13 Bundle oberdiek, subset generic (HO) +Package: hobsub 2012/05/28 v1.13 Construct package bundles (HO) +Package hobsub Info: Skipping package `infwarerr' (already loaded). +Package hobsub Info: Skipping package `ltxcmds' (already loaded). +Package hobsub Info: Skipping package `ifluatex' (already loaded). +Package: ifvtex 2010/03/01 v1.5 Detect VTeX and its facilities (HO) +Package ifvtex Info: VTeX not detected. +Package: intcalc 2007/09/27 v1.1 Expandable calculations with integers (HO) +Package hobsub Info: Skipping package `ifpdf' (already loaded). +Package hobsub Info: Skipping package `etexcmds' (already loaded). +Package hobsub Info: Skipping package `kvsetkeys' (already loaded). +Package hobsub Info: Skipping package `kvdefinekeys' (already loaded). +Package hobsub Info: Skipping package `pdftexcmds' (already loaded). +Package: pdfescape 2011/11/25 v1.13 Implements pdfTeX's escape features (HO) +Package: bigintcalc 2012/04/08 v1.3 Expandable calculations on big integers (HO +) +Package: bitset 2011/01/30 v1.1 Handle bit-vector datatype (HO) +Package: uniquecounter 2011/01/30 v1.2 Provide unlimited unique counter (HO) +) +Package hobsub Info: Skipping package `hobsub' (already loaded). +Package: letltxmacro 2010/09/02 v1.4 Let assignment for LaTeX macros (HO) +Package: hopatch 2012/05/28 v1.2 Wrapper for package hooks (HO) +Package: xcolor-patch 2011/01/30 xcolor patch +Package: atveryend 2011/06/30 v1.8 Hooks at the very end of document (HO) +Package hobsub Info: Skipping package `atbegshi' (already loaded). +Package: refcount 2011/10/16 v3.4 Data extraction from label references (HO) +Package: hycolor 2011/01/30 v1.7 Color options for hyperref/bookmark (HO) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/ifxetex/ifxetex.sty +Package: ifxetex 2010/09/12 v0.6 Provides ifxetex conditional +) +\@linkdim=\dimen198 +\Hy@linkcounter=\count164 +\Hy@pagecounter=\count165 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/hyperref/pd1enc.def +File: pd1enc.def 2012/11/06 v6.83m Hyperref: PDFDocEncoding definition (HO) +Now handling font encoding PD1 ... +... no UTF-8 mapping file for font encoding PD1 +) +\Hy@SavedSpaceFactor=\count166 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/latexconfig/hyperref.cfg +File: hyperref.cfg 2002/06/06 v1.2 hyperref configuration of TeXLive +) +Package hyperref Info: Hyper figures OFF on input line 4443. +Package hyperref Info: Link nesting OFF on input line 4448. +Package hyperref Info: Hyper index ON on input line 4451. +Package hyperref Info: Plain pages OFF on input line 4458. +Package hyperref Info: Backreferencing OFF on input line 4463. +Package hyperref Info: Implicit mode ON; LaTeX internals redefined. +Package hyperref Info: Bookmarks ON on input line 4688. +\c@Hy@tempcnt=\count167 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/url/url.sty +\Urlmuskip=\muskip17 +Package: url 2013/09/16 ver 3.4 Verb mode for urls, etc. +) +LaTeX Info: Redefining \url on input line 5041. +\XeTeXLinkMargin=\dimen199 +\Fld@menulength=\count168 +\Field@Width=\dimen200 +\Fld@charsize=\dimen201 +Package hyperref Info: Hyper figures OFF on input line 6295. +Package hyperref Info: Link nesting OFF on input line 6300. +Package hyperref Info: Hyper index ON on input line 6303. +Package hyperref Info: backreferencing OFF on input line 6310. +Package hyperref Info: Link coloring OFF on input line 6315. +Package hyperref Info: Link coloring with OCG OFF on input line 6320. +Package hyperref Info: PDF/A mode OFF on input line 6325. +LaTeX Info: Redefining \ref on input line 6365. +LaTeX Info: Redefining \pageref on input line 6369. +\Hy@abspage=\count169 +\c@Item=\count170 +\c@Hfootnote=\count171 +) + +Package hyperref Message: Driver (autodetected): hpdftex. + +(/usr/local/texlive/2015/texmf-dist/tex/latex/hyperref/hpdftex.def +File: hpdftex.def 2012/11/06 v6.83m Hyperref driver for pdfTeX +\HyAnn@Count=\count172 +\Fld@listcount=\count173 +\c@bookmark@seq@number=\count174 + +(/usr/local/texlive/2015/texmf-dist/tex/latex/oberdiek/rerunfilecheck.sty +Package: rerunfilecheck 2011/04/15 v1.7 Rerun checks for auxiliary files (HO) +Package uniquecounter Info: New unique counter `rerunfilecheck' on input line 2 +82. +) +\Hy@SectionHShift=\skip130 +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarycalc.code.tex +File: tikzlibrarycalc.code.tex 2013/07/15 v3.0.0 (rcs-revision 1.9) +) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryshapes.code.tex +File: tikzlibraryshapes.code.tex 2008/01/09 v3.0.0 (rcs-revision 1.1) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryshapes.geometric.code.tex +File: tikzlibraryshapes.geometric.code.tex 2008/01/09 v3.0.0 (rcs-revision 1.1) + + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/libraries/shapes/pgflibrary +shapes.geometric.code.tex +File: pgflibraryshapes.geometric.code.tex 2008/06/26 v3.0.0 (rcs-revision 1.1) +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryshapes.misc.code.tex +File: tikzlibraryshapes.misc.code.tex 2008/01/09 v3.0.0 (rcs-revision 1.1) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/libraries/shapes/pgflibrary +shapes.misc.code.tex +File: pgflibraryshapes.misc.code.tex 2013/07/18 v3.0.0 (rcs-revision 1.5) +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryshapes.symbols.code.tex +File: tikzlibraryshapes.symbols.code.tex 2008/01/09 v3.0.0 (rcs-revision 1.1) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/libraries/shapes/pgflibrary +shapes.symbols.code.tex +File: pgflibraryshapes.symbols.code.tex 2013/09/11 v3.0.0 (rcs-revision 1.6) +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryshapes.arrows.code.tex +File: tikzlibraryshapes.arrows.code.tex 2008/01/09 v3.0.0 (rcs-revision 1.1) + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/libraries/shapes/pgflibrary +shapes.arrows.code.tex +File: pgflibraryshapes.arrows.code.tex 2008/06/26 v3.0.0 (rcs-revision 1.1) +)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryshapes.callouts.code.tex +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/libraries/shapes/pgflibrary +shapes.callouts.code.tex)) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibraryshapes.multipart.code.tex +File: tikzlibraryshapes.multipart.code.tex 2008/01/09 v3.0.0 (rcs-revision 1.1) + + +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/libraries/shapes/pgflibrary +shapes.multipart.code.tex +File: pgflibraryshapes.multipart.code.tex 2010/01/07 v3.0.0 (rcs-revision 1.2) +\pgfnodepartlowerbox=\box61 +\pgfnodeparttwobox=\box62 +\pgfnodepartthreebox=\box63 +\pgfnodepartfourbox=\box64 +\pgfnodeparttwentybox=\box65 +\pgfnodepartnineteenbox=\box66 +\pgfnodeparteighteenbox=\box67 +\pgfnodepartseventeenbox=\box68 +\pgfnodepartsixteenbox=\box69 +\pgfnodepartfifteenbox=\box70 +\pgfnodepartfourteenbox=\box71 +\pgfnodepartthirteenbox=\box72 +\pgfnodeparttwelvebox=\box73 +\pgfnodepartelevenbox=\box74 +\pgfnodeparttenbox=\box75 +\pgfnodepartninebox=\box76 +\pgfnodeparteightbox=\box77 +\pgfnodepartsevenbox=\box78 +\pgfnodepartsixbox=\box79 +\pgfnodepartfivebox=\box80 +))) +(/usr/local/texlive/2015/texmf-dist/tex/generic/pgf/frontendlayer/tikz/librarie +s/tikzlibrarypositioning.code.tex +File: tikzlibrarypositioning.code.tex 2008/10/06 v3.0.0 (rcs-revision 1.7) +) +\c@relationship=\count175 + (./conv_arithmetic.aux) +\openout1 = `conv_arithmetic.aux'. + +LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 61. +LaTeX Font Info: ... okay on input line 61. +LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 61. +LaTeX Font Info: ... okay on input line 61. +LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 61. +LaTeX Font Info: ... okay on input line 61. +LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 61. +LaTeX Font Info: ... okay on input line 61. +LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 61. +LaTeX Font Info: ... okay on input line 61. +LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 61. +LaTeX Font Info: ... okay on input line 61. +LaTeX Font Info: Checking defaults for PD1/pdf/m/n on input line 61. +LaTeX Font Info: ... okay on input line 61. + +(/usr/local/texlive/2015/texmf-dist/tex/context/base/supp-pdf.mkii +[Loading MPS to PDF converter (version 2006.09.02).] +\scratchcounter=\count176 +\scratchdimen=\dimen202 +\scratchbox=\box81 +\nofMPsegments=\count177 +\nofMParguments=\count178 +\everyMPshowfont=\toks42 +\MPscratchCnt=\count179 +\MPscratchDim=\dimen203 +\MPnumerator=\count180 +\makeMPintoPDFobject=\count181 +\everyMPtoPDFconversion=\toks43 +) (/usr/local/texlive/2015/texmf-dist/tex/latex/oberdiek/epstopdf-base.sty +Package: epstopdf-base 2010/02/09 v2.5 Base part for package epstopdf + +(/usr/local/texlive/2015/texmf-dist/tex/latex/oberdiek/grfext.sty +Package: grfext 2010/08/19 v1.1 Manage graphics extensions (HO) +) +Package grfext Info: Graphics extension search list: +(grfext) [.png,.pdf,.jpg,.mps,.jpeg,.jbig2,.jb2,.PNG,.PDF,.JPG,.JPE +G,.JBIG2,.JB2,.eps] +(grfext) \AppendGraphicsExtensions on input line 452. + +(/usr/local/texlive/2015/texmf-dist/tex/latex/latexconfig/epstopdf-sys.cfg +File: epstopdf-sys.cfg 2010/07/13 v1.3 Configuration of (r)epstopdf for TeX Liv +e +)) +\AtBeginShipoutBox=\box82 + +ABD: EveryShipout initializing macros +Package caption Info: Begin \AtBeginDocument code. +Package caption Info: float package is loaded. +Package caption Info: hyperref package is loaded. +Package caption Info: End \AtBeginDocument code. +Package hyperref Info: Link coloring OFF on input line 61. + +(/usr/local/texlive/2015/texmf-dist/tex/latex/hyperref/nameref.sty +Package: nameref 2012/10/27 v2.43 Cross-referencing by name of section + +(/usr/local/texlive/2015/texmf-dist/tex/generic/oberdiek/gettitlestring.sty +Package: gettitlestring 2010/12/03 v1.4 Cleanup title references (HO) +) +\c@section@level=\count182 +) +LaTeX Info: Redefining \ref on input line 61. +LaTeX Info: Redefining \pageref on input line 61. +LaTeX Info: Redefining \nameref on input line 61. + +(./conv_arithmetic.out) (./conv_arithmetic.out) +\@outlinefile=\write4 +\openout4 = `conv_arithmetic.out'. + +LaTeX Font Info: Try loading font information for U+msa on input line 64. + +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsfonts/umsa.fd +File: umsa.fd 2013/01/14 v3.01 AMS symbols A +) +LaTeX Font Info: Try loading font information for U+msb on input line 64. + +(/usr/local/texlive/2015/texmf-dist/tex/latex/amsfonts/umsb.fd +File: umsb.fd 2013/01/14 v3.01 AMS symbols B +) [1 + +{/usr/local/texlive/2015/texmf-var/fonts/map/pdftex/updmap/pdftex.map}] [2 + +] +LaTeX Font Info: Try loading font information for T1+cmtt on input line 79. + +(/usr/local/texlive/2015/texmf-dist/tex/latex/base/t1cmtt.fd +File: t1cmtt.fd 2014/09/29 v2.5h Standard LaTeX font definitions +) [3 + +] +(./conv_arithmetic.toc) +\tf@toc=\write5 +\openout5 = `conv_arithmetic.toc'. + + [4 + +] +Chapter 1. + +Overfull \hbox (1.97696pt too wide) in paragraph at lines 131--134 +[]\T1/cmr/m/n/10 Provide an in-tu-itive un-der-stand-ing of the re-la-tion-ship + be-tween in-put shape, + [] + +[5 + +] +LaTeX Font Info: Try loading font information for OMS+cmr on input line 160. + + (/usr/local/texlive/2015/texmf-dist/tex/latex/base/omscmr.fd +File: omscmr.fd 2014/09/29 v2.5h Standard LaTeX font definitions +) +LaTeX Font Info: Font shape `OMS/cmr/m/n' in size <10> not available +(Font) Font shape `OMS/cmsy/m/n' tried instead on input line 160. + + +File: pdf/numerical_no_padding_no_strides_00.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_00.pdf used on inp +ut line 207. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_no_padding_no_strides_01.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_01.pdf used on inp +ut line 208. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_no_padding_no_strides_02.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_02.pdf used on inp +ut line 209. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_no_padding_no_strides_03.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_03.pdf used on inp +ut line 210. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_no_padding_no_strides_04.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_04.pdf used on inp +ut line 211. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_no_padding_no_strides_05.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_05.pdf used on inp +ut line 212. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_no_padding_no_strides_06.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_06.pdf used on inp +ut line 213. +(pdftex.def) Requested size: 110.40253pt x 74.59024pt. + + +File: pdf/numerical_no_padding_no_strides_07.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_07.pdf used on inp +ut line 214. +(pdftex.def) Requested size: 110.40253pt x 74.59024pt. + + +File: pdf/numerical_no_padding_no_strides_08.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/numerical_no_padding_no_strides_08.pdf used on inp +ut line 215. +(pdftex.def) Requested size: 110.40253pt x 74.59024pt. + + +File: pdf/numerical_padding_strides_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_00.pdf used on input lin +e 222. +(pdftex.def) Requested size: 110.40253pt x 77.46133pt. + + +File: pdf/numerical_padding_strides_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_01.pdf used on input lin +e 223. +(pdftex.def) Requested size: 110.40253pt x 77.46133pt. + + +File: pdf/numerical_padding_strides_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_02.pdf used on input lin +e 224. +(pdftex.def) Requested size: 110.40253pt x 77.46133pt. + + +File: pdf/numerical_padding_strides_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_03.pdf used on input lin +e 225. +(pdftex.def) Requested size: 110.40253pt x 77.46133pt. + + +File: pdf/numerical_padding_strides_04.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_04.pdf used on input lin +e 226. +(pdftex.def) Requested size: 110.40253pt x 77.46133pt. + + +File: pdf/numerical_padding_strides_05.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_05.pdf used on input lin +e 227. +(pdftex.def) Requested size: 110.40253pt x 77.46133pt. + + +File: pdf/numerical_padding_strides_06.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_06.pdf used on input lin +e 228. +(pdftex.def) Requested size: 110.40253pt x 79.93198pt. + + +File: pdf/numerical_padding_strides_07.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_07.pdf used on input lin +e 229. +(pdftex.def) Requested size: 110.40253pt x 79.93198pt. + + +File: pdf/numerical_padding_strides_08.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_padding_strides_08.pdf used on input lin +e 230. +(pdftex.def) Requested size: 110.40253pt x 79.93198pt. + [6] [7 <./pdf/numerical_no_padding_no_strides_00.pdf> <./pdf/numerical_no_padd +ing_no_strides_01.pdf> <./pdf/numerical_no_padding_no_strides_02.pdf> <./pdf/nu +merical_no_padding_no_strides_03.pdf> <./pdf/numerical_no_padding_no_strides_04 +.pdf> <./pdf/numerical_no_padding_no_strides_05.pdf> <./pdf/numerical_no_paddin +g_no_strides_06.pdf> <./pdf/numerical_no_padding_no_strides_07.pdf> <./pdf/nume +rical_no_padding_no_strides_08.pdf> <./pdf/numerical_padding_strides_00.pdf> <. +/pdf/numerical_padding_strides_01.pdf> <./pdf/numerical_padding_strides_02.pdf> + <./pdf/numerical_padding_strides_03.pdf> <./pdf/numerical_padding_strides_04.p +df> <./pdf/numerical_padding_strides_05.pdf> <./pdf/numerical_padding_strides_0 +6.pdf> <./pdf/numerical_padding_strides_07.pdf> <./pdf/numerical_padding_stride +s_08.pdf>] [8] [9] + +File: pdf/numerical_average_pooling_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_00.pdf used on input lin +e 425. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_01.pdf used on input lin +e 426. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_02.pdf used on input lin +e 427. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_03.pdf used on input lin +e 428. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_04.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_04.pdf used on input lin +e 429. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_05.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_05.pdf used on input lin +e 430. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_06.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_06.pdf used on input lin +e 431. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_07.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_07.pdf used on input lin +e 432. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_average_pooling_08.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_average_pooling_08.pdf used on input lin +e 433. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_00.pdf used on input line 44 +1. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_01.pdf used on input line 44 +2. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_02.pdf used on input line 44 +3. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_03.pdf used on input line 44 +4. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_04.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_04.pdf used on input line 44 +5. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_05.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_05.pdf used on input line 44 +6. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_06.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_06.pdf used on input line 44 +7. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_07.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_07.pdf used on input line 44 +8. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + + +File: pdf/numerical_max_pooling_08.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/numerical_max_pooling_08.pdf used on input line 44 +9. +(pdftex.def) Requested size: 110.40253pt x 71.68631pt. + [10] [11 <./pdf/numerical_average_pooling_00.pdf> <./pdf/numerical_average_poo +ling_01.pdf> <./pdf/numerical_average_pooling_02.pdf> <./pdf/numerical_average_ +pooling_03.pdf> <./pdf/numerical_average_pooling_04.pdf> <./pdf/numerical_avera +ge_pooling_05.pdf> <./pdf/numerical_average_pooling_06.pdf> <./pdf/numerical_av +erage_pooling_07.pdf> <./pdf/numerical_average_pooling_08.pdf> <./pdf/numerical +_max_pooling_00.pdf> <./pdf/numerical_max_pooling_01.pdf> <./pdf/numerical_max_ +pooling_02.pdf> <./pdf/numerical_max_pooling_03.pdf> <./pdf/numerical_max_pooli +ng_04.pdf> <./pdf/numerical_max_pooling_05.pdf> <./pdf/numerical_max_pooling_06 +.pdf> <./pdf/numerical_max_pooling_07.pdf> <./pdf/numerical_max_pooling_08.pdf> +] +Chapter 2. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 491. +[12 + +] +File: pdf/no_padding_no_strides_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_no_strides_00.pdf used on input line 50 +0. +(pdftex.def) Requested size: 82.8019pt x 87.81596pt. + + +File: pdf/no_padding_no_strides_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_no_strides_01.pdf used on input line 50 +1. +(pdftex.def) Requested size: 82.8019pt x 87.81738pt. + + +File: pdf/no_padding_no_strides_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_no_strides_02.pdf used on input line 50 +2. +(pdftex.def) Requested size: 82.8019pt x 87.81596pt. + + +File: pdf/no_padding_no_strides_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_no_strides_03.pdf used on input line 50 +3. +(pdftex.def) Requested size: 82.8019pt x 87.81596pt. + + +File: pdf/arbitrary_padding_no_strides_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_00.pdf used on input +line 511. +(pdftex.def) Requested size: 82.8019pt x 90.70862pt. + + +File: pdf/arbitrary_padding_no_strides_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_01.pdf used on input +line 512. +(pdftex.def) Requested size: 82.8019pt x 90.70862pt. + + +File: pdf/arbitrary_padding_no_strides_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_02.pdf used on input +line 513. +(pdftex.def) Requested size: 82.8019pt x 90.70862pt. + + +File: pdf/arbitrary_padding_no_strides_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_03.pdf used on input +line 514. +(pdftex.def) Requested size: 82.8019pt x 90.70862pt. + + +File: pdf/same_padding_no_strides_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/same_padding_no_strides_00.pdf used on input line +523. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/same_padding_no_strides_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/same_padding_no_strides_01.pdf used on input line +524. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/same_padding_no_strides_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/same_padding_no_strides_02.pdf used on input line +525. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/same_padding_no_strides_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/same_padding_no_strides_03.pdf used on input line +526. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/full_padding_no_strides_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/full_padding_no_strides_00.pdf used on input line +534. +(pdftex.def) Requested size: 82.8019pt x 96.38992pt. + + +File: pdf/full_padding_no_strides_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/full_padding_no_strides_01.pdf used on input line +535. +(pdftex.def) Requested size: 82.8019pt x 96.38992pt. + + +File: pdf/full_padding_no_strides_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/full_padding_no_strides_02.pdf used on input line +536. +(pdftex.def) Requested size: 82.8019pt x 96.38992pt. + + +File: pdf/full_padding_no_strides_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/full_padding_no_strides_03.pdf used on input line +537. +(pdftex.def) Requested size: 82.8019pt x 96.38992pt. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 551. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 569. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 591. + [13] [14 <./pdf/no_padding_no_strides_00.pdf> <./pdf/no_padding_no_strides_01. +pdf> <./pdf/no_padding_no_strides_02.pdf> <./pdf/no_padding_no_strides_03.pdf> +<./pdf/arbitrary_padding_no_strides_00.pdf> <./pdf/arbitrary_padding_no_strides +_01.pdf> <./pdf/arbitrary_padding_no_strides_02.pdf> <./pdf/arbitrary_padding_n +o_strides_03.pdf> <./pdf/same_padding_no_strides_00.pdf> <./pdf/same_padding_no +_strides_01.pdf> <./pdf/same_padding_no_strides_02.pdf> <./pdf/same_padding_no_ +strides_03.pdf> <./pdf/full_padding_no_strides_00.pdf> <./pdf/full_padding_no_s +trides_01.pdf> <./pdf/full_padding_no_strides_02.pdf> <./pdf/full_padding_no_st +rides_03.pdf>] +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 625. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 644. + [15] + +File: pdf/no_padding_strides_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_00.pdf used on input line 666. +(pdftex.def) Requested size: 82.8019pt x 81.13985pt. + + +File: pdf/no_padding_strides_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_01.pdf used on input line 667. +(pdftex.def) Requested size: 82.8019pt x 81.1416pt. + + +File: pdf/no_padding_strides_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_02.pdf used on input line 668. +(pdftex.def) Requested size: 82.8019pt x 81.13985pt. + + +File: pdf/no_padding_strides_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_03.pdf used on input line 669. +(pdftex.def) Requested size: 82.8019pt x 81.13985pt. + + +File: pdf/padding_strides_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_00.pdf used on input line 678. +(pdftex.def) Requested size: 82.8019pt x 80.01392pt. + + +File: pdf/padding_strides_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_01.pdf used on input line 679. +(pdftex.def) Requested size: 82.8019pt x 80.01392pt. + + +File: pdf/padding_strides_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_02.pdf used on input line 680. +(pdftex.def) Requested size: 82.8019pt x 80.01524pt. + + +File: pdf/padding_strides_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_03.pdf used on input line 681. +(pdftex.def) Requested size: 82.8019pt x 80.01392pt. + + +File: pdf/padding_strides_odd_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_00.pdf used on input line 690. + +(pdftex.def) Requested size: 82.8019pt x 76.48041pt. + + +File: pdf/padding_strides_odd_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_01.pdf used on input line 691. + +(pdftex.def) Requested size: 82.8019pt x 76.48041pt. + + +File: pdf/padding_strides_odd_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_02.pdf used on input line 692. + +(pdftex.def) Requested size: 82.8019pt x 76.48157pt. + + +File: pdf/padding_strides_odd_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_03.pdf used on input line 693. + +(pdftex.def) Requested size: 82.8019pt x 76.48041pt. + [16] [17 <./pdf/no_padding_strides_00.pdf> <./pdf/no_padding_strides_01.pdf> < +./pdf/no_padding_strides_02.pdf> <./pdf/no_padding_strides_03.pdf> <./pdf/paddi +ng_strides_00.pdf> <./pdf/padding_strides_01.pdf> <./pdf/padding_strides_02.pdf +> <./pdf/padding_strides_03.pdf> <./pdf/padding_strides_odd_00.pdf> <./pdf/padd +ing_strides_odd_01.pdf> <./pdf/padding_strides_odd_02.pdf> <./pdf/padding_strid +es_odd_03.pdf>] +Chapter 3. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 758. +[18 + +] +Chapter 4. +[19 + +] [20] +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 916. + [21] +Overfull \hbox (0.58594pt too wide) in paragraph at lines 925--927 +[]\T1/cmr/m/n/10 Interestingly, this cor-re-sponds to a fully padded con-vo-lu- +tion with unit strides. + [] + +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 941. + + +File: pdf/no_padding_no_strides_transposed_00.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/no_padding_no_strides_transposed_00.pdf used on in +put line 952. +(pdftex.def) Requested size: 82.8019pt x 92.74225pt. + + +File: pdf/no_padding_no_strides_transposed_01.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/no_padding_no_strides_transposed_01.pdf used on in +put line 953. +(pdftex.def) Requested size: 82.8019pt x 92.74225pt. + + +File: pdf/no_padding_no_strides_transposed_02.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/no_padding_no_strides_transposed_02.pdf used on in +put line 954. +(pdftex.def) Requested size: 82.8019pt x 92.74225pt. + + +File: pdf/no_padding_no_strides_transposed_03.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/no_padding_no_strides_transposed_03.pdf used on in +put line 955. +(pdftex.def) Requested size: 82.8019pt x 92.74326pt. + + +File: pdf/arbitrary_padding_no_strides_transposed_00.pdf Graphic file (type pdf +) + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_transposed_00.pdf use +d on input line 966. +(pdftex.def) Requested size: 82.8019pt x 89.1252pt. + + +File: pdf/arbitrary_padding_no_strides_transposed_01.pdf Graphic file (type pdf +) + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_transposed_01.pdf use +d on input line 967. +(pdftex.def) Requested size: 82.8019pt x 89.1252pt. + + +File: pdf/arbitrary_padding_no_strides_transposed_02.pdf Graphic file (type pdf +) + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_transposed_02.pdf use +d on input line 968. +(pdftex.def) Requested size: 82.8019pt x 89.1252pt. + + +File: pdf/arbitrary_padding_no_strides_transposed_03.pdf Graphic file (type pdf +) + +Package pdftex.def Info: pdf/arbitrary_padding_no_strides_transposed_03.pdf use +d on input line 969. +(pdftex.def) Requested size: 82.8019pt x 89.1252pt. + + +File: pdf/same_padding_no_strides_transposed_00.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/same_padding_no_strides_transposed_00.pdf used on +input line 981. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/same_padding_no_strides_transposed_01.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/same_padding_no_strides_transposed_01.pdf used on +input line 982. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/same_padding_no_strides_transposed_02.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/same_padding_no_strides_transposed_02.pdf used on +input line 983. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/same_padding_no_strides_transposed_03.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/same_padding_no_strides_transposed_03.pdf used on +input line 984. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 1001. + [22] [23 <./pdf/no_padding_no_strides_transposed_00.pdf> <./pdf/no_padding_no_ +strides_transposed_01.pdf> <./pdf/no_padding_no_strides_transposed_02.pdf> <./p +df/no_padding_no_strides_transposed_03.pdf> <./pdf/arbitrary_padding_no_strides +_transposed_00.pdf> <./pdf/arbitrary_padding_no_strides_transposed_01.pdf> <./p +df/arbitrary_padding_no_strides_transposed_02.pdf> <./pdf/arbitrary_padding_no_ +strides_transposed_03.pdf> <./pdf/same_padding_no_strides_transposed_00.pdf> <. +/pdf/same_padding_no_strides_transposed_01.pdf> <./pdf/same_padding_no_strides_ +transposed_02.pdf> <./pdf/same_padding_no_strides_transposed_03.pdf>] +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 1023. + +Overfull \hbox (1.8501pt too wide) in paragraph at lines 1038--1038 +[]\T1/cmr/bx/n/14.4 No zero padding, non-unit strides, trans-posed + [] + +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 1058. + + +File: pdf/full_padding_no_strides_transposed_00.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/full_padding_no_strides_transposed_00.pdf used on +input line 1073. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/full_padding_no_strides_transposed_01.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/full_padding_no_strides_transposed_01.pdf used on +input line 1074. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/full_padding_no_strides_transposed_02.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/full_padding_no_strides_transposed_02.pdf used on +input line 1075. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/full_padding_no_strides_transposed_03.pdf Graphic file (type pdf) + +Package pdftex.def Info: pdf/full_padding_no_strides_transposed_03.pdf used on +input line 1076. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/no_padding_strides_transposed_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_transposed_00.pdf used on input + line 1087. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/no_padding_strides_transposed_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_transposed_01.pdf used on input + line 1088. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/no_padding_strides_transposed_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_transposed_02.pdf used on input + line 1089. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/no_padding_strides_transposed_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/no_padding_strides_transposed_03.pdf used on input + line 1090. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/padding_strides_transposed_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_transposed_00.pdf used on input li +ne 1102. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/padding_strides_transposed_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_transposed_01.pdf used on input li +ne 1103. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/padding_strides_transposed_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_transposed_02.pdf used on input li +ne 1104. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. + + +File: pdf/padding_strides_transposed_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_transposed_03.pdf used on input li +ne 1105. +(pdftex.def) Requested size: 82.8019pt x 94.26703pt. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 1122. + [24] [25 <./pdf/full_padding_no_strides_transposed_00.pdf> <./pdf/full_padding +_no_strides_transposed_01.pdf> <./pdf/full_padding_no_strides_transposed_02.pdf +> <./pdf/full_padding_no_strides_transposed_03.pdf> <./pdf/no_padding_strides_t +ransposed_00.pdf> <./pdf/no_padding_strides_transposed_01.pdf> <./pdf/no_paddin +g_strides_transposed_02.pdf> <./pdf/no_padding_strides_transposed_03.pdf> <./pd +f/padding_strides_transposed_00.pdf> <./pdf/padding_strides_transposed_01.pdf> +<./pdf/padding_strides_transposed_02.pdf> <./pdf/padding_strides_transposed_03. +pdf>] +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 1143. + + +File: pdf/padding_strides_odd_transposed_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_transposed_00.pdf used on inpu +t line 1162. +(pdftex.def) Requested size: 82.8019pt x 95.44722pt. + + +File: pdf/padding_strides_odd_transposed_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_transposed_01.pdf used on inpu +t line 1163. +(pdftex.def) Requested size: 82.8019pt x 95.44722pt. + + +File: pdf/padding_strides_odd_transposed_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_transposed_02.pdf used on inpu +t line 1164. +(pdftex.def) Requested size: 82.8019pt x 95.44722pt. + + +File: pdf/padding_strides_odd_transposed_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/padding_strides_odd_transposed_03.pdf used on inpu +t line 1165. +(pdftex.def) Requested size: 82.8019pt x 95.44722pt. + [26] [27 <./pdf/padding_strides_odd_transposed_00.pdf> <./pdf/padding_strides_ +odd_transposed_01.pdf> <./pdf/padding_strides_odd_transposed_02.pdf> <./pdf/pad +ding_strides_odd_transposed_03.pdf>] +Chapter 5. +Package mdframed Info: mdframed detected package amsthm + changed the theorem header of amsthm +(mdframed) on input line 1211. + +File: pdf/dilation_00.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/dilation_00.pdf used on input line 1220. +(pdftex.def) Requested size: 82.8019pt x 80.01392pt. + + +File: pdf/dilation_01.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/dilation_01.pdf used on input line 1221. +(pdftex.def) Requested size: 82.8019pt x 80.01392pt. + + +File: pdf/dilation_02.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/dilation_02.pdf used on input line 1222. +(pdftex.def) Requested size: 82.8019pt x 80.01524pt. + + +File: pdf/dilation_03.pdf Graphic file (type pdf) + + +Package pdftex.def Info: pdf/dilation_03.pdf used on input line 1223. +(pdftex.def) Requested size: 82.8019pt x 80.01392pt. + + +LaTeX Warning: `h' float specifier changed to `ht'. + +(./conv_arithmetic.bbl [28 + +] [29 <./pdf/dilation_00.pdf> <./pdf/dilation_01.pdf> <./pdf/dilation_02.pdf> < +./pdf/dilation_03.pdf>] [30 + +]) [31] +Package atveryend Info: Empty hook `BeforeClearDocument' on input line 1235. +Package atveryend Info: Empty hook `AfterLastShipout' on input line 1235. + +(./conv_arithmetic.aux) +Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 1235. +Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 1235. + +Package rerunfilecheck Info: File `conv_arithmetic.out' has not changed. +(rerunfilecheck) Checksum: D91C3423AECCBF9F59457F68F657F613;1586. + + +LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. + + ) +Here is how much of TeX's memory you used: + 27542 strings out of 493089 + 553474 string characters out of 6134842 + 630002 words of memory out of 5000000 + 30347 multiletter control sequences out of 15000+600000 + 29522 words of font info for 74 fonts, out of 8000000 for 9000 + 1141 hyphenation exceptions out of 8191 + 67i,20n,74p,10423b,871s stack positions out of 5000i,500n,10000p,200000b,80000s +{/usr/local/texlive/2015/texmf-dist/fonts/enc/dvips/cm-super/cm-super-t1.enc} +< +/usr/local/texlive/2015/texmf-dist/fonts/type1/public/amsfonts/cm/cmex10.pfb> +Output written on conv_arithmetic.pdf (31 pages, 766592 bytes). +PDF statistics: + 989 PDF objects out of 1000 (max. 8388607) + 817 compressed objects within 9 object streams + 126 named destinations out of 1000 (max. 500000) + 669 words of extra memory for PDF output out of 10000 (max. 10000000) + diff --git a/doc/Articles/cnn/conv_arithmetic.out b/doc/Articles/cnn/conv_arithmetic.out new file mode 100644 index 000000000..fff6ed5c9 --- /dev/null +++ b/doc/Articles/cnn/conv_arithmetic.out @@ -0,0 +1,22 @@ +\BOOKMARK [0][-]{chapter.1}{Introduction}{}% 1 +\BOOKMARK [1][-]{section.1.1}{Discrete convolutions}{chapter.1}% 2 +\BOOKMARK [1][-]{section.1.2}{Pooling}{chapter.1}% 3 +\BOOKMARK [0][-]{chapter.2}{Convolution arithmetic}{}% 4 +\BOOKMARK [1][-]{section.2.1}{No zero padding, unit strides}{chapter.2}% 5 +\BOOKMARK [1][-]{section.2.2}{Zero padding, unit strides}{chapter.2}% 6 +\BOOKMARK [2][-]{subsection.2.2.1}{Half \(same\) padding}{section.2.2}% 7 +\BOOKMARK [2][-]{subsection.2.2.2}{Full padding}{section.2.2}% 8 +\BOOKMARK [1][-]{section.2.3}{No zero padding, non-unit strides}{chapter.2}% 9 +\BOOKMARK [1][-]{section.2.4}{Zero padding, non-unit strides}{chapter.2}% 10 +\BOOKMARK [0][-]{chapter.3}{Pooling arithmetic}{}% 11 +\BOOKMARK [0][-]{chapter.4}{Transposed convolution arithmetic}{}% 12 +\BOOKMARK [1][-]{section.4.1}{Convolution as a matrix operation}{chapter.4}% 13 +\BOOKMARK [1][-]{section.4.2}{Transposed convolution}{chapter.4}% 14 +\BOOKMARK [1][-]{section.4.3}{No zero padding, unit strides, transposed}{chapter.4}% 15 +\BOOKMARK [1][-]{section.4.4}{Zero padding, unit strides, transposed}{chapter.4}% 16 +\BOOKMARK [2][-]{subsection.4.4.1}{Half \(same\) padding, transposed}{section.4.4}% 17 +\BOOKMARK [2][-]{subsection.4.4.2}{Full padding, transposed}{section.4.4}% 18 +\BOOKMARK [1][-]{section.4.5}{No zero padding, non-unit strides, transposed}{chapter.4}% 19 +\BOOKMARK [1][-]{section.4.6}{Zero padding, non-unit strides, transposed}{chapter.4}% 20 +\BOOKMARK [0][-]{chapter.5}{Miscellaneous convolutions}{}% 21 +\BOOKMARK [1][-]{section.5.1}{Dilated convolutions}{chapter.5}% 22 diff --git a/doc/Articles/cnn/conv_arithmetic.toc b/doc/Articles/cnn/conv_arithmetic.toc new file mode 100644 index 000000000..7a9d0c3bb --- /dev/null +++ b/doc/Articles/cnn/conv_arithmetic.toc @@ -0,0 +1,22 @@ +\contentsline {chapter}{\numberline {1}Introduction}{5}{chapter.1} +\contentsline {section}{\numberline {1.1}Discrete convolutions}{6}{section.1.1} +\contentsline {section}{\numberline {1.2}Pooling}{10}{section.1.2} +\contentsline {chapter}{\numberline {2}Convolution arithmetic}{12}{chapter.2} +\contentsline {section}{\numberline {2.1}No zero padding, unit strides}{12}{section.2.1} +\contentsline {section}{\numberline {2.2}Zero padding, unit strides}{13}{section.2.2} +\contentsline {subsection}{\numberline {2.2.1}Half (same) padding}{13}{subsection.2.2.1} +\contentsline {subsection}{\numberline {2.2.2}Full padding}{13}{subsection.2.2.2} +\contentsline {section}{\numberline {2.3}No zero padding, non-unit strides}{15}{section.2.3} +\contentsline {section}{\numberline {2.4}Zero padding, non-unit strides}{15}{section.2.4} +\contentsline {chapter}{\numberline {3}Pooling arithmetic}{18}{chapter.3} +\contentsline {chapter}{\numberline {4}Transposed convolution arithmetic}{19}{chapter.4} +\contentsline {section}{\numberline {4.1}Convolution as a matrix operation}{20}{section.4.1} +\contentsline {section}{\numberline {4.2}Transposed convolution}{20}{section.4.2} +\contentsline {section}{\numberline {4.3}No zero padding, unit strides, transposed}{21}{section.4.3} +\contentsline {section}{\numberline {4.4}Zero padding, unit strides, transposed}{22}{section.4.4} +\contentsline {subsection}{\numberline {4.4.1}Half (same) padding, transposed}{22}{subsection.4.4.1} +\contentsline {subsection}{\numberline {4.4.2}Full padding, transposed}{22}{subsection.4.4.2} +\contentsline {section}{\numberline {4.5}No zero padding, non-unit strides, transposed}{24}{section.4.5} +\contentsline {section}{\numberline {4.6}Zero padding, non-unit strides, transposed}{24}{section.4.6} +\contentsline {chapter}{\numberline {5}Miscellaneous convolutions}{28}{chapter.5} +\contentsline {section}{\numberline {5.1}Dilated convolutions}{28}{section.5.1} diff --git a/doc/Articles/cnn/dill.txt~ b/doc/Articles/cnn/dill.txt~ new file mode 100644 index 000000000..7605c1e85 --- /dev/null +++ b/doc/Articles/cnn/dill.txt~ @@ -0,0 +1,1235 @@ +\documentclass[notitlepage]{report} +\usepackage{amsmath,amsfonts,amsthm,amssymb} +\usepackage{authblk} +\usepackage[T1]{fontenc} +\usepackage{graphicx} +\usepackage[utf8]{inputenc} +\usepackage[framemethod=tikz]{mdframed} +\usepackage{natbib} +\usepackage{subcaption} +\usepackage{tikz} +\usepackage{xcolor} +\usepackage{epigraph} +\usepackage{float} + +\usepackage{hyperref} + + +\definecolor{blue}{RGB}{38,139,210} +\definecolor{cyan}{RGB}{42,161,152} +\definecolor{violet}{RGB}{108,113,196} +\definecolor{red}{RGB}{220,50,47} +\definecolor{base01}{RGB}{88,110,117} +\definecolor{base02}{RGB}{7,54,66} +\definecolor{base03}{RGB}{0,43,54} + +\usetikzlibrary{calc,shapes,positioning} + +\newcommand{\todo}[1]{\textcolor{red}{TODO: #1}} + +\newtheorem{relationship}{Relationship} +\providecommand*{\relationshipautorefname}{Relationship} +\surroundwithmdframed[ + topline=false, + bottomline=false, + middlelinewidth=0.5pt, + linecolor=base01, + roundcorner=5pt, + innertopmargin=0pt, + leftmargin=15pt, + rightmargin=15pt, + nobreak=true, +]{relationship} + +\setcounter{MaxMatrixCols}{16} + +\let\originalepigraph\epigraph +\renewcommand\epigraph[2]{\originalepigraph{\textit{#1}}{\textsc{#2}}} + +% Use arabic numbers for thanks +\makeatletter +\let\@fnsymbol\@arabic +\makeatother + +\title{A guide to convolution arithmetic for deep learning} +\author[$\bigstar$]{Vincent Dumoulin\thanks{dumouliv@iro.umontreal.ca}} +\author[$\bigstar\dagger$]{Francesco Visin\thanks{francesco.visin@polimi.it}} +\affil[$\bigstar$]{MILA, Universit\'{e} de Montr\'{e}al} +\affil[$\dagger$]{AIRLab, Politecnico di Milano} +\date{\today} + +\begin{document} + +\maketitle +\thispagestyle{empty} +\clearpage + +\setlength{\epigraphwidth}{0.4\textwidth} +\epigraph{All models are wrong, but some are useful.}{George E. P. Box} +\clearpage + +\renewcommand{\abstractname}{Acknowledgements} +\begin{abstract} + The authors of this guide would like to thank David Warde-Farley, Guillaume + Alain and Caglar Gulcehre for their valuable feedback. We are likewise + grateful to all those who helped improve this tutorial with helpful + comments, constructive criticisms and code contributions. Keep them coming! + + Special thanks to Ethan Schoonover, creator of the Solarized color + scheme,\footnote{\url{http://ethanschoonover.com/solarized}} whose colors + were used for the figures. +\end{abstract} + +\renewcommand{\abstractname}{Feedback} +\begin{abstract} + Your feedback is welcomed! We did our best to be as precise, informative and + up to the point as possible, but should there be anything you feel might be + an error or could be rephrased to be more precise or comprehensible, please + don't refrain from contacting us. Likewise, drop us a line if you think + there is something that might fit this technical report and you would like + us to discuss -- we will make our best effort to update this document. +\end{abstract} + +\renewcommand{\abstractname}{Source code and animations} +\begin{abstract} + The code used to generate this guide along with its figures is available on + GitHub.\footnote{\url{https://github.com/vdumoulin/conv_arithmetic}} There + the reader can also find an animated version of the figures. +\end{abstract} + +\tableofcontents + +\chapter{Introduction} + +Deep convolutional neural networks (CNNs) have been at the heart of spectacular +advances in deep learning. Although CNNs have been used as early as the nineties +to solve character recognition tasks \citep{le1997reading}, their current +widespread application is due to much more recent work, when a deep CNN was used +to beat state-of-the-art in the ImageNet image classification challenge +\citep{krizhevsky2012imagenet}. + +Convolutional neural networks therefore constitute a very useful tool for +machine learning practitioners. However, learning to use CNNs for the first time +is generally an intimidating experience. A convolutional layer's output shape is +affected by the shape of its input as well as the choice of kernel shape, zero +padding and strides, and the relationship between these properties is not +trivial to infer. This contrasts with fully-connected layers, whose output size +is independent of the input size. Additionally, CNNs also usually feature a {\em +pooling\/} stage, adding yet another level of complexity with respect to +fully-connected networks. Finally, so-called transposed convolutional layers +(also known as fractionally strided convolutional layers) have been employed in +more and more work as of late \citep{zeiler2011adaptive,zeiler2014visualizing, +long2015fully,radford2015unsupervised,visin15,im2016generating}, and their +relationship with convolutional layers has been explained with various degrees +of clarity. + +This guide's objective is twofold: + +\begin{enumerate} + \item Explain the relationship between convolutional layers and transposed + convolutional layers. + \item Provide an intuitive understanding of the relationship between input + shape, kernel shape, zero padding, strides and output shape in + convolutional, pooling and transposed convolutional layers. +\end{enumerate} + +In order to remain broadly applicable, the results shown in this guide are +independent of implementation details and apply to all commonly used machine +learning frameworks, such as Theano +\citep{bergstra2010theano,bastien2012theano}, Torch \citep{collobert2011torch7}, +Tensorflow \citep{abaditensorflow} and Caffe \citep{jia2014caffe}. + +This chapter briefly reviews the main building blocks of CNNs, namely discrete +convolutions and pooling. For an in-depth treatment of the subject, see Chapter +9 of the Deep Learning textbook \citep{Goodfellow-et-al-2016-Book}. + +\section{Discrete convolutions} + +The bread and butter of neural networks is \emph{affine transformations}: a +vector is received as input and is multiplied with a matrix to produce an +output (to which a bias vector is usually added before passing the result +through a nonlinearity). This is applicable to any type of input, be it an +image, a sound clip or an unordered collection of features: whatever their +dimensionality, their representation can always be flattened into a vector +before the transformation. + +Images, sound clips and many other similar kinds of data have an intrinsic +structure. More formally, they share these important properties: + +\begin{itemize} + \item They are stored as multi-dimensional arrays. + \item They feature one or more axes for which ordering matters (e.g., width + and height axes for an image, time axis for a sound clip). + \item One axis, called the channel axis, is used to access different views + of the data (e.g., the red, green and blue channels of a color image, or + the left and right channels of a stereo audio track). +\end{itemize} + +These properties are not exploited when an affine transformation is applied; in +fact, all the axes are treated in the same way and the topological information +is not taken into account. Still, taking advantage of the implicit structure of +the data may prove very handy in solving some tasks, like computer vision and +speech recognition, and in these cases it would be best to preserve it. This is +where discrete convolutions come into play. + +A discrete convolution is a linear transformation that preserves this notion of +ordering. It is sparse (only a few input units contribute to a given output +unit) and reuses parameters (the same weights are applied to multiple locations +in the input). + +\autoref{fig:numerical_no_padding_no_strides} provides an example of a discrete +convolution. The light blue grid is called the {\em input feature map}. To keep +the drawing simple, a single input feature map is represented, but it is not +uncommon to have multiple feature maps stacked one onto another.\footnote{% + An example of this is what was referred to earlier as {\em channels\/} for + images and sound clips.} +A {\em kernel\/} (shaded area) of value + +\begin{figure}[H] + \centering + \begin{tikzpicture}[scale=.4,every node/.style={minimum size=1cm}, on grid] + \draw[fill=base02,opacity=0.4] (0,0) rectangle (3,3); + \draw[draw=base03,thick] (0,0) grid (3,3); + \node (00) at (0.5,2.5) {\tiny 0}; + \node (01) at (1.5,2.5) {\tiny 1}; + \node (02) at (2.5,2.5) {\tiny 2}; + \node (10) at (0.5,1.5) {\tiny 2}; + \node (11) at (1.5,1.5) {\tiny 2}; + \node (12) at (2.5,1.5) {\tiny 0}; + \node (20) at (0.5,0.5) {\tiny 0}; + \node (21) at (1.5,0.5) {\tiny 1}; + \node (22) at (2.5,0.5) {\tiny 2}; + \end{tikzpicture} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_00.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_01.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_02.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_03.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_04.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_05.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_06.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_07.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_no_padding_no_strides_08.pdf} + \caption{\label{fig:numerical_no_padding_no_strides} Computing the output + values of a discrete convolution.} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_00.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_01.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_02.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_03.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_04.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_05.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_06.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_07.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_padding_strides_08.pdf} + \caption{\label{fig:numerical_padding_strides} Computing the output values + of a discrete convolution for $N = 2$, $i_1 = i_2 = 5$, $k_1 = k_2 = 3$, + $s_1 = s_2 = 2$, and $p_1 = p_2 = 1$.} +\end{figure} + +\noindent slides across the input feature map. At each location, the product +between each element of the kernel and the input element it overlaps is computed +and the results are summed up to obtain the output in the current location. The +procedure can be repeated using different kernels to form as many output feature +maps as desired (\autoref{fig:full_picture}). The final outputs of this procedure +are called {\em output feature maps}.\footnote{% + While there is a distinction between convolution and cross-correlation from + a signal processing perspective, the two become interchangeable when the + kernel is learned. For the sake of simplicity and to stay consistent with + most of the machine learning literature, the term {\em convolution\/} + will be used in this guide.} +If there are multiple input feature maps, the kernel will have to be +3-dimensional -- or, equivalently each one of the feature maps will be +convolved with a distinct kernel -- and the resulting feature maps will +be summed up elementwise to produce the output feature map. + +The convolution depicted in \autoref{fig:numerical_no_padding_no_strides} is an +instance of a 2-D convolution, but it can be generalized to N-D convolutions. +For instance, in a 3-D convolution, the kernel would be a {\em cuboid\/} and +would slide across the height, width and depth of the input feature map. + +The collection of kernels defining a discrete convolution has a shape +corresponding to some permutation of $(n, m, k_1, \ldots, k_N)$, where + +\begin{equation*} +\begin{split} + n &\equiv \text{number of output feature maps},\\ + m &\equiv \text{number of input feature maps},\\ + k_j &\equiv \text{kernel size along axis $j$}. +\end{split} +\end{equation*} + +The following properties affect the output size $o_j$ of a convolutional layer +along axis $j$: + +\begin{itemize} + \item $i_j$: input size along axis $j$, + \item $k_j$: kernel size along axis $j$, + \item $s_j$: stride (distance between two consecutive positions of the + kernel) along axis $j$, + \item $p_j$: zero padding (number of zeros concatenated at the beginning and + at the end of an axis) along axis $j$. +\end{itemize} + +\noindent For instance, \autoref{fig:numerical_padding_strides} shows a $3 +\times 3$ kernel applied to a $5 \times 5$ input padded with a $1 \times 1$ +border of zeros using $2 \times 2$ strides. + +Note that strides constitute a form of \emph{subsampling}. As an alternative to +being interpreted as a measure of how much the kernel is translated, strides +can also be viewed as how much of the output is retained. For instance, moving +the kernel by hops of two is equivalent to moving the kernel by hops of one but +retaining only odd output elements (\autoref{fig:strides_subsampling}). + +\begin{figure}[p] + \centering + \begin{tikzpicture}[scale=.35,every node/.style={minimum size=1cm}, on grid] + \begin{scope}[xshift=0cm,yshift=0cm] + \begin{scope}[xshift=0cm,yshift=0cm] + \draw[draw=base03,fill=violet,thick] + (0,0) grid (5,5) rectangle (0,0); + \end{scope} + \begin{scope}[xshift=0.5cm,yshift=0.5cm] + \draw[draw=base03,fill=blue,thick] + (0,0) grid (5,5) rectangle (0,0); + \end{scope} + \end{scope} + \foreach \x in {-10,1,11} {% + \begin{scope}[xshift=\x cm,yshift=10cm] + \begin{scope}[xshift=0cm,yshift=0cm] + \draw[draw=base03,fill=violet,thick] + (0,0) grid (3,3) rectangle (0,0); + \end{scope} + \begin{scope}[xshift=0.5cm,yshift=0.5cm] + \draw[draw=base03,fill=blue,thick] + (0,0) grid (3,3) rectangle (0,0); + \end{scope} + \end{scope} + \begin{scope}[xshift=\x cm,yshift=20cm]\begin{scope}[xshift=0.5cm] + \draw[draw=base03,fill=cyan,thick] + (0,0) grid (3,3) rectangle (0,0); + \end{scope}\end{scope} + } + \begin{scope}[xshift=1cm,yshift=30cm] + \foreach \s in {0.0,0.5,1.0} {% + \begin{scope}[xshift=\s cm,yshift=\s cm] + \draw[draw=base03,fill=cyan,thick] + (0,0) grid (3,3) rectangle (0,0); + \end{scope} + } + \end{scope} + \draw[->, thick] (-0.5,2.5) to (-8.5,9.5); + \draw[->, thick] (3,6) to (3,9.5); + \draw[->, thick] (6,3.5) to (12.5,9.5); + \draw[thick] (-8,14.5) to (-8,16); + \draw[->, thick] (-8,18) to (-8,19.5); + \node[thick] (p1) at (-8,17) {$+$}; + \draw[thick] (3,14.5) to (3,16); + \draw[->, thick] (3,18) to (3,19.5); + \node[thick] (p2) at (3,17) {$+$}; + \draw[thick] (13,14.5) to (13,16); + \draw[->, thick] (13,18) to (13,19.5); + \node[thick] (p3) at (13,17) {$+$}; + \draw[->, thick] (-8,23.5) to (2,29.5); + \draw[->, thick] (3,23.5) to (2.5,29.5); + \draw[->, thick] (13,23.5) to (3,29.5); + \end{tikzpicture} + \caption{\label{fig:full_picture} A convolution mapping from two input + feature maps to three output feature maps using a $3 \times 2 \times 3 + \times 3$ collection of kernels $\mathbf{w}$. In the left pathway, input + feature map 1 is convolved with kernel $\mathbf{w}_{1,1}$ and input + feature map 2 is convolved with kernel $\mathbf{w}_{1,2}$, and the + results are summed together elementwise to form the first output feature + map. The same is repeated for the middle and right pathways to form the + second and third feature maps, and all three output feature maps are + grouped together to form the output.} +\end{figure} + +\begin{figure}[p] + \centering + \begin{tikzpicture}[scale=.35,every node/.style={minimum size=1cm}, on grid] + \begin{scope}[xshift=0,yshift=0cm] + \begin{scope}[xshift=0cm,yshift=0cm] + \draw[draw=base03,fill=blue,thick] (0,0) grid (5,5) rectangle (0,0); + \draw[fill=base02, opacity=0.4] (0,2) rectangle (3,5); + \end{scope} + \begin{scope}[xshift=7cm,yshift=1.5cm] + \draw[draw=base03,fill=cyan,thick] (0,0) grid (2,2) rectangle (0,0); + \end{scope} + \end{scope} + \draw[draw=base03, ->, thick] (2.6,3.5) to (4.5,3.5); + \draw[draw=base03, ->, thick] (1.5,2.4) to (1.5,0.5); + \draw[draw=base03, ->, thick] (5.25, 2.5) to (6.75, 2.5); + \begin{scope}[xshift=12cm,yshift=0cm] + \begin{scope}[xshift=0cm,yshift=0cm] + \draw[draw=base03,fill=blue,thick] (0,0) grid (5,5) rectangle (0,0); + \draw[fill=base02, opacity=0.4] (0,2) rectangle (3,5); + \end{scope} + \begin{scope}[xshift=7cm,yshift=1cm] + \draw[draw=base03,fill=cyan,thick] (0,0) grid (3,3) rectangle (0,0); + \draw[draw=base03] (1,0) -- (2,1) -- (2,0) -- (1,1); + \draw[draw=base03] (0,1) -- (1,2) -- (1,1) -- (0,2); + \draw[draw=base03] (1,1) -- (2,2) -- (2,1) -- (1,2); + \draw[draw=base03] (2,1) -- (3,2) -- (3,1) -- (2,2); + \draw[draw=base03] (1,2) -- (2,3) -- (2,2) -- (1,3); + \end{scope} + \begin{scope}[xshift=12cm,yshift=1.5cm] + \draw[draw=base03,fill=cyan,thick] (0,0) grid (2,2) rectangle (0,0); + \end{scope} + \end{scope} + \draw[draw=base03, ->, thick] (14.6,3.5) to (15.5,3.5); + \draw[draw=base03, ->, thick] (15.6,3.5) to (16.5,3.5); + \draw[draw=base03, ->, thick] (13.5,2.4) to (13.5,1.5); + \draw[draw=base03, ->, thick] (13.5,1.4) to (13.5,0.5); + \draw[draw=base03, ->, thick] (17.25, 2.5) to (18.75, 2.5); + \draw[draw=base03, ->, thick] (22.25, 2.5) to (23.75, 2.5); + \end{tikzpicture} + \caption{\label{fig:strides_subsampling} An alternative way of viewing + strides. Instead of translating the $3 \times 3$ kernel by increments of + $s = 2$ (left), the kernel is translated by increments of $1$ and only + one in $s = 2$ output elements is retained (right).} +\end{figure} + +\section{Pooling} + +In addition to discrete convolutions themselves, {\em pooling\/} operations +make up another important building block in CNNs. Pooling operations reduce +the size of feature maps by using some function to summarize subregions, such +as taking the average or the maximum value. + +Pooling works by sliding a window across the input and feeding the content of +the window to a {\em pooling function}. In some sense, pooling works very much +like a discrete convolution, but replaces the linear combination described by +the kernel with some other function. \autoref{fig:numerical_average_pooling} +provides an example for average pooling, and \autoref{fig:numerical_max_pooling} +does the same for max pooling. + +The following properties affect the output size $o_j$ of a pooling layer +along axis $j$: + +\begin{itemize} + \item $i_j$: input size along axis $j$, + \item $k_j$: pooling window size along axis $j$, + \item $s_j$: stride (distance between two consecutive positions of the + pooling window) along axis $j$. +\end{itemize} + +\begin{figure}[p] + \centering + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_00.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_01.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_02.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_03.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_04.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_05.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_06.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_07.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_average_pooling_08.pdf} + \caption{\label{fig:numerical_average_pooling} Computing the output values + of a $3 \times 3$ average pooling operation on a $5 \times 5$ input + using $1 \times 1$ strides.} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_00.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_01.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_02.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_03.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_04.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_05.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_06.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_07.pdf} + \includegraphics[width=0.32\textwidth]{pdf/numerical_max_pooling_08.pdf} + \caption{\label{fig:numerical_max_pooling} Computing the output values of a + $3 \times 3$ max pooling operation on a $5 \times 5$ input using $1 + \times 1$ strides.} +\end{figure} + +\chapter{Convolution arithmetic} + +The analysis of the relationship between convolutional layer properties is eased +by the fact that they don't interact across axes, i.e., the choice of kernel +size, stride and zero padding along axis $j$ only affects the output size of +axis $j$. Because of that, this chapter will focus on the following simplified +setting: + +\begin{itemize} + \item 2-D discrete convolutions ($N = 2$), + \item square inputs ($i_1 = i_2 = i$), + \item square kernel size ($k_1 = k_2 = k$), + \item same strides along both axes ($s_1 = s_2 = s$), + \item same zero padding along both axes ($p_1 = p_2 = p$). +\end{itemize} + +This facilitates the analysis and the visualization, but keep in mind that the +results outlined here also generalize to the N-D and non-square cases. + +\section{No zero padding, unit strides} + +The simplest case to analyze is when the kernel just slides across every +position of the input (i.e., $s = 1$ and $p = 0$). +\autoref{fig:no_padding_no_strides} provides an example for $i = 4$ and $k = +3$. + +One way of defining the output size in this case is by the number of possible +placements of the kernel on the input. Let's consider the width axis: the kernel +starts on the leftmost part of the input feature map and slides by steps of one +until it touches the right side of the input. The size of the output will be +equal to the number of steps made, plus one, accounting for the initial position +of the kernel (\autoref{fig:no_padding_no_strides_explained}). The same logic +applies for the height axis. + +More formally, the following relationship can be inferred: + +\begin{relationship}\label{rel:no_padding_no_strides} +For any $i$ and $k$, and for $s = 1$ and $p = 0$, +\begin{equation*} + o = (i - k) + 1. +\end{equation*} +\end{relationship} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_03.pdf} + \caption{\label{fig:no_padding_no_strides} (No padding, unit strides) + Convolving a $3 \times 3$ kernel over a $4 \times 4$ input using unit + strides (i.e., $i = 4$, $k = 3$, $s = 1$ and $p = 0$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_03.pdf} + \caption{\label{fig:arbitrary_padding_no_strides} (Arbitrary padding, unit + strides) Convolving a $4 \times 4$ kernel over a $5 \times 5$ input + padded with a $2 \times 2$ border of zeros using unit strides (i.e., + $i = 5$, $k = 4$, $s = 1$ and $p = 2$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_03.pdf} + \caption{\label{fig:same_padding_no_strides} (Half padding, unit strides) + Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half + padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 1$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_03.pdf} + \caption{\label{fig:full_padding_no_strides} (Full padding, unit strides) + Convolving a $3 \times 3$ kernel over a $5 \times 5$ input using full + padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 2$).} +\end{figure} + +\section{Zero padding, unit strides} + +To factor in zero padding (i.e., only restricting to $s = 1$), let's consider +its effect on the effective input size: padding with $p$ zeros changes the +effective input size from $i$ to $i + 2p$. In the general case, +\autoref{rel:no_padding_no_strides} can then be used to infer the following +relationship: + +\begin{relationship}\label{rel:arbitrary_padding_no_strides} +For any $i$, $k$ and $p$, and for $s = 1$, +\begin{equation*} + o = (i - k) + 2p + 1. +\end{equation*} +\end{relationship} + +\noindent \autoref{fig:arbitrary_padding_no_strides} provides an example for $i += 5$, $k = 4$ and $p = 2$. + +In practice, two specific instances of zero padding are used quite extensively +because of their respective properties. Let's discuss them in more detail. + +\subsection{Half (same) padding} + +Having the output size be the same as the input size (i.e., $o = i$) can be a +desirable property: + +\begin{relationship}\label{rel:same_padding_no_strides} +For any $i$ and for $k$ odd ($k = 2n + 1, \quad n \in \mathbb{N}$), $s = 1$ and +$p = \lfloor k / 2 \rfloor = n$, +\begin{equation*} +\begin{split} + o &= i + 2 \lfloor k / 2 \rfloor - (k - 1) \\ + &= i + 2n - 2n \\ + &= i. +\end{split} +\end{equation*} +\end{relationship} + +\noindent This is sometimes referred to as {\em half\/} (or {\em same\/}) +padding. \autoref{fig:same_padding_no_strides} provides an example for +$i = 5$, $k = 3$ and (therefore) $p = 1$. + +\subsection{Full padding} + +While convolving a kernel generally {\em decreases\/} the output size with +respect to the input size, sometimes the opposite is required. This can be +achieved with proper zero padding: + +\begin{relationship}\label{rel:full_padding_no_strides} +For any $i$ and $k$, and for $p = k - 1$ and $s = 1$, +\begin{equation*} +\begin{split} + o &= i + 2(k - 1) - (k - 1) \\ + &= i + (k - 1). +\end{split} +\end{equation*} +\end{relationship} + +\noindent This is sometimes referred to as {\em full\/} padding, because in this +setting every possible partial or complete superimposition of the kernel on the +input feature map is taken into account. \autoref{fig:full_padding_no_strides} +provides an example for $i = 5$, $k = 3$ and (therefore) $p = 2$. + +\section{No zero padding, non-unit strides} + +All relationships derived so far only apply for unit-strided convolutions. +Incorporating non unitary strides requires another inference leap. To +facilitate the analysis, let's momentarily ignore zero padding (i.e., $s > 1$ +and $p = 0$). \autoref{fig:no_padding_strides} provides an example for $i = +5$, $k = 3$ and $s = 2$. + +Once again, the output size can be defined in terms of the number of possible +placements of the kernel on the input. Let's consider the width axis: the +kernel starts as usual on the leftmost part of the input, but this time it +slides by steps of size $s$ until it touches the right side of the input. The +size of the output is again equal to the number of steps made, plus one, +accounting for the initial position of the kernel +(\autoref{fig:no_padding_strides_explained}). The same logic applies for the +height axis. + +From this, the following relationship can be inferred: + +\begin{relationship}\label{rel:no_padding_strides} +For any $i$, $k$ and $s$, and for $p = 0$, +\begin{equation*} + o = \left\lfloor \frac{i - k}{s} \right\rfloor + 1. +\end{equation*} +\end{relationship} + +\noindent The floor function accounts for the fact that sometimes the last +possible step does {\em not\/} coincide with the kernel reaching the end of the +input, i.e., some input units are left out (see +\autoref{fig:padding_strides_odd} for an example of such a case). + +\section{Zero padding, non-unit strides} + +The most general case (convolving over a zero padded input using non-unit +strides) can be derived by applying \autoref{rel:no_padding_strides} on an +effective input of size $i + 2p$, in analogy to what was done for +\autoref{rel:arbitrary_padding_no_strides}: + +\begin{relationship}\label{rel:padding_strides} +For any $i$, $k$, $p$ and $s$, +\begin{equation*} + o = \left\lfloor \frac{i + 2p - k}{s} \right\rfloor + 1. +\end{equation*} +\end{relationship} + +\noindent As before, the floor function means that in some cases a convolution +will produce the same output size for multiple input sizes. More specifically, +if $i + 2p - k$ is a multiple of $s$, then any input size $j = i + a, \quad a +\in \{0,\ldots,s - 1\}$ will produce the same output size. Note that this +ambiguity applies only for $s > 1$. + +\autoref{fig:padding_strides} shows an example with $i = 5$, $k = 3$, $s = 2$ +and $p = 1$, while \autoref{fig:padding_strides_odd} provides an example for +$i = 6$, $k = 3$, $s = 2$ and $p = 1$. Interestingly, despite having different +input sizes these convolutions share the same output size. While this doesn't +affect the analysis for {\em convolutions}, this will complicate the analysis +in the case of {\em transposed convolutions}. + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_03.pdf} + \caption{\label{fig:no_padding_strides} (No zero padding, arbitrary + strides) Convolving a $3 \times 3$ kernel over a $5 \times 5$ input + using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and + $p = 0$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_03.pdf} + \caption{\label{fig:padding_strides} (Arbitrary padding and strides) + Convolving a $3 \times 3$ kernel over a $5 \times 5$ input padded with + a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., + $i = 5$, $k = 3$, $s = 2$ and $p = 1$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_03.pdf} + \caption{\label{fig:padding_strides_odd} (Arbitrary padding and strides) + Convolving a $3 \times 3$ kernel over a $6 \times 6$ input padded with + a $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., + $i = 6$, $k = 3$, $s = 2$ and $p = 1$). In this case, the bottom row + and right column of the zero padded input are not covered by the + kernel.} +\end{figure} + +\begin{figure}[p] + \centering + \begin{subfigure}[t]{0.48\textwidth} + \centering + \begin{tikzpicture}[scale=.35,every node/.style={minimum size=1cm}, + on grid] + \draw[fill=blue] (0,0) rectangle (5,5); + \draw[draw=base03, thick] (0,0) grid (5,5); + \draw[fill=base02, opacity=0.4] (0,2) rectangle (3,5); + \draw[step=10mm, base03, thick] (0,2) grid (3,5); + \draw[draw=base03, ->, thick] (2.6,3.5) to (3.5,3.5); + \draw[draw=base03, ->, thick] (3.6,3.5) to (4.5,3.5); + \draw[draw=base03, ->, thick] (1.5,2.4) to (1.5,1.5); + \draw[draw=base03, ->, thick] (1.5,1.4) to (1.5,0.5); + \end{tikzpicture} + \caption{\label{fig:no_padding_no_strides_explained} The kernel has to + slide two steps to the right to touch the right side of the input + (and equivalently downwards). Adding one to account for the + initial kernel position, the output size is $3 \times 3$.} + \end{subfigure} + ~ + \begin{subfigure}[t]{0.48\textwidth} + \centering + \begin{tikzpicture}[scale=.35,every node/.style={minimum size=1cm}, + on grid] + \draw[fill=blue] (0,0) rectangle (5,5); + \draw[draw=base03, thick] (0,0) grid (5,5); + \draw[fill=base02, opacity=0.4] (0,2) rectangle (3,5); + \draw[step=10mm, base03, thick] (0,2) grid (3,5); + \draw[draw=base03, ->, thick] (2.5,3.5) to (4.5,3.5); + \draw[draw=base03, ->, thick] (1.5,2.5) to (1.5,0.5); + \end{tikzpicture} + \caption{\label{fig:no_padding_strides_explained} The kernel has to + slide one step of size two to the right to touch the right side of + the input (and equivalently downwards). Adding one to account for + the initial kernel position, the output size is $2 \times 2$.} + \end{subfigure} + \caption{Counting kernel positions.} +\end{figure} + +\chapter{Pooling arithmetic} + +In a neural network, pooling layers provide invariance to small translations of +the input. The most common kind of pooling is \emph{max pooling}, which +consists in splitting the input in (usually non-overlapping) patches and +outputting the maximum value of each patch. Other kinds of pooling exist, e.g., +mean or average pooling, which all share the same idea of aggregating the input +locally by applying a non-linearity to the content of some patches \citep{% +boureau-cvpr-10,boureau-icml-10,boureau-iccv-11,ICML2011Saxe_551}. + +Some readers may have noticed that the treatment of convolution arithmetic only +relies on the assumption that some function is repeatedly applied onto subsets +of the input. This means that the relationships derived in the previous chapter +can be reused in the case of pooling arithmetic. Since pooling does not involve +zero padding, the relationship describing the general case is as follows: + +\begin{relationship}\label{rel:pooling} +For any $i$, $k$ and $s$, +\begin{equation*} + o = \left\lfloor \frac{i - k}{s} \right\rfloor + 1. +\end{equation*} +\end{relationship} + +\noindent This relationship holds for any type of pooling. + +\chapter{Transposed convolution arithmetic} + +The need for transposed convolutions generally arises from the desire to use a +transformation going in the opposite direction of a normal convolution, i.e., +from something that has the shape of the output of some convolution to +something that has the shape of its input while maintaining a connectivity +pattern that is compatible with said convolution. For instance, one might use +such a transformation as the decoding layer of a convolutional autoencoder or to +project feature maps to a higher-dimensional space. + +Once again, the convolutional case is considerably more complex than the +fully-connected case, which only requires to use a weight matrix whose shape +has been transposed. However, since every convolution boils down to an +efficient implementation of a matrix operation, the insights gained from the +fully-connected case are useful in solving the convolutional case. + +Like for convolution arithmetic, the dissertation about transposed convolution +arithmetic is simplified by the fact that transposed convolution properties +don't interact across axes. + +The chapter will focus on the following setting: + +\begin{itemize} + \item 2-D transposed convolutions ($N = 2$), + \item square inputs ($i_1 = i_2 = i$), + \item square kernel size ($k_1 = k_2 = k$), + \item same strides along both axes ($s_1 = s_2 = s$), + \item same zero padding along both axes ($p_1 = p_2 = p$). +\end{itemize} + +\noindent Once again, the results outlined generalize to the N-D and non-square +cases. + +\section{Convolution as a matrix operation} + +Take for example the convolution represented in +\autoref{fig:no_padding_no_strides}. If the input and output were to be unrolled +into vectors from left to right, top to bottom, the convolution could be +represented as a sparse matrix $\mathbf{C}$ where the non-zero elements are the +elements $w_{i,j}$ of the kernel (with $i$ and $j$ being the row and column of +the kernel respectively): +\begin{equation*} +\resizebox{.98\hsize}{!}{$ + \begin{pmatrix} + w_{0,0} & w_{0,1} & w_{0,2} & 0 & w_{1,0} & w_{1,1} & w_{1,2} & 0 & + w_{2,0} & w_{2,1} & w_{2,2} & 0 & 0 & 0 & 0 & 0 \\ + 0 & w_{0,0} & w_{0,1} & w_{0,2} & 0 & w_{1,0} & w_{1,1} & w_{1,2} & + 0 & w_{2,0} & w_{2,1} & w_{2,2} & 0 & 0 & 0 & 0 \\ + 0 & 0 & 0 & 0 & w_{0,0} & w_{0,1} & w_{0,2} & 0 & + w_{1,0} & w_{1,1} & w_{1,2} & 0 & w_{2,0} & w_{2,1} & w_{2,2} & 0 \\ + 0 & 0 & 0 & 0 & 0 & w_{0,0} & w_{0,1} & w_{0,2} & + 0 & w_{1,0} & w_{1,1} & w_{1,2} & 0 & w_{2,0} & w_{2,1} & w_{2,2} \\ + \end{pmatrix}$} +\end{equation*} + +This linear operation takes the input matrix flattened as a 16-dimensional +vector and produces a 4-dimensional vector that is later reshaped as the $2 +\times 2$ output matrix. + +Using this representation, the backward pass is easily obtained by transposing +$\mathbf{C}$; in other words, the error is backpropagated by multiplying the +loss with $\mathbf{C}^T$. This operation takes a 4-dimensional vector as input +and produces a 16-dimensional vector as output, and its connectivity pattern is +compatible with $\mathbf{C}$ by construction. + +Notably, the kernel $\mathbf{w}$ defines both the matrices $\mathbf{C}$ and +$\mathbf{C}^T$ used for the forward and backward passes. + +\section{Transposed convolution} + +Let's now consider what would be required to go the other way around, i.e., map +from a 4-dimensional space to a 16-dimensional space, while keeping the +connectivity pattern of the convolution depicted in +\autoref{fig:no_padding_no_strides}. This operation is known as a {\em +transposed convolution}. + +Transposed convolutions -- also called {\em fractionally strided convolutions\/} +or {\em deconvolutions\/}\footnote{The term ``deconvolution'' is sometimes used +in the literature, but we advocate against it on the grounds that a +deconvolution is mathematically defined as the inverse of a convolution, which +is different from a transposed convolution.} -- work by swapping the forward and +backward passes of a convolution. One way to put it is to note that the kernel +defines a convolution, but whether it's a direct convolution or a transposed +convolution is determined by how the forward and backward passes are computed. + +For instance, although the kernel $\mathbf{w}$ defines a convolution whose +forward and backward passes are computed by multiplying with $\mathbf{C}$ and +$\mathbf{C}^T$ respectively, it {\em also\/} defines a transposed convolution +whose forward and backward passes are computed by multiplying with +$\mathbf{C}^T$ and $(\mathbf{C}^T)^T = \mathbf{C}$ respectively.\footnote{The + transposed convolution operation can be thought of as the gradient of {\em + some\/} convolution with respect to its input, which is usually how + transposed convolutions are implemented in practice.} + +Finally note that it is always possible to emulate a transposed convolution with +a direct convolution. The disadvantage is that it usually involves adding many +columns and rows of zeros to the input, resulting in a much less efficient +implementation. + +Building on what has been introduced so far, this chapter will proceed somewhat +backwards with respect to the convolution arithmetic chapter, deriving the +properties of each transposed convolution by referring to the direct +convolution with which it shares the kernel, and defining the equivalent direct +convolution. + +\section{No zero padding, unit strides, transposed} + +The simplest way to think about a transposed convolution on a given input is to +imagine such an input as being the result of a direct convolution applied on +some initial feature map. The trasposed convolution can be then considered as +the operation that allows to recover the \emph{shape}~\footnote{Note that the + transposed convolution does not guarantee to recover the input itself, as it + is not defined as the inverse of the convolution, but rather just returns a + feature map that has the same width and height.} of this initial feature map. + +Let's consider the convolution of a $3 \times 3$ kernel on a $4 \times 4$ +input with unitary stride and no padding (i.e., $i = 4$, $k = 3$, $s = 1$ and +$p = 0$). As depicted in \autoref{fig:no_padding_no_strides}, this produces a +$2 \times 2$ output. The transpose of this convolution will then have an output +of shape $4 \times 4$ when applied on a $2 \times 2$ input. + +Another way to obtain the result of a transposed convolution is to apply an +equivalent -- but much less efficient -- direct convolution. The example +described so far could be tackled by convolving a $3 \times 3$ kernel over a +$2 \times 2$ input padded with a $2 \times 2$ border of zeros using unit +strides (i.e., $i' = 2$, $k' = k$, $s' = 1$ and $p' = 2$), as shown in +\autoref{fig:no_padding_no_strides_transposed}. Notably, the kernel's and +stride's sizes remain the same, but the input of the transposed convolution is +now zero padded.\footnote{Note that although + equivalent to applying the transposed matrix, this visualization adds a lot + of zero multiplications in the form of zero padding. This is done here for + illustration purposes, but it is inefficient, and software implementations + will normally not perform the useless zero multiplications.} + +One way to understand the logic behind zero padding is to consider the +connectivity pattern of the transposed convolution and use it to guide the +design of the equivalent convolution. For example, the top left pixel of the +input of the direct convolution only contribute to the top left pixel of the +output, the top right pixel is only connected to the top right output pixel, +and so on. + +To maintain the same connectivity pattern in the equivalent convolution it is +necessary to zero pad the input in such a way that the first (top-left) +application of the kernel only touches the top-left pixel, i.e., the padding +has to be equal to the size of the kernel minus one. + +Proceeding in the same fashion it is possible to determine similar observations +for the other elements of the image, giving rise to the following relationship: + +\begin{relationship}\label{rel:no_padding_no_strides_transposed} +A convolution described by $s = 1$, $p = 0$ and $k$ has an associated +transposed convolution described by $k' = k$, $s' = s$ and $p' = k - 1$ and its +output size is +\begin{equation*} + o' = i' + (k - 1). +\end{equation*} +\end{relationship} + +Interestingly, this corresponds to a fully padded convolution with unit +strides. + +\section{Zero padding, unit strides, transposed} + +Knowing that the transpose of a non-padded convolution is equivalent to +convolving a zero padded input, it would be reasonable to suppose that the +transpose of a zero padded convolution is equivalent to convolving an input +padded with {\em less\/} zeros. + +It is indeed the case, as shown in +\autoref{fig:arbitrary_padding_no_strides_transposed} for $i = 5$, $k = 4$ and +$p = 2$. + +Formally, the following relationship applies for zero padded convolutions: + +\begin{relationship}\label{rel:arbitrary_padding_no_strides_transposed} +A convolution described by $s = 1$, $k$ and $p$ has an +associated transposed convolution described by $k' = k$, $s' = s$ and $p' = k - +p - 1$ and its output size is +\begin{equation*} + o' = i' + (k - 1) - 2p. +\end{equation*} +\end{relationship} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_transposed_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_transposed_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_transposed_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_no_strides_transposed_03.pdf} + \caption{\label{fig:no_padding_no_strides_transposed} The transpose of + convolving a $3 \times 3$ kernel over a $4 \times 4$ input using unit + strides (i.e., $i = 4$, $k = 3$, $s = 1$ and $p = 0$). It is equivalent + to convolving a $3 \times 3$ kernel over a $2 \times 2$ input padded + with a $2 \times 2$ border of zeros using unit strides (i.e., $i' = 2$, + $k' = k$, $s' = 1$ and $p' = 2$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_transposed_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_transposed_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_transposed_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/arbitrary_padding_no_strides_transposed_03.pdf} + \caption{\label{fig:arbitrary_padding_no_strides_transposed} The transpose + of convolving a $4 \times 4$ kernel over a $5 \times 5$ input padded + with a $2 \times 2$ border of zeros using unit strides (i.e., $i = 5$, + $k = 4$, $s = 1$ and $p = 2$). It is equivalent to convolving a $4 + \times 4$ kernel over a $6 \times 6$ input padded with a $1 \times 1$ + border of zeros using unit strides (i.e., $i' = 6$, $k' = k$, $s' = 1$ + and $p' = 1$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_transposed_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_transposed_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_transposed_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/same_padding_no_strides_transposed_03.pdf} + \caption{\label{fig:same_padding_no_strides_transposed} The transpose of + convolving a $3 \times 3$ kernel over a $5 \times 5$ input using half + padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 1$). + It is equivalent to convolving a $3 \times 3$ kernel over a $5 \times 5$ + input using half padding and unit strides (i.e., $i' = 5$, $k' = k$, $s' + = 1$ and $p' = 1$).} +\end{figure} + +\subsection{Half (same) padding, transposed} + +By applying the same inductive reasoning as before, it is reasonable to expect +that the equivalent convolution of the transpose of a half padded convolution +is itself a half padded convolution, given that the output size of a half +padded convolution is the same as its input size. Thus the following relation +applies: + +\begin{relationship}\label{rel:half_padding_no_strides_transposed} +A convolution described by $k = 2n + 1, \quad n \in \mathbb{N}$, $s = 1$ and $p += \lfloor k / 2 \rfloor = n$ has an associated transposed convolution described +by $k' = k$, $s' = s$ and $p' = p$ and its output size is +\begin{equation*} +\begin{split} + o' &= i' + (k - 1) - 2p \\ + &= i' + 2n - 2n \\ + &= i'. +\end{split} +\end{equation*} +\end{relationship} + +\autoref{fig:same_padding_no_strides_transposed} provides an example for $i = +5$, $k = 3$ and (therefore) $p = 1$. + +\subsection{Full padding, transposed} + +Knowing that the equivalent convolution of the transpose of a non-padded +convolution involves full padding, it is unsurprising that the equivalent of +the transpose of a fully padded convolution is a non-padded convolution: + +\begin{relationship}\label{rel:full_padding_no_strides_transposed} +A convolution described by $s = 1$, $k$ and $p = k - 1$ has an +associated transposed convolution described by $k' = k$, $s' = s$ and $p' = 0$ +and its output size is +\begin{equation*} +\begin{split} + o' &= i' + (k - 1) - 2p \\ + &= i' - (k - 1) +\end{split} +\end{equation*} +\end{relationship} + +\autoref{fig:full_padding_no_strides_transposed} provides an example for $i = +5$, $k = 3$ and (therefore) $p = 2$. + +\section{No zero padding, non-unit strides, transposed} + +Using the same kind of inductive logic as for zero padded convolutions, one +might expect that the transpose of a convolution with $s > 1$ involves an +equivalent convolution with $s < 1$. As will be explained, this is a valid +intuition, which is why transposed convolutions are sometimes called {\em +fractionally strided convolutions}. + +\autoref{fig:no_padding_strides_transposed} provides an example for $i = 5$, $k += 3$ and $s = 2$ which helps understand what fractional strides involve: zeros +are inserted {\em between\/} input units, which makes the kernel move around at +a slower pace than with unit strides.\footnote{Doing so is inefficient and + real-world implementations avoid useless multiplications by zero, but + conceptually it is how the transpose of a strided convolution can be + thought of.} + +For the moment, it will be assumed that the convolution is non-padded ($p = 0$) +and that its input size $i$ is such that $i - k$ is a multiple of $s$. In that +case, the following relationship holds: + +\begin{relationship}\label{rel:no_padding_strides_transposed} +A convolution described by $p = 0$, $k$ and $s$ and whose input +size is such that $i - k$ is a multiple of $s$, has an associated transposed +convolution described by $\tilde{i}'$, $k' = k$, $s' = 1$ and $p' = k - 1$, +where $\tilde{i}'$ is the size of the stretched input obtained by adding +$s - 1$ zeros between each input unit, and its output size is +\begin{equation*} +\begin{split} + o' = s (i' - 1) + k. +\end{split} +\end{equation*} +\end{relationship} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_transposed_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_transposed_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_transposed_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/full_padding_no_strides_transposed_03.pdf} + \caption{\label{fig:full_padding_no_strides_transposed} The transpose of + convolving a $3 \times 3$ kernel over a $5 \times 5$ input using full + padding and unit strides (i.e., $i = 5$, $k = 3$, $s = 1$ and $p = 2$). + It is equivalent to convolving a $3 \times 3$ kernel over a $7 \times 7$ + input using unit strides (i.e., $i' = 7$, $k' = k$, $s' = 1$ and $p' = + 0$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_transposed_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_transposed_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_transposed_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/no_padding_strides_transposed_03.pdf} + \caption{\label{fig:no_padding_strides_transposed} The transpose of + convolving a $3 \times 3$ kernel over a $5 \times 5$ input using $2 + \times 2$ strides (i.e., $i = 5$, $k = 3$, $s = 2$ and $p = 0$). It is + equivalent to convolving a $3 \times 3$ kernel over a $2 \times 2$ input + (with $1$ zero inserted between inputs) padded with a $2 \times 2$ + border of zeros using unit strides (i.e., $i' = 2$, $\tilde{i}' = 3$, $k' + = k$, $s' = 1$ and $p' = 2$).} +\end{figure} + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_transposed_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_transposed_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_transposed_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_transposed_03.pdf} + \caption{\label{fig:padding_strides_transposed} The transpose of convolving + a $3 \times 3$ kernel over a $5 \times 5$ input padded with a $1 \times + 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 5$, $k = 3$, $s + = 2$ and $p = 1$). It is equivalent to convolving a $3 \times 3$ kernel + over a $3 \times 3$ input (with $1$ zero inserted between inputs) padded + with a $1 \times 1$ border of zeros using unit strides (i.e., $i' = 3$, + $\tilde{i}' = 5$, $k' = k$, $s' = 1$ and $p' = 1$).} +\end{figure} + +\section{Zero padding, non-unit strides, transposed} + +When the convolution's input size $i$ is such that $i + 2p - k$ is a multiple +of $s$, the analysis can extended to the zero padded case by combining +\autoref{rel:arbitrary_padding_no_strides_transposed} and +\autoref{rel:no_padding_strides_transposed}: + +\begin{relationship}\label{rel:padding_strides_transposed} +A convolution described by $k$, $s$ and $p$ and whose +input size $i$ is such that $i + 2p - k$ is a multiple of $s$ has an associated +transposed convolution described by $\tilde{i}'$, $k' = k$, $s' = 1$ and +$p' = k - p - 1$, where $\tilde{i}'$ is the size of the stretched input +obtained by adding $s - 1$ zeros between each input unit, and its output size +is +\begin{equation*} +\begin{split} + o' = s (i' - 1) + k - 2p. +\end{split} +\end{equation*} +\end{relationship} + +\autoref{fig:padding_strides_transposed} provides an example for $i = 5$, $k = +3$, $s = 2$ and $p = 1$. + +The constraint on the size of the input $i$ can be relaxed by introducing +another parameter $a \in \{0, \ldots, s - 1\}$ that allows to distinguish +between the $s$ different cases that all lead to the same $i'$: + +\begin{relationship}\label{rel:padding_strides_transposed_odd} +A convolution described by $k$, $s$ and $p$ has an +associated transposed convolution described by $a$, $\tilde{i}'$, $k' = k$, $s' += 1$ and $p' = k - p - 1$, where $\tilde{i}'$ is the size of the stretched +input obtained by adding $s - 1$ zeros between each input unit, and $a = (i + +2p - k) \mod s$ represents the number of zeros added to the bottom and right edges +of the input, and its output size is +\begin{equation*} +\begin{split} + o' = s (i' - 1) + a + k - 2p. +\end{split} +\end{equation*} +\end{relationship} + +\autoref{fig:padding_strides_odd_transposed} provides an example for $i = 6$, $k += 3$, $s = 2$ and $p = 1$. + +\begin{figure}[p] + \centering + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_transposed_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_transposed_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_transposed_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/padding_strides_odd_transposed_03.pdf} + \caption{\label{fig:padding_strides_odd_transposed} The transpose of + convolving a $3 \times 3$ kernel over a $6 \times 6$ input padded with a + $1 \times 1$ border of zeros using $2 \times 2$ strides (i.e., $i = 6$, + $k = 3$, $s = 2$ and $p = 1$). It is equivalent to convolving a $3 + \times 3$ kernel over a $2 \times 2$ input (with $1$ zero inserted + between inputs) padded with a $1 \times 1$ border of zeros (with an + additional border of size $1$ added to the bottom and right edges) using + unit strides (i.e., $i' = 3$, $\tilde{i}' = 5$, $a = 1$, $k' = k$, $s' = + 1$ and $p' = 1$).} +\end{figure} + +\chapter{Miscellaneous convolutions} + +\section{Dilated convolutions} + +Readers familiar with the deep learning literature may have noticed the term +``dilated convolutions'' (or ``atrous convolutions'', from the French expression +{\em convolutions \`{a} trous}) appear in recent papers. Here we attempt to +provide an intuitive understanding of dilated convolutions. For a more in-depth +description and to understand in what contexts they are applied, see +\citet{chen2014semantic,yu2015multi}. + +Dilated convolutions ``inflate'' the kernel by inserting spaces between the +kernel elements. The dilation ``rate'' is controlled by an additional +hyperparameter $d$. Implementations may vary, but there are usually $d - 1$ +spaces inserted between kernel elements such that $d = 1$ corresponds to a +regular convolution. + +Dilated convolutions are used to cheaply increase the receptive field of output +units without increasing the kernel size, which is especially effective +when multiple dilated convolutions are stacked one after another. For a +concrete example, see \citet{oord2016wavenet}, in which the proposed WaveNet +model implements an autoregressive generative model for raw audio which uses +dilated convolutions to condition new audio frames on a large context of past +audio frames. + +To understand the relationship tying the dilation rate $d$ and the output size +$o$, it is useful to think of the impact of $d$ on the {\em effective kernel +size}. A kernel of size $k$ dilated by a factor $d$ has an effective size +\begin{equation*} + \hat{k} = k + (k - 1)(d - 1). +\end{equation*} +This can be combined with \autoref{rel:padding_strides} to form the following +relationship for dilated convolutions: + +\begin{relationship}\label{rel:dilation} +For any $i$, $k$, $p$ and $s$, and for a dilation rate $d$, +\begin{equation*} + o = \left\lfloor \frac{i + 2p - k - (k - 1)(d - 1)}{s} \right\rfloor + 1. +\end{equation*} +\end{relationship} + +\begin{figure}[h] + \centering + \includegraphics[width=0.24\textwidth]{pdf/dilation_00.pdf} + \includegraphics[width=0.24\textwidth]{pdf/dilation_01.pdf} + \includegraphics[width=0.24\textwidth]{pdf/dilation_02.pdf} + \includegraphics[width=0.24\textwidth]{pdf/dilation_03.pdf} + \caption{\label{fig:dilation} (Dilated convolution) + Convolving a $3 \times 3$ kernel over a $7 \times 7$ input with a + dilation factor of 2 (i.e., $i = 7$, $k = 3$, $d = 2$, $s = 1$ and + $p = 0$).} +\end{figure} + +\noindent \autoref{fig:dilation} provides an example for $i = 7$, $k = 3$ and +$d = 2$. + +\bibliography{bibliography} +\bibliographystyle{natbib} +\end{document} diff --git a/doc/Articles/cnn/pdf/padding_strides_08.log b/doc/Articles/cnn/pdf/padding_strides_08.log new file mode 100644 index 000000000..200df8fde --- /dev/null +++ b/doc/Articles/cnn/pdf/padding_strides_08.log @@ -0,0 +1,721 @@ +This is pdfTeX, Version 3.1415926-2.5-1.40.14 (TeX Live 2013/Debian) (format=pdflatex 2017.9.13) 11 JAN 2018 11:57 +entering extended mode + restricted \write18 enabled. + %&-line parsing enabled. +**\documentclass[class=minimal,border=10pt]{standalone} + +*\usepackage[paperwidth=4in]{geometry} +(/usr/share/texlive/texmf-dist/tex/latex/standalone/standalone.cls +Document Class: standalone 2012/09/15 v1.1b Class to compile TeX sub-files stan +dalone +(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/ifluatex.sty +Package: ifluatex 2010/03/01 v1.3 Provides the ifluatex switch (HO) +Package ifluatex Info: LuaTeX not detected. +) +(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/ifpdf.sty +Package: ifpdf 2011/01/30 v2.3 Provides the ifpdf switch (HO) +Package ifpdf Info: pdfTeX in PDF mode is detected. +) +(/usr/share/texlive/texmf-dist/tex/generic/ifxetex/ifxetex.sty +Package: ifxetex 2010/09/12 v0.6 Provides ifxetex conditional +) +(/usr/share/texlive/texmf-dist/tex/latex/xkeyval/xkeyval.sty +Package: xkeyval 2012/10/14 v2.6b package option processing (HA) + +(/usr/share/texlive/texmf-dist/tex/generic/xkeyval/xkeyval.tex +\XKV@toks=\toks14 +\XKV@tempa@toks=\toks15 +\XKV@depth=\count79 +File: xkeyval.tex 2012/10/14 v2.6b key=value parser (HA) + +(/usr/share/texlive/texmf-dist/tex/generic/xkeyval/keyval.tex))) +\sa@internal=\count80 + +(/usr/share/texlive/texmf-dist/tex/latex/standalone/standalone.cfg +File: standalone.cfg 2012/09/15 v1.1b Default configuration file for 'standalon +e' class +) +(/usr/share/texlive/texmf-dist/tex/latex/base/minimal.cls +Document Class: minimal 2001/05/25 Standard LaTeX minimal class +) +\sa@box=\box26 +) +*\usepackage{tikz} +(/usr/share/texlive/texmf-dist/tex/latex/geometry/geometry.sty +Package: geometry 2010/09/12 v5.6 Page Geometry + +(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/ifvtex.sty +Package: ifvtex 2010/03/01 v1.5 Detect VTeX and its facilities (HO) +Package ifvtex Info: VTeX not detected. +) +\Gm@cnth=\count81 +\Gm@cntv=\count82 +\c@Gm@tempcnt=\count83 +\Gm@bindingoffset=\dimen102 +\Gm@wd@mp=\dimen103 +\Gm@odd@mp=\dimen104 +\Gm@even@mp=\dimen105 +\Gm@layoutwidth=\dimen106 +\Gm@layoutheight=\dimen107 +\Gm@layouthoffset=\dimen108 +\Gm@layoutvoffset=\dimen109 +\Gm@dimlist=\toks16 + + +! Package geometry Error: \paperheight (0.0pt) too short. + +See the geometry package documentation for explanation. +Type H for immediate help. + ... + +l.995 \Gm@process + +? \usepackage{xcolor} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \definecolor{blue}{RGB}{38,139,210} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \definecolor{cyan}{RGB}{42,161,152} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \definecolor{base01}{RGB}{88,110,117} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \definecolor{base02}{RGB}{7,54,66} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \definecolor{base03}{RGB}{0,43,54} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \usetikzlibrary{calc,shapes,positioning} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \begin{document} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \begin{tikzpicture}[scale=.5,every node/.style={minimum size=1cm},on grid] +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \begin{scope}[node/.append style={yslant=0.5,xslant=-0.7}, +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? yslant=0.5,xslant=-0.7] +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[step=10mm, base03, dashed, thick] (0,0) grid (7,7); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (1,1) rectangle (2,2); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (1,2) rectangle (2,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (1,3) rectangle (2,4); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (1,4) rectangle (2,5); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (1,5) rectangle (2,6); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (2,1) rectangle (3,2); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (2,2) rectangle (3,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (2,3) rectangle (3,4); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (2,4) rectangle (3,5); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (2,5) rectangle (3,6); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (3,1) rectangle (4,2); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (3,2) rectangle (4,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (3,3) rectangle (4,4); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (3,4) rectangle (4,5); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (3,5) rectangle (4,6); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (4,1) rectangle (5,2); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (4,2) rectangle (5,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (4,3) rectangle (5,4); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (4,4) rectangle (5,5); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (4,5) rectangle (5,6); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (5,1) rectangle (6,2); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (5,2) rectangle (6,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (5,3) rectangle (6,4); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (5,4) rectangle (6,5); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[draw=base03, fill=blue, thick] (5,5) rectangle (6,6); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, E to edit your file, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? +) +* +(/usr/share/texmf/tex/latex/pgf/frontendlayer/tikz.sty +(/usr/share/texmf/tex/latex/pgf/basiclayer/pgf.sty +(/usr/share/texmf/tex/latex/pgf/utilities/pgfrcs.sty +(/usr/share/texmf/tex/generic/pgf/utilities/pgfutil-common.tex +\pgfutil@everybye=\toks17 +) +(/usr/share/texmf/tex/generic/pgf/utilities/pgfutil-latex.def +\pgfutil@abb=\box27 + +(/usr/share/texlive/texmf-dist/tex/latex/ms/everyshi.sty +Package: everyshi 2001/05/15 v3.00 EveryShipout Package (MS) +)) +(/usr/share/texmf/tex/generic/pgf/utilities/pgfrcs.code.tex +Package: pgfrcs 2010/10/25 v2.10 (rcs-revision 1.24) +)) +Package: pgf 2008/01/15 v2.10 (rcs-revision 1.12) + +(/usr/share/texmf/tex/latex/pgf/basiclayer/pgfcore.sty +(/usr/share/texlive/texmf-dist/tex/latex/graphics/graphicx.sty +Package: graphicx 1999/02/16 v1.0f Enhanced LaTeX Graphics (DPC,SPQR) + +(/usr/share/texlive/texmf-dist/tex/latex/graphics/graphics.sty +Package: graphics 2009/02/05 v1.0o Standard LaTeX Graphics (DPC,SPQR) + +(/usr/share/texlive/texmf-dist/tex/latex/graphics/trig.sty +Package: trig 1999/03/16 v1.09 sin cos tan (DPC) +) +(/usr/share/texlive/texmf-dist/tex/latex/latexconfig/graphics.cfg +File: graphics.cfg 2010/04/23 v1.9 graphics configuration of TeX Live +) +Package graphics Info: Driver file: pdftex.def on input line 91. + +(/usr/share/texlive/texmf-dist/tex/latex/pdftex-def/pdftex.def +File: pdftex.def 2011/05/27 v0.06d Graphics/color for pdfTeX + +(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/infwarerr.sty +Package: infwarerr 2010/04/08 v1.3 Providing info/warning/error messages (HO) +) +(/usr/share/texlive/texmf-dist/tex/generic/oberdiek/ltxcmds.sty +Package: ltxcmds 2011/11/09 v1.22 LaTeX kernel commands for general use (HO) +) +\Gread@gobject=\count84 +)) +\Gin@req@height=\dimen110 +\Gin@req@width=\dimen111 +) +(/usr/share/texmf/tex/latex/pgf/systemlayer/pgfsys.sty +(/usr/share/texmf/tex/generic/pgf/systemlayer/pgfsys.code.tex +Package: pgfsys 2010/06/30 v2.10 (rcs-revision 1.37) + +(/usr/share/texmf/tex/generic/pgf/utilities/pgfkeys.code.tex +\pgfkeys@pathtoks=\toks18 +\pgfkeys@temptoks=\toks19 + +(/usr/share/texmf/tex/generic/pgf/utilities/pgfkeysfiltered.code.tex +\pgfkeys@tmptoks=\toks20 +)) +\pgf@x=\dimen112 +\pgf@y=\dimen113 +\pgf@xa=\dimen114 +\pgf@ya=\dimen115 +\pgf@xb=\dimen116 +\pgf@yb=\dimen117 +\pgf@xc=\dimen118 +\pgf@yc=\dimen119 +\w@pgf@writea=\write3 +\r@pgf@reada=\read1 +\c@pgf@counta=\count85 +\c@pgf@countb=\count86 +\c@pgf@countc=\count87 +\c@pgf@countd=\count88 + +(/usr/share/texmf/tex/generic/pgf/systemlayer/pgf.cfg +File: pgf.cfg 2008/05/14 (rcs-revision 1.7) +) +Package pgfsys Info: Driver file for pgf: pgfsys-pdftex.def on input line 900. + +(/usr/share/texmf/tex/generic/pgf/systemlayer/pgfsys-pdftex.def +File: pgfsys-pdftex.def 2009/05/22 (rcs-revision 1.26) + +(/usr/share/texmf/tex/generic/pgf/systemlayer/pgfsys-common-pdf.def +File: pgfsys-common-pdf.def 2008/05/19 (rcs-revision 1.10) +))) +(/usr/share/texmf/tex/generic/pgf/systemlayer/pgfsyssoftpath.code.tex +File: pgfsyssoftpath.code.tex 2008/07/18 (rcs-revision 1.7) +\pgfsyssoftpath@smallbuffer@items=\count89 +\pgfsyssoftpath@bigbuffer@items=\count90 +) +(/usr/share/texmf/tex/generic/pgf/systemlayer/pgfsysprotocol.code.tex +File: pgfsysprotocol.code.tex 2006/10/16 (rcs-revision 1.4) +)) +(/usr/share/texmf/tex/latex/xcolor/xcolor.sty +Package: xcolor 2007/01/21 v2.11 LaTeX color extensions (UK) + +(/usr/share/texlive/texmf-dist/tex/latex/latexconfig/color.cfg +File: color.cfg 2007/01/18 v1.5 color configuration of teTeX/TeXLive +) +Package xcolor Info: Driver file: pdftex.def on input line 225. +Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1337. +Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1341. +Package xcolor Info: Model `RGB' extended on input line 1353. +Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1355. +Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1356. +Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1357. +Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1358. +Package xcolor Info: Model `Gray' substituted by `gray' on input line 1359. +Package xcolor Info: Model `wave' substituted by `hsb' on input line 1360. +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcore.code.tex +Package: pgfcore 2010/04/11 v2.10 (rcs-revision 1.7) + +(/usr/share/texmf/tex/generic/pgf/math/pgfmath.code.tex +(/usr/share/texmf/tex/generic/pgf/math/pgfmathcalc.code.tex +(/usr/share/texmf/tex/generic/pgf/math/pgfmathutil.code.tex) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathparser.code.tex +\pgfmath@dimen=\dimen120 +\pgfmath@count=\count91 +\pgfmath@box=\box28 +\pgfmath@toks=\toks21 +\pgfmath@stack@operand=\toks22 +\pgfmath@stack@operation=\toks23 +) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.code.tex +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.basic.code.tex) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.trigonometric.code.tex) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.random.code.tex) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.comparison.code.tex) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.base.code.tex) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.round.code.tex) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfunctions.misc.code.tex))) +(/usr/share/texmf/tex/generic/pgf/math/pgfmathfloat.code.tex +\c@pgfmathroundto@lastzeros=\count92 +)) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorepoints.code.tex +File: pgfcorepoints.code.tex 2010/04/09 (rcs-revision 1.20) +\pgf@picminx=\dimen121 +\pgf@picmaxx=\dimen122 +\pgf@picminy=\dimen123 +\pgf@picmaxy=\dimen124 +\pgf@pathminx=\dimen125 +\pgf@pathmaxx=\dimen126 +\pgf@pathminy=\dimen127 +\pgf@pathmaxy=\dimen128 +\pgf@xx=\dimen129 +\pgf@xy=\dimen130 +\pgf@yx=\dimen131 +\pgf@yy=\dimen132 +\pgf@zx=\dimen133 +\pgf@zy=\dimen134 +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorepathconstruct.code.tex +File: pgfcorepathconstruct.code.tex 2010/08/03 (rcs-revision 1.24) +\pgf@path@lastx=\dimen135 +\pgf@path@lasty=\dimen136 +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorepathusage.code.tex +File: pgfcorepathusage.code.tex 2008/04/22 (rcs-revision 1.12) +\pgf@shorten@end@additional=\dimen137 +\pgf@shorten@start@additional=\dimen138 +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorescopes.code.tex +File: pgfcorescopes.code.tex 2010/09/08 (rcs-revision 1.34) +\pgfpic=\box29 +\pgf@hbox=\box30 +\pgf@layerbox@main=\box31 +\pgf@picture@serial@count=\count93 +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcoregraphicstate.code.tex +File: pgfcoregraphicstate.code.tex 2008/04/22 (rcs-revision 1.9) +\pgflinewidth=\dimen139 +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcoretransformations.code.tex +File: pgfcoretransformations.code.tex 2009/06/10 (rcs-revision 1.11) +\pgf@pt@x=\dimen140 +\pgf@pt@y=\dimen141 +\pgf@pt@temp=\dimen142 +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorequick.code.tex +File: pgfcorequick.code.tex 2008/10/09 (rcs-revision 1.3) +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcoreobjects.code.tex +File: pgfcoreobjects.code.tex 2006/10/11 (rcs-revision 1.2) +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorepathprocessing.code.tex +File: pgfcorepathprocessing.code.tex 2008/10/09 (rcs-revision 1.8) +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorearrows.code.tex +File: pgfcorearrows.code.tex 2008/04/23 (rcs-revision 1.11) +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcoreshade.code.tex +File: pgfcoreshade.code.tex 2008/11/23 (rcs-revision 1.13) +\pgf@max=\dimen143 +\pgf@sys@shading@range@num=\count94 +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcoreimage.code.tex +File: pgfcoreimage.code.tex 2010/03/25 (rcs-revision 1.16) + +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcoreexternal.code.tex +File: pgfcoreexternal.code.tex 2010/09/01 (rcs-revision 1.17) +\pgfexternal@startupbox=\box32 +)) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorelayers.code.tex +File: pgfcorelayers.code.tex 2010/08/27 (rcs-revision 1.2) +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcoretransparency.code.tex +File: pgfcoretransparency.code.tex 2008/01/17 (rcs-revision 1.2) +) +(/usr/share/texmf/tex/generic/pgf/basiclayer/pgfcorepatterns.code.tex +File: pgfcorepatterns.code.tex 2009/07/02 (rcs-revision 1.3) +))) +(/usr/share/texmf/tex/generic/pgf/modules/pgfmoduleshapes.code.tex +File: pgfmoduleshapes.code.tex 2010/09/09 (rcs-revision 1.13) +\pgfnodeparttextbox=\box33 +) +(/usr/share/texmf/tex/generic/pgf/modules/pgfmoduleplot.code.tex +File: pgfmoduleplot.code.tex 2010/10/22 (rcs-revision 1.8) +) +(/usr/share/texmf/tex/latex/pgf/compatibility/pgfcomp-version-0-65.sty +Package: pgfcomp-version-0-65 2007/07/03 v2.10 (rcs-revision 1.7) +\pgf@nodesepstart=\dimen144 +\pgf@nodesepend=\dimen145 +) +(/usr/share/texmf/tex/latex/pgf/compatibility/pgfcomp-version-1-18.sty +Package: pgfcomp-version-1-18 2007/07/23 v2.10 (rcs-revision 1.1) +)) +(/usr/share/texmf/tex/latex/pgf/utilities/pgffor.sty +(/usr/share/texmf/tex/latex/pgf/utilities/pgfkeys.sty +(/usr/share/texmf/tex/generic/pgf/utilities/pgfkeys.code.tex)) +(/usr/share/texmf/tex/generic/pgf/utilities/pgffor.code.tex +Package: pgffor 2010/03/23 v2.10 (rcs-revision 1.18) +\pgffor@iter=\dimen146 +\pgffor@skip=\dimen147 +\pgffor@stack=\toks24 +\pgffor@toks=\toks25 +)) +(/usr/share/texmf/tex/generic/pgf/frontendlayer/tikz/tikz.code.tex +Package: tikz 2010/10/13 v2.10 (rcs-revision 1.76) + +(/usr/share/texmf/tex/generic/pgf/libraries/pgflibraryplothandlers.code.tex +File: pgflibraryplothandlers.code.tex 2010/05/31 v2.10 (rcs-revision 1.15) +\pgf@plot@mark@count=\count95 +\pgfplotmarksize=\dimen148 +) +\tikz@lastx=\dimen149 +\tikz@lasty=\dimen150 +\tikz@lastxsaved=\dimen151 +\tikz@lastysaved=\dimen152 +\tikzleveldistance=\dimen153 +\tikzsiblingdistance=\dimen154 +\tikz@figbox=\box34 +\tikz@tempbox=\box35 +\tikztreelevel=\count96 +\tikznumberofchildren=\count97 +\tikznumberofcurrentchild=\count98 +\tikz@fig@count=\count99 + +(/usr/share/texmf/tex/generic/pgf/modules/pgfmodulematrix.code.tex +File: pgfmodulematrix.code.tex 2010/08/24 (rcs-revision 1.4) +\pgfmatrixcurrentrow=\count100 +\pgfmatrixcurrentcolumn=\count101 +\pgf@matrix@numberofcolumns=\count102 +) +\tikz@expandcount=\count103 + +(/usr/share/texmf/tex/generic/pgf/frontendlayer/tikz/libraries/tikzlibrarytopat +hs.code.tex +File: tikzlibrarytopaths.code.tex 2008/06/17 v2.10 (rcs-revision 1.2) +))) +(Please type a command or say `\end') +* \foreach \x in { 4,\number\numexpr 4+1,...,\number\numexpr 7-1 } { + +* \foreach \y in { 0,\number\numexpr 0+1,...,\number\numexpr 3-1 } { + + +* \draw[fill=base02, opacity=0.4] (\x,\y) rectangle + +* (\x+1,\y+1); + +* } + +* } +! Undefined control sequence. +\pgffor@body -> \draw + [fill=base02, opacity=0.4] (\x ,\y ) rectangle (\x +1,... +<*> } + +? \draw[step=10mm, base03, thick] (4, 0) grid +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? (7, 3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \coordinate (BL) at (4,0); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \coordinate (BR) at (7,0); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \coordinate (TL) at (4,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \coordinate (TR) at (7,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \end{scope} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \begin{scope}[xshift=-5, yshift=8cm, +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? every node/.append style={yslant=0.5,xslant=-0.7}, +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? yslant=0.5,xslant=-0.7] +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw (BL) -- (2,0) (BR) -- (3,0) +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? (TL) -- (2,1) (TR) -- (3,1); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[fill=cyan] (0,0) rectangle (3,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[step=10mm, base03, thick] (0,0) grid (3,3); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[fill=base02, opacity=0.4] (2,0) rectangle +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? (3,1); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \draw[base03, thick] (2,0) rectangle (3,1); +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \end{scope} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \end{tikzpicture} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? \end{document} +Type to proceed, S to scroll future error messages, +R to run without stopping, Q to run quietly, +I to insert something, +1 or ... or 9 to ignore the next 1 to 9 tokens of input, +H for help, X to quit. +? +! Emergency stop. +\pgffor@body -> \draw + [fill=base02, opacity=0.4] (\x ,\y ) rectangle (\x +1,... +<*> } + +End of file on the terminal! + + +Here is how much of TeX's memory you used: + 9773 strings out of 495028 + 187135 string characters out of 6181497 + 226517 words of memory out of 5000000 + 12831 multiletter control sequences out of 15000+600000 + 3640 words of font info for 14 fonts, out of 8000000 for 9000 + 14 hyphenation exceptions out of 8191 + 55i,0n,56p,385b,98s stack positions out of 5000i,500n,10000p,200000b,80000s +! ==> Fatal error occurred, no output PDF file produced! diff --git a/doc/BookChapters/chapter1.dlog b/doc/BookChapters/chapter1.dlog new file mode 100644 index 000000000..ae2474b0b --- /dev/null +++ b/doc/BookChapters/chapter1.dlog @@ -0,0 +1,17 @@ +Translating doconce text in chapter1.do.txt to ipynb +*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax) + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{cases} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. +Failed to remove ans_at_end environment +Failed to remove sol_at_end environment +output in chapter1.ipynb diff --git a/doc/BookChapters/chapter1.do.txt b/doc/BookChapters/chapter1.do.txt new file mode 100644 index 000000000..fbc0b5b01 --- /dev/null +++ b/doc/BookChapters/chapter1.do.txt @@ -0,0 +1,2489 @@ +======= Linear Regression, basic Elements ======= + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureAug21.mp4?vrtx=view-as-webpage" + + +===== Introduction ===== + + + + + +Our emphasis throughout this series of lectures +is on understanding the mathematical aspects of +different algorithms used in the fields of data analysis and machine learning. + +However, where possible we will emphasize the +importance of using available software. We start thus with a hands-on +and top-down approach to machine learning. The aim is thus to start with +relevant data or data we have produced +and use these to introduce statistical data analysis +concepts and machine learning algorithms before we delve into the +algorithms themselves. The examples we will use in the beginning, start with simple +polynomials with random noise added. We will use the Python +software package "Scikit-Learn":"http://scikit-learn.org/stable/" and +introduce various machine learning algorithms to make fits of +the data and predictions. We move thereafter to more interesting +cases such as data from say experiments (below we will look at experimental nuclear binding energies as an example). +These are examples where we can easily set up the data and +then use machine learning algorithms included in for example +_Scikit-Learn_. + +These examples will serve us the purpose of getting +started. Furthermore, they allow us to catch more than two birds with +a stone. They will allow us to bring in some programming specific +topics and tools as well as showing the power of various Python +libraries for machine learning and statistical data analysis. + +Here, we will mainly focus on two +specific Python packages for Machine Learning, Scikit-Learn and +Tensorflow (see below for links etc). Moreover, the examples we +introduce will serve as inputs to many of our discussions later, as +well as allowing you to set up models and produce your own data and +get started with programming. + + + +===== What is Machine Learning? ===== + +Statistics, data science and machine learning form important fields of +research in modern science. They describe how to learn and make +predictions from data, as well as allowing us to extract important +correlations about physical process and the underlying laws of motion +in large data sets. The latter, big data sets, appear frequently in +essentially all disciplines, from the traditional Science, Technology, +Mathematics and Engineering fields to Life Science, Law, education +research, the Humanities and the Social Sciences. + +It has become more +and more common to see research projects on big data in for example +the Social Sciences where extracting patterns from complicated survey +data is one of many research directions. Having a solid grasp of data +analysis and machine learning is thus becoming central to scientific +computing in many fields, and competences and skills within the fields +of machine learning and scientific computing are nowadays strongly +requested by many potential employers. The latter cannot be +overstated, familiarity with machine learning has almost become a +prerequisite for many of the most exciting employment opportunities, +whether they are in bioinformatics, life science, physics or finance, +in the private or the public sector. This author has had several +students or met students who have been hired recently based on their +skills and competences in scientific computing and data science, often +with marginal knowledge of machine learning. + +Machine learning is a subfield of computer science, and is closely +related to computational statistics. It evolved from the study of +pattern recognition in artificial intelligence (AI) research, and has +made contributions to AI tasks like computer vision, natural language +processing and speech recognition. Many of the methods we will study are also +strongly rooted in basic mathematics and physics research. + +Ideally, machine learning represents the science of giving computers +the ability to learn without being explicitly programmed. The idea is +that there exist generic algorithms which can be used to find patterns +in a broad class of data sets without having to write code +specifically for each problem. The algorithm will build its own logic +based on the data. You should however always keep in mind that +machines and algorithms are to a large extent developed by humans. The +insights and knowledge we have about a specific system, play a central +role when we develop a specific machine learning algorithm. + +Machine learning is an extremely rich field, in spite of its young +age. The increases we have seen during the last three decades in +computational capabilities have been followed by developments of +methods and techniques for analyzing and handling large date sets, +relying heavily on statistics, computer science and mathematics. The +field is rather new and developing rapidly. Popular software packages +written in Python for machine learning like +"Scikit-learn":"http://scikit-learn.org/stable/", +"Tensorflow":"https://www.tensorflow.org/", +"PyTorch":"http://pytorch.org/" and "Keras":"https://keras.io/", all +freely available at their respective GitHub sites, encompass +communities of developers in the thousands or more. And the number of +code developers and contributors keeps increasing. Not all the +algorithms and methods can be given a rigorous mathematical +justification, opening up thereby large rooms for experimenting and +trial and error and thereby exciting new developments. However, a +solid command of linear algebra, multivariate theory, probability +theory, statistical data analysis, understanding errors and Monte +Carlo methods are central elements in a proper understanding of many +of algorithms and methods we will discuss. + + + +The approaches to machine learning are many, but are often split into +two main categories. In *supervised learning* we know the answer to a +problem, and let the computer deduce the logic behind it. On the other +hand, *unsupervised learning* is a method for finding patterns and +relationship in data sets without any prior knowledge of the system. +Some authours also operate with a third category, namely +*reinforcement learning*. This is a paradigm of learning inspired by +behavioral psychology, where learning is achieved by trial-and-error, +solely from rewards and punishment. + +Another way to categorize machine learning tasks is to consider the +desired output of a system. Some of the most common tasks are: + + * Classification: Outputs are divided into two or more classes. The goal is to produce a model that assigns inputs into one of these classes. An example is to identify digits based on pictures of hand-written ones. Classification is typically supervised learning. + + * Regression: Finding a functional relationship between an input data set and a reference data set. The goal is to construct a function that maps input data to continuous output values. + + * Clustering: Data are divided into groups with certain common traits, without knowing the different groups beforehand. It is thus a form of unsupervised learning. + + +The methods we cover have three main topics in common, irrespective of +whether we deal with supervised or unsupervised learning. The first +ingredient is normally our data set (which can be subdivided into +training and test data), the second item is a model which is normally a +function of some parameters. The model reflects our knowledge of the system (or lack thereof). As an example, if we know that our data show a behavior similar to what would be predicted by a polynomial, fitting our data to a polynomial of some degree would then determin our model. + +The last ingredient is a so-called _cost_ +function which allows us to present an estimate on how good our model +is in reproducing the data it is supposed to train. +At the heart of basically all ML algorithms there are so-called minimization algorithms, often we end up with various variants of _gradient_ methods. + + + + + + + +===== Software and needed installations ===== + +We will make extensive use of Python as programming language and its +myriad of available libraries. You will find +Jupyter notebooks invaluable in your work. You can run _R_ +codes in the Jupyter/IPython notebooks, with the immediate benefit of +visualizing your data. You can also use compiled languages like C++, +Rust, Julia, Fortran etc if you prefer. The focus in these lectures will be +on Python. + + +If you have Python installed (we strongly recommend Python3) and you feel +pretty familiar with installing different packages, we recommend that +you install the following Python packages via _pip_ as + +o pip install numpy scipy matplotlib ipython scikit-learn mglearn sympy pandas pillow + +For Python3, replace _pip_ with _pip3_. + +For OSX users we recommend, after having installed Xcode, to +install _brew_. Brew allows for a seamless installation of additional +software via for example + +o brew install python3 + +For Linux users, with its variety of distributions like for example the widely popular Ubuntu distribution, +you can use _pip_ as well and simply install Python as + +o sudo apt-get install python3 (or python for pyhton2.7) + +etc etc. + + + +===== Python installers ===== + +If you don't want to perform these operations separately and venture +into the hassle of exploring how to set up dependencies and paths, we +recommend two widely used distrubutions which set up all relevant +dependencies for Python, namely + +* "Anaconda":"https://docs.anaconda.com/", + +which is an open source +distribution of the Python and R programming languages for large-scale +data processing, predictive analytics, and scientific computing, that +aims to simplify package management and deployment. Package versions +are managed by the package management system _conda_. + +* "Enthought canopy":"https://www.enthought.com/product/canopy/" + +is a Python +distribution for scientific and analytic computing distribution and +analysis environment, available for free and under a commercial +license. + +Furthermore, "Google's Colab":"https://colab.research.google.com/notebooks/welcome.ipynb" is a free Jupyter notebook environment that requires +no setup and runs entirely in the cloud. Try it out! + + +===== Useful Python libraries ===== +Here we list several useful Python libraries we strongly recommend (if you use anaconda many of these are already there) + +* "NumPy":"https://www.numpy.org/" is a highly popular library for large, multi-dimensional arrays and matrices, along with a large collection of high-level mathematical functions to operate on these arrays +* "The pandas":"https://pandas.pydata.org/" library provides high-performance, easy-to-use data structures and data analysis tools +* "Xarray":"http://xarray.pydata.org/en/stable/" is a Python package that makes working with labelled multi-dimensional arrays simple, efficient, and fun! +* "Scipy":"https://www.scipy.org/" (pronounced “Sigh Pie”) is a Python-based ecosystem of open-source software for mathematics, science, and engineering. +* "Matplotlib":"https://matplotlib.org/" is a Python 2D plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms. +* "Autograd":"https://github.com/HIPS/autograd" can automatically differentiate native Python and Numpy code. It can handle a large subset of Python's features, including loops, ifs, recursion and closures, and it can even take derivatives of derivatives of derivatives +* "SymPy":"https://www.sympy.org/en/index.html" is a Python library for symbolic mathematics. +* "scikit-learn":"https://scikit-learn.org/stable/" has simple and efficient tools for machine learning, data mining and data analysis +* "TensorFlow":"https://www.tensorflow.org/" is a Python library for fast numerical computing created and released by Google +* "Keras":"https://keras.io/" is a high-level neural networks API, written in Python and capable of running on top of TensorFlow, CNTK, or Theano +* And many more such as "pytorch":"https://pytorch.org/", "Theano":"https://pypi.org/project/Theano/" etc + + +===== Installing R, C++, cython or Julia ===== + +You will also find it convenient to utilize _R_. We will mainly +use Python during our lectures and in various projects and exercises. +Those of you +already familiar with _R_ should feel free to continue using _R_, keeping +however an eye on the parallel Python set ups. Similarly, if you are a +Python afecionado, feel free to explore _R_ as well. Jupyter/Ipython +notebook allows you to run _R_ codes interactively in your +browser. The software library _R_ is really tailored for statistical data analysis +and allows for an easy usage of the tools and algorithms we will discuss in these +lectures. + +To install _R_ with Jupyter notebook +"follow the link here":"https://mpacer.org/maths/r-kernel-for-ipython-notebook" + + + + +===== Installing R, C++, cython, Numba etc ===== + + +For the C++ aficionados, Jupyter/IPython notebook allows you also to +install C++ and run codes written in this language interactively in +the browser. Since we will emphasize writing many of the algorithms +yourself, you can thus opt for either Python or C++ (or Fortran or other compiled languages) as programming +languages. + +To add more entropy, _cython_ can also be used when running your +notebooks. It means that Python with the jupyter notebook +setup allows you to integrate widely popular softwares and tools for +scientific computing. Similarly, the +"Numba Python package":"https://numba.pydata.org/" delivers increased performance +capabilities with minimal rewrites of your codes. With its +versatility, including symbolic operations, Python offers a unique +computational environment. Your jupyter notebook can easily be +converted into a nicely rendered _PDF_ file or a Latex file for +further processing. For example, convert to latex as + +!bc +pycod jupyter nbconvert filename.ipynb --to latex +!ec + +And to add more versatility, the Python package "SymPy":"http://www.sympy.org/en/index.html" is a Python library for symbolic mathematics. It aims to become a full-featured computer algebra system (CAS) and is entirely written in Python. + +Finally, if you wish to use the light mark-up language +"doconce":"https://github.com/hplgit/doconce" you can convert a standard ascii text file into various HTML +formats, ipython notebooks, latex files, pdf files etc with minimal edits. These lectures were generated using _doconce_. + + + +===== Numpy examples and Important Matrix and vector handling packages ===== + +There are several central software libraries for linear algebra and eigenvalue problems. Several of the more +popular ones have been wrapped into ofter software packages like those from the widely used text _Numerical Recipes_. The original source codes in many of the available packages are often taken from the widely used +software package LAPACK, which follows two other popular packages +developed in the 1970s, namely EISPACK and LINPACK. We describe them shortly here. + + * LINPACK: package for linear equations and least square problems. + * LAPACK:package for solving symmetric, unsymmetric and generalized eigenvalue problems. From LAPACK's website URL: "http://www.netlib.org" it is possible to download for free all source codes from this library. Both C/C++ and Fortran versions are available. + * BLAS (I, II and III): (Basic Linear Algebra Subprograms) are routines that provide standard building blocks for performing basic vector and matrix operations. Blas I is vector operations, II vector-matrix operations and III matrix-matrix operations. Highly parallelized and efficient codes, all available for download from URL: "http://www.netlib.org". + + +===== Basic Matrix Features ===== + +Matrix properties reminder +!bt +\[ + \mathbf{A} = + \begin{bmatrix} a_{11} & a_{12} & a_{13} & a_{14} \\ + a_{21} & a_{22} & a_{23} & a_{24} \\ + a_{31} & a_{32} & a_{33} & a_{34} \\ + a_{41} & a_{42} & a_{43} & a_{44} + \end{bmatrix}\qquad +\mathbf{I} = + \begin{bmatrix} 1 & 0 & 0 & 0 \\ + 0 & 1 & 0 & 0 \\ + 0 & 0 & 1 & 0 \\ + 0 & 0 & 0 & 1 + \end{bmatrix} +\] +!et + + + +The inverse of a matrix is defined by + +!bt +\[ +\mathbf{A}^{-1} \cdot \mathbf{A} = I +\] +!et + + +|----------------------------------------------------------------------| +| Relations | Name | matrix elements | +|----------------------------------------------------------------------| +| $A = A^{T}$ | symmetric | $a_{ij} = a_{ji}$ | +| $A = \left (A^{T} \right )^{-1}$ | real orthogonal | $\sum_k a_{ik} a_{jk} = \sum_k a_{ki} a_{kj} = \delta_{ij}$ | +| $A = A^{ * }$ | real matrix | $a_{ij} = a_{ij}^{ * }$ | +| $A = A^{\dagger}$ | hermitian | $a_{ij} = a_{ji}^{ * }$ | +| $A = \left (A^{\dagger} \right )^{-1}$ | unitary | $\sum_k a_{ik} a_{jk}^{ * } = \sum_k a_{ki}^{ * } a_{kj} = \delta_{ij}$ | +|----------------------------------------------------------------------| + + + +=== Some famous Matrices === + + * Diagonal if $a_{ij}=0$ for $i\ne j$ + * Upper triangular if $a_{ij}=0$ for $i > j$ + * Lower triangular if $a_{ij}=0$ for $i < j$ + * Upper Hessenberg if $a_{ij}=0$ for $i > j+1$ + * Lower Hessenberg if $a_{ij}=0$ for $i < j+1$ + * Tridiagonal if $a_{ij}=0$ for $|i -j| > 1$ + * Lower banded with bandwidth $p$: $a_{ij}=0$ for $i > j+p$ + * Upper banded with bandwidth $p$: $a_{ij}=0$ for $i < j+p$ + * Banded, block upper triangular, block lower triangular.... + + + +=== More Basic Matrix Features === + +Some Equivalent Statements +For an $N\times N$ matrix $\mathbf{A}$ the following properties are all equivalent + + * If the inverse of $\mathbf{A}$ exists, $\mathbf{A}$ is nonsingular. + * The equation $\mathbf{Ax}=0$ implies $\mathbf{x}=0$. + * The rows of $\mathbf{A}$ form a basis of $R^N$. + * The columns of $\mathbf{A}$ form a basis of $R^N$. + * $\mathbf{A}$ is a product of elementary matrices. + * $0$ is not eigenvalue of $\mathbf{A}$. + + + +===== Numpy and arrays ===== +"Numpy":"http://www.numpy.org/" provides an easy way to handle arrays in Python. The standard way to import this library is as + +!bc pycod +import numpy as np +!ec +Here follows a simple example where we set up an array of ten elements, all determined by random numbers drawn according to the normal distribution, +!bc pycod +n = 10 +x = np.random.normal(size=n) +print(x) +!ec +We defined a vector $x$ with $n=10$ elements with its values given by the Normal distribution $N(0,1)$. +Another alternative is to declare a vector as follows +!bc pycod +import numpy as np +x = np.array([1, 2, 3]) +print(x) +!ec +Here we have defined a vector with three elements, with $x_0=1$, $x_1=2$ and $x_2=3$. Note that both Python and C++ +start numbering array elements from $0$ and on. This means that a vector with $n$ elements has a sequence of entities $x_0, x_1, x_2, \dots, x_{n-1}$. We could also let (recommended) Numpy to compute the logarithms of a specific array as +!bc pycod +import numpy as np +x = np.log(np.array([4, 7, 8])) +print(x) +!ec + +In the last example we used Numpy's unary function $np.log$. This function is +highly tuned to compute array elements since the code is vectorized +and does not require looping. We normaly recommend that you use the +Numpy intrinsic functions instead of the corresponding _log_ function +from Python's _math_ module. The looping is done explicitely by the +_np.log_ function. The alternative, and slower way to compute the +logarithms of a vector would be to write + +!bc pycod +import numpy as np +from math import log +x = np.array([4, 7, 8]) +for i in range(0, len(x)): + x[i] = log(x[i]) +print(x) +!ec +We note that our code is much longer already and we need to import the _log_ function from the _math_ module. +The attentive reader will also notice that the output is $[1, 1, 2]$. Python interprets automagically our numbers as integers (like the _automatic_ keyword in C++). To change this we could define our array elements to be double precision numbers as +!bc pycod +import numpy as np +x = np.log(np.array([4, 7, 8], dtype = np.float64)) +print(x) +!ec +or simply write them as double precision numbers (Python uses 64 bits as default for floating point type variables), that is +!bc pycod +import numpy as np +x = np.log(np.array([4.0, 7.0, 8.0]) +print(x) +!ec +To check the number of bytes (remember that one byte contains eight bits for double precision variables), you can use simple use the _itemsize_ functionality (the array $x$ is actually an object which inherits the functionalities defined in Numpy) as +!bc pycod +import numpy as np +x = np.log(np.array([4.0, 7.0, 8.0]) +print(x.itemsize) +!ec + + +===== Matrices in Python ===== + +Having defined vectors, we are now ready to try out matrices. We can +define a $3 \times 3 $ real matrix $\hat{A}$ as (recall that we user +lowercase letters for vectors and uppercase letters for matrices) + +!bc pycod +import numpy as np +A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ])) +print(A) +!ec +If we use the _shape_ function we would get $(3, 3)$ as output, that is verifying that our matrix is a $3\times 3$ matrix. We can slice the matrix and print for example the first column (Python organized matrix elements in a row-major order, see below) as +!bc pycod +import numpy as np +A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ])) +# print the first column, row-major order and elements start with 0 +print(A[:,0]) +!ec +We can continue this was by printing out other columns or rows. The example here prints out the second column +!bc pycod +import numpy as np +A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ])) +# print the first column, row-major order and elements start with 0 +print(A[1,:]) +!ec +Numpy contains many other functionalities that allow us to slice, subdivide etc etc arrays. We strongly recommend that you look up the "Numpy website for more details":"http://www.numpy.org/". Useful functions when defining a matrix are the _np.zeros_ function which declares a matrix of a given dimension and sets all elements to zero +!bc pycod +import numpy as np +n = 10 +# define a matrix of dimension 10 x 10 and set all elements to zero +A = np.zeros( (n, n) ) +print(A) +!ec +or initializing all elements to +!bc pycod +import numpy as np +n = 10 +# define a matrix of dimension 10 x 10 and set all elements to one +A = np.ones( (n, n) ) +print(A) +!ec +or as unitarily distributed random numbers (see the material on random number generators in the statistics part) +!bc pycod +import numpy as np +n = 10 +# define a matrix of dimension 10 x 10 and set all elements to random numbers with x \in [0, 1] +A = np.random.rand(n, n) +print(A) +!ec + +As we will see throughout these lectures, there are several extremely useful functionalities in Numpy. +As an example, consider the discussion of the covariance matrix. Suppose we have defined three vectors +$\hat{x}, \hat{y}, \hat{z}$ with $n$ elements each. The covariance matrix is defined as +!bt +\[ +\hat{\Sigma} = \begin{bmatrix} \sigma_{xx} & \sigma_{xy} & \sigma_{xz} \\ + \sigma_{yx} & \sigma_{yy} & \sigma_{yz} \\ + \sigma_{zx} & \sigma_{zy} & \sigma_{zz} + \end{bmatrix}, +\] +!et +where for example +!bt +\[ +\sigma_{xy} =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\] +!et +The Numpy function _np.cov_ calculates the covariance elements using the factor $1/(n-1)$ instead of $1/n$ since it assumes we do not have the exact mean values. +The following simple function uses the _np.vstack_ function which takes each vector of dimension $1\times n$ and produces a $3\times n$ matrix $\hat{W}$ +!bt +\[ +\hat{W} = \begin{bmatrix} x_0 & y_0 & z_0 \\ + x_1 & y_1 & z_1 \\ + x_2 & y_2 & z_2 \\ + \dots & \dots & \dots \\ + x_{n-2} & y_{n-2} & z_{n-2} \\ + x_{n-1} & y_{n-1} & z_{n-1} + \end{bmatrix}, +\] +!et + +which in turn is converted into into the $3\times 3$ covariance matrix +$\hat{\Sigma}$ via the Numpy function _np.cov()_. We note that we can also calculate +the mean value of each set of samples $\hat{x}$ etc using the Numpy +function _np.mean(x)_. We can also extract the eigenvalues of the +covariance matrix through the _np.linalg.eig()_ function. + +!bc pycod +# Importing various packages +import numpy as np + +n = 100 +x = np.random.normal(size=n) +print(np.mean(x)) +y = 4+3*x+np.random.normal(size=n) +print(np.mean(y)) +z = x**3+np.random.normal(size=n) +print(np.mean(z)) +W = np.vstack((x, y, z)) +Sigma = np.cov(W) +print(Sigma) +Eigvals, Eigvecs = np.linalg.eig(Sigma) +print(Eigvals) +!ec + + +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from scipy import sparse +eye = np.eye(4) +print(eye) +sparse_mtx = sparse.csr_matrix(eye) +print(sparse_mtx) +x = np.linspace(-10,10,100) +y = np.sin(x) +plt.plot(x,y,marker='x') +plt.show() +!ec + + +===== Meet the Pandas ===== + + +#FIGURE: [fig/pandas.jpg, width=600 frac=0.8] + +Another useful Python package is +"pandas":"https://pandas.pydata.org/", which is an open source library +providing high-performance, easy-to-use data structures and data +analysis tools for Python. _pandas_ stands for panel data, a term borrowed from econometrics and is an efficient library for data analysis with an emphasis on tabular data. +_pandas_ has two major classes, the _DataFrame_ class with two-dimensional data objects and tabular data organized in columns and the class _Series_ with a focus on one-dimensional data objects. Both classes allow you to index data easily as we will see in the examples below. +_pandas_ allows you also to perform mathematical operations on the data, spanning from simple reshapings of vectors and matrices to statistical operations. + +The following simple example shows how we can, in an easy way make tables of our data. Here we define a data set which includes names, place of birth and date of birth, and displays the data in an easy to read way. We will see repeated use of _pandas_, in particular in connection with classification of data. + +!bc pycod +import pandas as pd +from IPython.display import display +data = {'First Name': ["Frodo", "Bilbo", "Aragorn II", "Samwise"], + 'Last Name': ["Baggins", "Baggins","Elessar","Gamgee"], + 'Place of birth': ["Shire", "Shire", "Eriador", "Shire"], + 'Date of Birth T.A.': [2968, 2890, 2931, 2980] + } +data_pandas = pd.DataFrame(data) +display(data_pandas) +!ec + +In the above we have imported _pandas_ with the shorthand _pd_, the latter has become the standard way we import _pandas_. We make then a list of various variables +and reorganize the aboves lists into a _DataFrame_ and then print out a neat table with specific column labels as *Name*, *place of birth* and *date of birth*. +Displaying these results, we see that the indices are given by the default numbers from zero to three. +_pandas_ is extremely flexible and we can easily change the above indices by defining a new type of indexing as +!bc pycod +data_pandas = pd.DataFrame(data,index=['Frodo','Bilbo','Aragorn','Sam']) +display(data_pandas) +!ec +Thereafter we display the content of the row which begins with the index _Aragorn_ +!bc pycod +display(data_pandas.loc['Aragorn']) +!ec + +We can easily append data to this, for example +!bc pycod +new_hobbit = {'First Name': ["Peregrin"], + 'Last Name': ["Took"], + 'Place of birth': ["Shire"], + 'Date of Birth T.A.': [2990] + } +data_pandas=data_pandas.append(pd.DataFrame(new_hobbit, index=['Pippin'])) +display(data_pandas) +!ec + + +Here are other examples where we use the _DataFrame_ functionality to handle arrays, now with more interesting features for us, namely numbers. We set up a matrix +of dimensionality $10\times 5$ and compute the mean value and standard deviation of each column. Similarly, we can perform mathematial operations like squaring the matrix elements and many other operations. +!bc pycod +import numpy as np +import pandas as pd +from IPython.display import display +np.random.seed(100) +# setting up a 10 x 5 matrix +rows = 10 +cols = 5 +a = np.random.randn(rows,cols) +df = pd.DataFrame(a) +display(df) +print(df.mean()) +print(df.std()) +display(df**2) +!ec + +Thereafter we can select specific columns only and plot final results +!bc pycod +df.columns = ['First', 'Second', 'Third', 'Fourth', 'Fifth'] +df.index = np.arange(10) + +display(df) +print(df['Second'].mean() ) +print(df.info()) +print(df.describe()) + +from pylab import plt, mpl +plt.style.use('seaborn') +mpl.rcParams['font.family'] = 'serif' + +df.cumsum().plot(lw=2.0, figsize=(10,6)) +plt.show() + + +df.plot.bar(figsize=(10,6), rot=15) +plt.show() +!ec +We can produce a $4\times 4$ matrix +!bc pycod +b = np.arange(16).reshape((4,4)) +print(b) +df1 = pd.DataFrame(b) +print(df1) +!ec +and many other operations. + +The _Series_ class is another important class included in +_pandas_. You can view it as a specialization of _DataFrame_ but where +we have just a single column of data. It shares many of the same features as _DataFrame. As with _DataFrame_, +most operations are vectorized, achieving thereby a high performance when dealing with computations of arrays, in particular labeled arrays. +As we will see below it leads also to a very concice code close to the mathematical operations we may be interested in. +For multidimensional arrays, we recommend strongly "xarray":"http://xarray.pydata.org/en/stable/". _xarray_ has much of the same flexibility as _pandas_, but allows for the extension to higher dimensions than two. We will see examples later of the usage of both _pandas_ and _xarray_. + + + + + + +In order to study various Machine Learning algorithms, we need to +access data. Acccessing data is an essential step in all machine +learning algorithms. In particular, setting up the so-called _design +matrix_ (to be defined below) is often the first element we need in +order to perform our calculations. To set up the design matrix means +reading (and later, when the calculations are done, writing) data +in various formats, The formats span from reading files from disk, +loading data from databases and interacting with online sources +like web application programming interfaces (APIs). + +In handling various input formats, as discussed above, we will mainly stay with _pandas_, +a Python package which allows us, in a seamless and painless way, to +deal with a multitude of formats, from standard _csv_ (comma separated +values) files, via _excel_, _html_ to _hdf5_ formats. With _pandas_ +and the _DataFrame_ and _Series_ functionalities we are able to convert text data +into the calculational formats we need for a specific algorithm. And our code is going to be +pretty close the basic mathematical expressions. + +Our first data set is going to be a classic from nuclear physics, namely all +available data on binding energies. Don't be intimidated if you are not familiar with nuclear physics. It serves simply as an example here of a data set. + +We will show some of the +strengths of packages like _Scikit-Learn_ in fitting nuclear binding energies to +specific functions using linear regression first. Then, as a teaser, we will show you how +you can easily implement other algorithms like decision trees and random forests and neural networks. + +But before we really start with nuclear physics data, let's just look at some simpler polynomial fitting cases, such as, +(don't be offended) fitting straight lines! + + + + +===== Simple linear regression model using _scikit-learn_ ===== + +We start with perhaps our simplest possible example, using _Scikit-Learn_ to perform linear regression analysis on a data set produced by us. + +What follows is a simple Python code where we have defined a function +$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries. +The numbers in the vector $\hat{x}$ are given +by random numbers generated with a uniform distribution with entries +$x_i \in [0,1]$ (more about probability distribution functions +later). These values are then used to define a function $y(x)$ +(tabulated again as a vector) with a linear dependence on $x$ plus a +random noise added via the normal distribution. + + +The Numpy functions are imported used the _import numpy as np_ +statement and the random number generator for the uniform distribution +is called using the function _np.random.rand()_, where we specificy +that we want $100$ random variables. Using Numpy we define +automatically an array with the specified number of elements, $100$ in +our case. With the Numpy function _randn()_ we can compute random +numbers with the normal distribution (mean value $\mu$ equal to zero and +variance $\sigma^2$ set to one) and produce the values of $y$ assuming a linear +dependence as function of $x$ + +!bt +\[ +y = 2x+N(0,1), +\] +!et + +where $N(0,1)$ represents random numbers generated by the normal +distribution. From _Scikit-Learn_ we import then the +_LinearRegression_ functionality and make a prediction $\tilde{y} = +\alpha + \beta x$ using the function _fit(x,y)_. We call the set of +data $(\hat{x},\hat{y})$ for our training data. The Python package +_scikit-learn_ has also a functionality which extracts the above +fitting parameters $\alpha$ and $\beta$ (see below). Later we will +distinguish between training data and test data. + +For plotting we use the Python package +"matplotlib":"https://matplotlib.org/" which produces publication +quality figures. Feel free to explore the extensive +"gallery":"https://matplotlib.org/gallery/index.html" of examples. In +this example we plot our original values of $x$ and $y$ as well as the +prediction _ypredict_ ($\tilde{y}$), which attempts at fitting our +data with a straight line. + +The Python code follows here. +!bc pycod +# Importing various packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression + +x = np.random.rand(100,1) +y = 2*x+np.random.randn(100,1) +linreg = LinearRegression() +linreg.fit(x,y) +xnew = np.array([[0],[1]]) +ypredict = linreg.predict(xnew) + +plt.plot(xnew, ypredict, "r-") +plt.plot(x, y ,'ro') +plt.axis([0,1.0,0, 5.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Simple Linear Regression') +plt.show() +!ec + +This example serves several aims. It allows us to demonstrate several +aspects of data analysis and later machine learning algorithms. The +immediate visualization shows that our linear fit is not +impressive. It goes through the data points, but there are many +outliers which are not reproduced by our linear regression. We could +now play around with this small program and change for example the +factor in front of $x$ and the normal distribution. Try to change the +function $y$ to + +!bt +\[ +y = 10x+0.01 \times N(0,1), +\] +!et + +where $x$ is defined as before. Does the fit look better? Indeed, by +reducing the role of the noise given by the normal distribution we see immediately that +our linear prediction seemingly reproduces better the training +set. However, this testing 'by the eye' is obviouly not satisfactory in the +long run. Here we have only defined the training data and our model, and +have not discussed a more rigorous approach to the _cost_ function. + +We need more rigorous criteria in defining whether we have succeeded or +not in modeling our training data. You will be surprised to see that +many scientists seldomly venture beyond this 'by the eye' approach. A +standard approach for the *cost* function is the so-called $\chi^2$ +function (a variant of the mean-squared error (MSE)) + +!bt +\[ \chi^2 = \frac{1}{n} +\sum_{i=0}^{n-1}\frac{(y_i-\tilde{y}_i)^2}{\sigma_i^2}, +\] +!et + +where $\sigma_i^2$ is the variance (to be defined later) of the entry +$y_i$. We may not know the explicit value of $\sigma_i^2$, it serves +however the aim of scaling the equations and make the cost function +dimensionless. + +Minimizing the cost function is a central aspect of +our discussions to come. Finding its minima as function of the model +parameters ($\alpha$ and $\beta$ in our case) will be a recurring +theme in these series of lectures. Essentially all machine learning +algorithms we will discuss center around the minimization of the +chosen cost function. This depends in turn on our specific +model for describing the data, a typical situation in supervised +learning. Automatizing the search for the minima of the cost function is a +central ingredient in all algorithms. Typical methods which are +employed are various variants of _gradient_ methods. These will be +discussed in more detail later. Again, you'll be surprised to hear that +many practitioners minimize the above function ''by the eye', popularly dubbed as +'chi by the eye'. That is, change a parameter and see (visually and numerically) that +the $\chi^2$ function becomes smaller. + +There are many ways to define the cost function. A simpler approach is to look at the relative difference between the training data and the predicted data, that is we define +the relative error (why would we prefer the MSE instead of the relative error?) as + +!bt +\[ +\epsilon_{\mathrm{relative}}= \frac{\vert \hat{y} -\hat{\tilde{y}}\vert}{\vert \hat{y}\vert}. +\] +!et + +The squared cost function results in an arithmetic mean-unbiased +estimator, and the absolute-value cost function results in a +median-unbiased estimator (in the one-dimensional case, and a +geometric median-unbiased estimator for the multi-dimensional +case). The squared cost function has the disadvantage that it has the tendency +to be dominated by outliers. + +We can modify easily the above Python code and plot the relative error instead +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression + +x = np.random.rand(100,1) +y = 5*x+0.01*np.random.randn(100,1) +linreg = LinearRegression() +linreg.fit(x,y) +ypredict = linreg.predict(x) + +plt.plot(x, np.abs(ypredict-y)/abs(y), "ro") +plt.axis([0,1.0,0.0, 0.5]) +plt.xlabel(r'$x$') +plt.ylabel(r'$\epsilon_{\mathrm{relative}}$') +plt.title(r'Relative error') +plt.show() +!ec + +Depending on the parameter in front of the normal distribution, we may +have a small or larger relative error. Try to play around with +different training data sets and study (graphically) the value of the +relative error. + +As mentioned above, _Scikit-Learn_ has an impressive functionality. +We can for example extract the values of $\alpha$ and $\beta$ and +their error estimates, or the variance and standard deviation and many +other properties from the statistical data analysis. + +Here we show an +example of the functionality of _Scikit-Learn_. +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression +from sklearn.metrics import mean_squared_error, r2_score, mean_squared_log_error, mean_absolute_error + +x = np.random.rand(100,1) +y = 2.0+ 5*x+0.5*np.random.randn(100,1) +linreg = LinearRegression() +linreg.fit(x,y) +ypredict = linreg.predict(x) +print('The intercept alpha: \n', linreg.intercept_) +print('Coefficient beta : \n', linreg.coef_) +# The mean squared error +print("Mean squared error: %.2f" % mean_squared_error(y, ypredict)) +# Explained variance score: 1 is perfect prediction +print('Variance score: %.2f' % r2_score(y, ypredict)) +# Mean squared log error +print('Mean squared log error: %.2f' % mean_squared_log_error(y, ypredict) ) +# Mean absolute error +print('Mean absolute error: %.2f' % mean_absolute_error(y, ypredict)) +plt.plot(x, ypredict, "r-") +plt.plot(x, y ,'ro') +plt.axis([0.0,1.0,1.5, 7.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Linear Regression fit ') +plt.show() + +!ec +The function _coef_ gives us the parameter $\beta$ of our fit while _intercept_ yields +$\alpha$. Depending on the constant in front of the normal distribution, we get values near or far from $alpha =2$ and $\beta =5$. Try to play around with different parameters in front of the normal distribution. The function _meansquarederror_ gives us the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error or loss defined as +!bt +\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n} +\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2, +\] +!et + +The smaller the value, the better the fit. Ideally we would like to +have an MSE equal zero. The attentive reader has probably recognized +this function as being similar to the $\chi^2$ function defined above. + +The _r2score_ function computes $R^2$, the coefficient of +determination. It provides a measure of how well future samples are +likely to be predicted by the model. Best possible score is 1.0 and it +can be negative (because the model can be arbitrarily worse). A +constant model that always predicts the expected value of $\hat{y}$, +disregarding the input features, would get a $R^2$ score of $0.0$. + +If $\tilde{\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as +!bt +\[ +R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2}, +\] +!et +where we have defined the mean value of $\hat{y}$ as +!bt +\[ +\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i. +\] +!et +Another quantity taht we will meet again in our discussions of regression analysis is + the mean absolute error (MAE), a risk metric corresponding to the expected value of the absolute error loss or what we call the $l1$-norm loss. In our discussion above we presented the relative error. +The MAE is defined as follows +!bt +\[ +\text{MAE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n-1} \left| y_i - \tilde{y}_i \right|. +\] +!et +We present the +squared logarithmic (quadratic) error +!bt +\[ +\text{MSLE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n - 1} (\log_e (1 + y_i) - \log_e (1 + \tilde{y}_i) )^2, +\] +!et + +where $\log_e (x)$ stands for the natural logarithm of $x$. This error +estimate is best to use when targets having exponential growth, such +as population counts, average sales of a commodity over a span of +years etc. + + +Finally, another cost function is the Huber cost function used in robust regression. + +The rationale behind this possible cost function is its reduced +sensitivity to outliers in the data set. In our discussions on +dimensionality reduction and normalization of data we will meet other +ways of dealing with outliers. + +The Huber cost function is defined as +!bt +\[ +H_{\delta}(a)={\begin{cases}{\frac {1}{2}}{a^{2}}&{\text{for }}|a|\leq \delta ,\\\delta (|a|-{\frac {1}{2}}\delta ),&{\text{otherwise.}}\end{cases}}}. +\] +!et +Here $a=\bm{y} - \bm{\tilde{y}}$. +We will discuss in more +detail these and other functions in the various lectures. We conclude this part with another example. Instead of +a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn. + +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +import random +from sklearn.linear_model import Ridge +from sklearn.preprocessing import PolynomialFeatures +from sklearn.pipeline import make_pipeline +from sklearn.linear_model import LinearRegression + +x=np.linspace(0.02,0.98,200) +noise = np.asarray(random.sample((range(200)),200)) +y=x**3*noise +yn=x**3*100 +poly3 = PolynomialFeatures(degree=3) +X = poly3.fit_transform(x[:,np.newaxis]) +clf3 = LinearRegression() +clf3.fit(X,y) + +Xplot=poly3.fit_transform(x[:,np.newaxis]) +poly3_plot=plt.plot(x, clf3.predict(Xplot), label='Cubic Fit') +plt.plot(x,yn, color='red', label="True Cubic") +plt.scatter(x, y, label='Data', color='orange', s=15) +plt.legend() +plt.show() + +def error(a): + for i in y: + err=(y-yn)/yn + return abs(np.sum(err))/len(err) + +print (error(y)) +!ec + + + +Let us now dive into nuclear physics and remind ourselves briefly about some basic features about binding +energies. A basic quantity which can be measured for the ground +states of nuclei is the atomic mass $M(N, Z)$ of the neutral atom with +atomic mass number $A$ and charge $Z$. The number of neutrons is $N$. There are indeed several sophisticated experiments worldwide which allow us to measure this quantity to high precision (parts per million even). + +Atomic masses are usually tabulated in terms of the mass excess defined by +!bt +\[ +\Delta M(N, Z) = M(N, Z) - uA, +\] +!et +where $u$ is the Atomic Mass Unit +!bt +\[ +u = M(^{12}\mathrm{C})/12 = 931.4940954(57) \hspace{0.1cm} \mathrm{MeV}/c^2. +\] +!et +The nucleon masses are +!bt +\[ +m_p = 1.00727646693(9)u, +\] +!et +and +!bt +\[ +m_n = 939.56536(8)\hspace{0.1cm} \mathrm{MeV}/c^2 = 1.0086649156(6)u. +\] +!et + +In the "2016 mass evaluation of by W.J.Huang, G.Audi, M.Wang, F.G.Kondev, S.Naimi and X.Xu":"http://nuclearmasses.org/resources_folder/Wang_2017_Chinese_Phys_C_41_030003.pdf" +there are data on masses and decays of 3437 nuclei. + +The nuclear binding energy is defined as the energy required to break +up a given nucleus into its constituent parts of $N$ neutrons and $Z$ +protons. In terms of the atomic masses $M(N, Z)$ the binding energy is +defined by + + +!bt +\[ +BE(N, Z) = ZM_H c^2 + Nm_n c^2 - M(N, Z)c^2 , +\] +!et +where $M_H$ is the mass of the hydrogen atom and $m_n$ is the mass of the neutron. +In terms of the mass excess the binding energy is given by +!bt +\[ +BE(N, Z) = Z\Delta_H c^2 + N\Delta_n c^2 -\Delta(N, Z)c^2 , +\] +!et +where $\Delta_H c^2 = 7.2890$ MeV and $\Delta_n c^2 = 8.0713$ MeV. + + +A popular and physically intuitive model which can be used to parametrize +the experimental binding energies as function of $A$, is the so-called +_liquid drop model_. The ansatz is based on the following expression + +!bt +\[ +BE(N,Z) = a_1A-a_2A^{2/3}-a_3\frac{Z^2}{A^{1/3}}-a_4\frac{(N-Z)^2}{A}, +\] +!et + +where $A$ stands for the number of nucleons and the $a_i$s are parameters which are determined by a fit +to the experimental data. + + + + +To arrive at the above expression we have assumed that we can make the following assumptions: + + * There is a volume term $a_1A$ proportional with the number of nucleons (the energy is also an extensive quantity). When an assembly of nucleons of the same size is packed together into the smallest volume, each interior nucleon has a certain number of other nucleons in contact with it. This contribution is proportional to the volume. + + * There is a surface energy term $a_2A^{2/3}$. The assumption here is that a nucleon at the surface of a nucleus interacts with fewer other nucleons than one in the interior of the nucleus and hence its binding energy is less. This surface energy term takes that into account and is therefore negative and is proportional to the surface area. + + + * There is a Coulomb energy term $a_3\frac{Z^2}{A^{1/3}}$. The electric repulsion between each pair of protons in a nucleus yields less binding. + + * There is an asymmetry term $a_4\frac{(N-Z)^2}{A}$. This term is associated with the Pauli exclusion principle and reflects the fact that the proton-neutron interaction is more attractive on the average than the neutron-neutron and proton-proton interactions. + +We could also add a so-called pairing term, which is a correction term that +arises from the tendency of proton pairs and neutron pairs to +occur. An even number of particles is more stable than an odd number. + + +=== Organizing our data === + +Let us start with reading and organizing our data. +We start with the compilation of masses and binding energies from 2016. +After having downloaded this file to our own computer, we are now ready to read the file and start structuring our data. + + +We start with preparing folders for storing our calculations and the data file over masses and binding energies. We import also various modules that we will find useful in order to present various Machine Learning methods. Here we focus mainly on the functionality of _scikit-learn_. +!bc pycod +# Common imports +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import sklearn.linear_model as skl +from sklearn.model_selection import train_test_split +from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error +import os + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("MassEval2016.dat"),'r') +!ec + + +Before we proceed, we define also a function for making our plots. You can obviously avoid this and simply set up various _matplotlib_ commands every time you need them. You may however find it convenient to collect all such commands in one function and simply call this function. +!bc pycod +from pylab import plt, mpl +plt.style.use('seaborn') +mpl.rcParams['font.family'] = 'serif' + +def MakePlot(x,y, styles, labels, axlabels): + plt.figure(figsize=(10,6)) + for i in range(len(x)): + plt.plot(x[i], y[i], styles[i], label = labels[i]) + plt.xlabel(axlabels[0]) + plt.ylabel(axlabels[1]) + plt.legend(loc=0) +!ec + +Our next step is to read the data on experimental binding energies and +reorganize them as functions of the mass number $A$, the number of +protons $Z$ and neutrons $N$ using _pandas_. Before we do this it is +always useful (unless you have a binary file or other types of compressed +data) to actually open the file and simply take a look at it! + + +In particular, the program that outputs the final nuclear masses is written in Fortran with a specific format. It means that we need to figure out the format and which columns contain the data we are interested in. Pandas comes with a function that reads formatted output. After having admired the file, we are now ready to start massaging it with _pandas_. The file begins with some basic format information. +!bc pycod +""" +This is taken from the data file of the mass 2016 evaluation. +All files are 3436 lines long with 124 character per line. + Headers are 39 lines long. + col 1 : Fortran character control: 1 = page feed 0 = line feed + format : a1,i3,i5,i5,i5,1x,a3,a4,1x,f13.5,f11.5,f11.3,f9.3,1x,a2,f11.3,f9.3,1x,i3,1x,f12.5,f11.5 + These formats are reflected in the pandas widths variable below, see the statement + widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1), + Pandas has also a variable header, with length 39 in this case. +""" +!ec + +The data we are interested in are in columns 2, 3, 4 and 11, giving us +the number of neutrons, protons, mass numbers and binding energies, +respectively. We add also for the sake of completeness the element name. The data are in fixed-width formatted lines and we will +covert them into the _pandas_ DataFrame structure. + +!bc pycod +# Read the experimental data with Pandas +Masses = pd.read_fwf(infile, usecols=(2,3,4,6,11), + names=('N', 'Z', 'A', 'Element', 'Ebinding'), + widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1), + header=39, + index_col=False) + +# Extrapolated values are indicated by '#' in place of the decimal place, so +# the Ebinding column won't be numeric. Coerce to float and drop these entries. +Masses['Ebinding'] = pd.to_numeric(Masses['Ebinding'], errors='coerce') +Masses = Masses.dropna() +# Convert from keV to MeV. +Masses['Ebinding'] /= 1000 + +# Group the DataFrame by nucleon number, A. +Masses = Masses.groupby('A') +# Find the rows of the grouped DataFrame with the maximum binding energy. +Masses = Masses.apply(lambda t: t[t.Ebinding==t.Ebinding.max()]) +!ec + +We have now read in the data, grouped them according to the variables we are interested in. +We see how easy it is to reorganize the data using _pandas_. If we +were to do these operations in C/C++ or Fortran, we would have had to +write various functions/subroutines which perform the above +reorganizations for us. Having reorganized the data, we can now start +to make some simple fits using both the functionalities in _numpy_ and +_Scikit-Learn_ afterwards. + +Now we define five variables which contain +the number of nucleons $A$, the number of protons $Z$ and the number of neutrons $N$, the element name and finally the energies themselves. +!bc pycod +A = Masses['A'] +Z = Masses['Z'] +N = Masses['N'] +Element = Masses['Element'] +Energies = Masses['Ebinding'] +print(Masses) +!ec +The next step, and we will define this mathematically later, is to set up the so-called _design matrix_. We will throughout call this matrix $\bm{X}$. +It has dimensionality $p\times n$, where $n$ is the number of data points and $p$ are the so-called predictors. In our case here they are given by the number of polynomials in $A$ we wish to include in the fit. +!bc pycod +# Now we set up the design matrix X +X = np.zeros((len(A),5)) +X[:,0] = 1 +X[:,1] = A +X[:,2] = A**(2.0/3.0) +X[:,3] = A**(-1.0/3.0) +X[:,4] = A**(-1.0) +!ec +With _scikitlearn_ we are now ready to use linear regression and fit our data. +!bc pycod +clf = skl.LinearRegression().fit(X, Energies) +fity = clf.predict(X) +!ec +Pretty simple! +Now we can print measures of how our fit is doing, the coefficients from the fits and plot the final fit together with our data. +!bc pycod +# The mean squared error +print("Mean squared error: %.2f" % mean_squared_error(Energies, fity)) +# Explained variance score: 1 is perfect prediction +print('Variance score: %.2f' % r2_score(Energies, fity)) +# Mean absolute error +print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity)) +print(clf.coef_, clf.intercept_) + +Masses['Eapprox'] = fity +# Generate a plot comparing the experimental with the fitted values values. +fig, ax = plt.subplots() +ax.set_xlabel(r'$A = N + Z$') +ax.set_ylabel(r'$E_\mathrm{bind}\,/\mathrm{MeV}$') +ax.plot(Masses['A'], Masses['Ebinding'], alpha=0.7, lw=2, + label='Ame2016') +ax.plot(Masses['A'], Masses['Eapprox'], alpha=0.7, lw=2, c='m', + label='Fit') +ax.legend() +save_fig("Masses2016") +plt.show() +!ec + + +As a teaser, let us now see how we can do this with decision trees using _scikit-learn_. Later we will switch to so-called _random forests_! + + +!bc pycod + +#Decision Tree Regression +from sklearn.tree import DecisionTreeRegressor +regr_1=DecisionTreeRegressor(max_depth=5) +regr_2=DecisionTreeRegressor(max_depth=7) +regr_3=DecisionTreeRegressor(max_depth=9) +regr_1.fit(X, Energies) +regr_2.fit(X, Energies) +regr_3.fit(X, Energies) + + +y_1 = regr_1.predict(X) +y_2 = regr_2.predict(X) +y_3=regr_3.predict(X) +Masses['Eapprox'] = y_3 +# Plot the results +plt.figure() +plt.plot(A, Energies, color="blue", label="Data", linewidth=2) +plt.plot(A, y_1, color="red", label="max_depth=5", linewidth=2) +plt.plot(A, y_2, color="green", label="max_depth=7", linewidth=2) +plt.plot(A, y_3, color="m", label="max_depth=9", linewidth=2) + +plt.xlabel("$A$") +plt.ylabel("$E$[MeV]") +plt.title("Decision Tree Regression") +plt.legend() +save_fig("Masses2016Trees") +plt.show() +print(Masses) +print(np.mean( (Energies-y_1)**2)) +!ec + + +The _seaborn_ package allows us to visualize data in an efficient way. Note that we use _scikit-learn_'s multi-layer perceptron (or feed forward neural network) +functionality. +!bc pycod +from sklearn.neural_network import MLPRegressor +from sklearn.metrics import accuracy_score +import seaborn as sns + +X_train = X +Y_train = Energies +n_hidden_neurons = 100 +epochs = 100 +# store models for later use +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store the models for later use +DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +sns.set() +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = MLPRegressor(hidden_layer_sizes=(n_hidden_neurons), activation='logistic', + alpha=lmbd, learning_rate_init=eta, max_iter=epochs) + dnn.fit(X_train, Y_train) + DNN_scikit[i][j] = dnn + train_accuracy[i][j] = dnn.score(X_train, Y_train) + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + + + +!ec + + + +===== Linear Regression, basic elements ===== + + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureAug27.mp4?vrtx=view-as-webpage". + + +Fitting a continuous function with linear parameterization in terms of the parameters $\bm{\beta}$. +* Method of choice for fitting a continuous function! +* Gives an excellent introduction to central Machine Learning features with _understandable pedagogical_ links to other methods like _Neural Networks_, _Support Vector Machines_ etc +* Analytical expression for the fitting parameters $\bm{\beta}$ +* Analytical expressions for statistical propertiers like mean values, variances, confidence intervals and more +* Analytical relation with probabilistic interpretations +* Easy to introduce basic concepts like bias-variance tradeoff, cross-validation, resampling and regularization techniques and many other ML topics +* Easy to code! And links well with classification problems and logistic regression and neural networks +* Allows for _easy_ hands-on understanding of gradient descent methods +* and many more features + +For more discussions of Ridge and Lasso regression, "Wessel van Wieringen's":"https://arxiv.org/abs/1509.09169" article is highly recommended. +Similarly, "Mehta et al's article":"https://arxiv.org/abs/1803.08823" is also recommended. + + + +Regression modeling deals with the description of the sampling distribution of a given random variable $y$ and how it varies as function of another variable or a set of such variables $\bm{x} =[x_0, x_1,\dots, x_{n-1}]^T$. +The first variable is called the _dependent_, the _outcome_ or the _response_ variable while the set of variables $\bm{x}$ is called the independent variable, or the predictor variable or the explanatory variable. + +A regression model aims at finding a likelihood function $p(\bm{y}\vert \bm{x})$, that is the conditional distribution for $\bm{y}$ with a given $\bm{x}$. The estimation of $p(\bm{y}\vert \bm{x})$ is made using a data set with +* $n$ cases $i = 0, 1, 2, \dots, n-1$ +* Response (target, dependent or outcome) variable $y_i$ with $i = 0, 1, 2, \dots, n-1$ +* $p$ so-called explanatory (independent or predictor) variables $\bm{x}_i=[x_{i0}, x_{i1}, \dots, x_{ip-1}]$ with $i = 0, 1, 2, \dots, n-1$ and explanatory variables running from $0$ to $p-1$. See below for more explicit examples. + The goal of the regression analysis is to extract/exploit relationship between $\bm{y}$ and $\bm{x}$ in or to infer causal dependencies, approximations to the likelihood functions, functional relationships and to make predictions, making fits and many other things. + + +Consider an experiment in which $p$ characteristics of $n$ samples are +measured. The data from this experiment, for various explanatory variables $p$ are normally represented by a matrix +$\mathbf{X}$. + +The matrix $\mathbf{X}$ is called the *design +matrix*. Additional information of the samples is available in the +form of $\bm{y}$ (also as above). The variable $\bm{y}$ is +generally referred to as the *response variable*. The aim of +regression analysis is to explain $\bm{y}$ in terms of +$\bm{X}$ through a functional relationship like $y_i = +f(\mathbf{X}_{i,\ast})$. When no prior knowledge on the form of +$f(\cdot)$ is available, it is common to assume a linear relationship +between $\bm{X}$ and $\bm{y}$. This assumption gives rise to +the *linear regression model* where $\bm{\beta} = [\beta_0, \ldots, +\beta_{p-1}]^{T}$ are the *regression parameters*. + +Linear regression gives us a set of analytical equations for the parameters $\beta_j$. + + +In order to understand the relation among the predictors $p$, the set of data $n$ and the target (outcome, output etc) $\bm{y}$, +consider the model we discussed for describing nuclear binding energies. + +There we assumed that we could parametrize the data using a polynomial approximation based on the liquid drop model. +Assuming +!bt +\[ +BE(A) = a_0+a_1A+a_2A^{2/3}+a_3A^{-1/3}+a_4A^{-1}, +\] +!et +we have five predictors, that is the intercept, the $A$ dependent term, the $A^{2/3}$ term and the $A^{-1/3}$ and $A^{-1}$ terms. +This gives $p=0,1,2,3,4$. Furthermore we have $n$ entries for each predictor. It means that our design matrix is a +$p\times n$ matrix $\bm{X}$. + +Here the predictors are based on a model we have made. A popular data set which is widely encountered in ML applications is the +so-called "credit card default data from Taiwan":"https://www.sciencedirect.com/science/article/pii/S0957417407006719?via%3Dihub". The data set contains data on $n=30000$ credit card holders with predictors like gender, marital status, age, profession, education, etc. In total there are $24$ such predictors or attributes leading to a design matrix of dimensionality $24 \times 30000$. This is however a classification problem and we will come back to it when we discuss Logistic Regression. + + +Before we proceed let us study a case from linear algebra where we aim at fitting a set of data $\bm{y}=[y_0,y_1,\dots,y_{n-1}]$. We could think of these data as a result of an experiment or a complicated numerical experiment. These data are functions of a series of variables $\bm{x}=[x_0,x_1,\dots,x_{n-1}]$, that is $y_i = y(x_i)$ with $i=0,1,2,\dots,n-1$. The variables $x_i$ could represent physical quantities like time, temperature, position etc. We assume that $y(x)$ is a smooth function. + +Since obtaining these data points may not be trivial, we want to use these data to fit a function which can allow us to make predictions for values of $y$ which are not in the present set. The perhaps simplest approach is to assume we can parametrize our function in terms of a polynomial of degree $n-1$ with $n$ points, that is +!bt +\[ +y=y(x) \rightarrow y(x_i)=\tilde{y}_i+\epsilon_i=\sum_{j=0}^{n-1} \beta_j x_i^j+\epsilon_i, +\] +!et +where $\epsilon_i$ is the error in our approximation. + + +For every set of values $y_i,x_i$ we have thus the corresponding set of equations +!bt +\begin{align*} +y_0&=\beta_0+\beta_1x_0^1+\beta_2x_0^2+\dots+\beta_{n-1}x_0^{n-1}+\epsilon_0\\ +y_1&=\beta_0+\beta_1x_1^1+\beta_2x_1^2+\dots+\beta_{n-1}x_1^{n-1}+\epsilon_1\\ +y_2&=\beta_0+\beta_1x_2^1+\beta_2x_2^2+\dots+\beta_{n-1}x_2^{n-1}+\epsilon_2\\ +\dots & \dots \\ +y_{n-1}&=\beta_0+\beta_1x_{n-1}^1+\beta_2x_{n-1}^2+\dots+\beta_{n-1}x_{n-1}^{n-1}+\epsilon_{n-1}.\\ +\end{align*} +!et + + + + +Defining the vectors +!bt +\[ +\bm{y} = [y_0,y_1, y_2,\dots, y_{n-1}]^T, +\] +!et +and +!bt +\[ +\bm{\beta} = [\beta_0,\beta_1, \beta_2,\dots, \beta_{n-1}]^T, +\] +!et +and +!bt +\[ +\bm{\epsilon} = [\epsilon_0,\epsilon_1, \epsilon_2,\dots, \epsilon_{n-1}]^T, +\] +!et +and the design matrix +!bt +\[ +\bm{X}= +\begin{bmatrix} +1& x_{0}^1 &x_{0}^2& \dots & \dots &x_{0}^{n-1}\\ +1& x_{1}^1 &x_{1}^2& \dots & \dots &x_{1}^{n-1}\\ +1& x_{2}^1 &x_{2}^2& \dots & \dots &x_{2}^{n-1}\\ +\dots& \dots &\dots& \dots & \dots &\dots\\ +1& x_{n-1}^1 &x_{n-1}^2& \dots & \dots &x_{n-1}^{n-1}\\ +\end{bmatrix} +\] +!et +we can rewrite our equations as +!bt +\[ +\bm{y} = \bm{X}\bm{\beta}+\bm{\epsilon}. +\] +!et +The above design matrix is called a "Vandermonde matrix":"https://en.wikipedia.org/wiki/Vandermonde_matrix". + +We are obviously not limited to the above polynomial expansions. We +could replace the various powers of $x$ with elements of Fourier +series or instead of $x_i^j$ we could have $\cos{(j x_i)}$ or $\sin{(j +x_i)}$, or time series or other orthogonal functions. For every set +of values $y_i,x_i$ we can then generalize the equations to + +!bt +\begin{align*} +y_0&=\beta_0x_{00}+\beta_1x_{01}+\beta_2x_{02}+\dots+\beta_{n-1}x_{0n-1}+\epsilon_0\\ +y_1&=\beta_0x_{10}+\beta_1x_{11}+\beta_2x_{12}+\dots+\beta_{n-1}x_{1n-1}+\epsilon_1\\ +y_2&=\beta_0x_{20}+\beta_1x_{21}+\beta_2x_{22}+\dots+\beta_{n-1}x_{2n-1}+\epsilon_2\\ +\dots & \dots \\ +y_{i}&=\beta_0x_{i0}+\beta_1x_{i1}+\beta_2x_{i2}+\dots+\beta_{n-1}x_{in-1}+\epsilon_i\\ +\dots & \dots \\ +y_{n-1}&=\beta_0x_{n-1,0}+\beta_1x_{n-1,2}+\beta_2x_{n-1,2}+\dots+\beta_{n-1}x_{n-1,n-1}+\epsilon_{n-1}.\\ +\end{align*} +!et + +_Note that we have $p=n$ here. The matrix is symmetric. This is generally not the case!_ + +We redefine in turn the matrix $\bm{X}$ as +!bt +\[ +\bm{X}= +\begin{bmatrix} +x_{00}& x_{01} &x_{02}& \dots & \dots &x_{0,n-1}\\ +x_{10}& x_{11} &x_{12}& \dots & \dots &x_{1,n-1}\\ +x_{20}& x_{21} &x_{22}& \dots & \dots &x_{2,n-1}\\ +\dots& \dots &\dots& \dots & \dots &\dots\\ +x_{n-1,0}& x_{n-1,1} &x_{n-1,2}& \dots & \dots &x_{n-1,n-1}\\ +\end{bmatrix} +\] +!et +and without loss of generality we rewrite again our equations as +!bt +\[ +\bm{y} = \bm{X}\bm{\beta}+\bm{\epsilon}. +\] +!et +The left-hand side of this equation is kwown. Our error vector $\bm{\epsilon}$ and the parameter vector $\bm{\beta}$ are our unknow quantities. How can we obtain the optimal set of $\beta_i$ values? + +We have defined the matrix $\bm{X}$ via the equations +!bt +\begin{align*} +y_0&=\beta_0x_{00}+\beta_1x_{01}+\beta_2x_{02}+\dots+\beta_{n-1}x_{0n-1}+\epsilon_0\\ +y_1&=\beta_0x_{10}+\beta_1x_{11}+\beta_2x_{12}+\dots+\beta_{n-1}x_{1n-1}+\epsilon_1\\ +y_2&=\beta_0x_{20}+\beta_1x_{21}+\beta_2x_{22}+\dots+\beta_{n-1}x_{2n-1}+\epsilon_1\\ +\dots & \dots \\ +y_{i}&=\beta_0x_{i0}+\beta_1x_{i1}+\beta_2x_{i2}+\dots+\beta_{n-1}x_{in-1}+\epsilon_1\\ +\dots & \dots \\ +y_{n-1}&=\beta_0x_{n-1,0}+\beta_1x_{n-1,2}+\beta_2x_{n-1,2}+\dots+\beta_{n-1}x_{n-1,n-1}+\epsilon_{n-1}.\\ +\end{align*} +!et + +As we noted above, we stayed with a system with the design matrix + $\bm{X}\in {\mathbb{R}}^{n\times n}$, that is we have $p=n$. For reasons to come later (algorithmic arguments) we will hereafter define +our matrix as $\bm{X}\in {\mathbb{R}}^{n\times p}$, with the predictors refering to the column numbers and the entries $n$ being the row elements. + +In our "introductory notes":"https://compphysics.github.io/MachineLearning/doc/pub/How2ReadData/html/How2ReadData.html" we looked at the so-called "liquid drop model":"https://en.wikipedia.org/wiki/Semi-empirical_mass_formula". Let us remind ourselves about what we did by looking at the code. + +We restate the parts of the code we are most interested in. +!bc pycod +# Common imports +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from IPython.display import display +import os + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("MassEval2016.dat"),'r') + + +# Read the experimental data with Pandas +Masses = pd.read_fwf(infile, usecols=(2,3,4,6,11), + names=('N', 'Z', 'A', 'Element', 'Ebinding'), + widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1), + header=39, + index_col=False) + +# Extrapolated values are indicated by '#' in place of the decimal place, so +# the Ebinding column won't be numeric. Coerce to float and drop these entries. +Masses['Ebinding'] = pd.to_numeric(Masses['Ebinding'], errors='coerce') +Masses = Masses.dropna() +# Convert from keV to MeV. +Masses['Ebinding'] /= 1000 + +# Group the DataFrame by nucleon number, A. +Masses = Masses.groupby('A') +# Find the rows of the grouped DataFrame with the maximum binding energy. +Masses = Masses.apply(lambda t: t[t.Ebinding==t.Ebinding.max()]) +A = Masses['A'] +Z = Masses['Z'] +N = Masses['N'] +Element = Masses['Element'] +Energies = Masses['Ebinding'] + +# Now we set up the design matrix X +X = np.zeros((len(A),5)) +X[:,0] = 1 +X[:,1] = A +X[:,2] = A**(2.0/3.0) +X[:,3] = A**(-1.0/3.0) +X[:,4] = A**(-1.0) +# Then nice printout using pandas +DesignMatrix = pd.DataFrame(X) +DesignMatrix.index = A +DesignMatrix.columns = ['1', 'A', 'A^(2/3)', 'A^(-1/3)', '1/A'] +display(DesignMatrix) +!ec + +With $\bm{\beta}\in {\mathbb{R}}^{p\times 1}$, it means that we will hereafter write our equations for the approximation as +!bt +\[ +\bm{\tilde{y}}= \bm{X}\bm{\beta}, +\] +!et +throughout these lectures. + +With the above we use the design matrix to define the approximation $\bm{\tilde{y}}$ via the unknown quantity $\bm{\beta}$ as +!bt +\[ +\bm{\tilde{y}}= \bm{X}\bm{\beta}, +\] +!et +and in order to find the optimal parameters $\beta_i$ instead of solving the above linear algebra problem, we define a function which gives a measure of the spread between the values $y_i$ (which represent hopefully the exact values) and the parameterized values $\tilde{y}_i$, namely +!bt +\[ +C(\bm{\beta})=\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\left\{\left(\bm{y}-\bm{\tilde{y}}\right)^T\left(\bm{y}-\bm{\tilde{y}}\right)\right\}, +\] +!et +or using the matrix $\bm{X}$ and in a more compact matrix-vector notation as +!bt +\[ +C(\bm{\beta})=\frac{1}{n}\left\{\left(\bm{y}-\bm{X}\bm{\beta}\right)^T\left(\bm{y}-\bm{X}\bm{\beta}\right)\right\}. +\] +!et +This function is one possible way to define the so-called cost function. + + + +It is also common to define +the function $C$ as + +!bt +\[ +C(\bm{\beta})=\frac{1}{2n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2, +\] +!et +since when taking the first derivative with respect to the unknown parameters $\beta$, the factor of $2$ cancels out. + +The function +!bt +\[ +C(\bm{\beta})=\frac{1}{n}\left\{\left(\bm{y}-\bm{X}\bm{\beta}\right)^T\left(\bm{y}-\bm{X}\bm{\beta}\right)\right\}, +\] +!et +can be linked to the variance of the quantity $y_i$ if we interpret the latter as the mean value. +When linking (see the discussion below) with the maximum likelihood approach below, we will indeed interpret $y_i$ as a mean value +!bt +\[ +y_{i}=\langle y_i \rangle = \beta_0x_{i,0}+\beta_1x_{i,1}+\beta_2x_{i,2}+\dots+\beta_{n-1}x_{i,n-1}+\epsilon_i, +\] +!et + +where $\langle y_i \rangle$ is the mean value. Keep in mind also that +till now we have treated $y_i$ as the exact value. Normally, the +response (dependent or outcome) variable $y_i$ the outcome of a +numerical experiment or another type of experiment and is thus only an +approximation to the true value. It is then always accompanied by an +error estimate, often limited to a statistical error estimate given by +the standard deviation discussed earlier. In the discussion here we +will treat $y_i$ as our exact value for the response variable. + +In order to find the parameters $\beta_i$ we will then minimize the spread of $C(\bm{\beta})$, that is we are going to solve the problem +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\bm{y}-\bm{X}\bm{\beta}\right)^T\left(\bm{y}-\bm{X}\bm{\beta}\right)\right\}. +\] +!et +In practical terms it means we will require +!bt +\[ +\frac{\partial C(\bm{\beta})}{\partial \beta_j} = \frac{\partial }{\partial \beta_j}\left[ \frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)^2\right]=0, +\] +!et +which results in +!bt +\[ +\frac{\partial C(\bm{\beta})}{\partial \beta_j} = -\frac{2}{n}\left[ \sum_{i=0}^{n-1}x_{ij}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)\right]=0, +\] +!et +or in a matrix-vector form as +!bt +\[ +\frac{\partial C(\bm{\beta})}{\partial \bm{\beta}} = 0 = \bm{X}^T\left( \bm{y}-\bm{X}\bm{\beta}\right). +\] +!et + + + +We can rewrite +!bt +\[ +\frac{\partial C(\bm{\beta})}{\partial \bm{\beta}} = 0 = \bm{X}^T\left( \bm{y}-\bm{X}\bm{\beta}\right), +\] +!et +as +!bt +\[ +\bm{X}^T\bm{y} = \bm{X}^T\bm{X}\bm{\beta}, +\] +!et +and if the matrix $\bm{X}^T\bm{X}$ is invertible we have the solution +!bt +\[ +\bm{\beta} =\left(\bm{X}^T\bm{X}\right)^{-1}\bm{X}^T\bm{y}. +\] +!et + +We note also that since our design matrix is defined as $\bm{X}\in +{\mathbb{R}}^{n\times p}$, the product $\bm{X}^T\bm{X} \in +{\mathbb{R}}^{p\times p}$. In the above case we have that $p \ll n$, +in our case $p=5$ meaning that we end up with inverting a small +$5\times 5$ matrix. This is a rather common situation, in many cases we end up with low-dimensional +matrices to invert. The methods discussed here and for many other +supervised learning algorithms like classification with logistic +regression or support vector machines, exhibit dimensionalities which +allow for the usage of direct linear algebra methods such as _LU_ decomposition or _Singular Value Decomposition_ (SVD) for finding the inverse of the matrix +$\bm{X}^T\bm{X}$. + +_Small question_: Do you think the example we have at hand here (the nuclear binding energies) can lead to problems in inverting the matrix $\bm{X}^T\bm{X}$? What kind of problems can we expect? + + +The following matrix and vector relation will be useful here and for the rest of the course. Vectors are always written as boldfaced lower case letters and +matrices as upper case boldfaced letters. + +!bt +\[ +\frac{\partial (\bm{b}^T\bm{a})}{\partial \bm{a}} = \bm{b}, +\] +!et +!bt +\[ +\frac{\partial (\bm{a}^T\bm{A}\bm{a})}{\partial \bm{a}} = (\bm{A}+\bm{A}^T)\bm{a}, +\] +!et +!bt +\[ +\frac{\partial tr(\bm{B}\bm{A})}{\partial \bm{A}} = \bm{B}^T, +\] +!et +!bt +\[ +\frac{\partial \log{\vert\bm{A}\vert}}{\partial \bm{A}} = (\bm{A}^{-1})^T. +\] +!et + + +The residuals $\bm{\epsilon}$ are in turn given by +!bt +\[ +\bm{\epsilon} = \bm{y}-\bm{\tilde{y}} = \bm{y}-\bm{X}\bm{\beta}, +\] +!et +and with +!bt +\[ +\bm{X}^T\left( \bm{y}-\bm{X}\bm{\beta}\right)= 0, +\] +!et +we have +!bt +\[ +\bm{X}^T\bm{\epsilon}=\bm{X}^T\left( \bm{y}-\bm{X}\bm{\beta}\right)= 0, +\] +!et +meaning that the solution for $\bm{\beta}$ is the one which minimizes the residuals. Later we will link this with the maximum likelihood approach. + + +Let us now return to our nuclear binding energies and simply code the above equations. + + +It is rather straightforward to implement the matrix inversion and obtain the parameters $\bm{\beta}$. After having defined the matrix $\bm{X}$ we simply need to +write +!bc pycod +# matrix inversion to find beta +beta = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Energies) +# and then make the prediction +ytilde = X @ beta +!ec +Alternatively, you can use the least squares functionality in _Numpy_ as +!bc pycod +fit = np.linalg.lstsq(X, Energies, rcond =None)[0] +ytildenp = np.dot(fit,X.T) +!ec + +And finally we plot our fit with and compare with data +!bc pycod +Masses['Eapprox'] = ytilde +# Generate a plot comparing the experimental with the fitted values values. +fig, ax = plt.subplots() +ax.set_xlabel(r'$A = N + Z$') +ax.set_ylabel(r'$E_\mathrm{bind}\,/\mathrm{MeV}$') +ax.plot(Masses['A'], Masses['Ebinding'], alpha=0.7, lw=2, + label='Ame2016') +ax.plot(Masses['A'], Masses['Eapprox'], alpha=0.7, lw=2, c='m', + label='Fit') +ax.legend() +save_fig("Masses2016OLS") +plt.show() +!ec + +We can easily test our fit by computing the $R2$ score that we discussed in connection with the functionality of _Scikit-Learn_ in the introductory slides. +Since we are not using _Scikit-Learn_ here we can define our own $R2$ function as +!bc pycod +def R2(y_data, y_model): + return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2) +!ec +and we would be using it as +!bc pycod +print(R2(Energies,ytilde)) +!ec + +We can easily add our _MSE_ score as +!bc pycod +def MSE(y_data,y_model): + n = np.size(y_model) + return np.sum((y_data-y_model)**2)/n + +print(MSE(Energies,ytilde)) +!ec +and finally the relative error as +!bc pycod +def RelativeError(y_data,y_model): + return abs((y_data-y_model)/y_data) +print(RelativeError(Energies, ytilde)) +!ec + + + +=== The $\chi^2$ function === + +Normally, the response (dependent or outcome) variable $y_i$ is the +outcome of a numerical experiment or another type of experiment and is +thus only an approximation to the true value. It is then always +accompanied by an error estimate, often limited to a statistical error +estimate given by the standard deviation discussed earlier. In the +discussion here we will treat $y_i$ as our exact value for the +response variable. + +Introducing the standard deviation $\sigma_i$ for each measurement +$y_i$, we define now the $\chi^2$ function (omitting the $1/n$ term) +as + +!bt +\[ +\chi^2(\bm{\beta})=\frac{1}{n}\sum_{i=0}^{n-1}\frac{\left(y_i-\tilde{y}_i\right)^2}{\sigma_i^2}=\frac{1}{n}\left\{\left(\bm{y}-\bm{\tilde{y}}\right)^T\frac{1}{\bm{\Sigma^2}}\left(\bm{y}-\bm{\tilde{y}}\right)\right\}, +\] +!et +where the matrix $\bm{\Sigma}$ is a diagonal matrix with $\sigma_i$ as matrix elements. + + +In order to find the parameters $\beta_i$ we will then minimize the spread of $\chi^2(\bm{\beta})$ by requiring +!bt +\[ +\frac{\partial \chi^2(\bm{\beta})}{\partial \beta_j} = \frac{\partial }{\partial \beta_j}\left[ \frac{1}{n}\sum_{i=0}^{n-1}\left(\frac{y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}}{\sigma_i}\right)^2\right]=0, +\] +!et +which results in +!bt +\[ +\frac{\partial \chi^2(\bm{\beta})}{\partial \beta_j} = -\frac{2}{n}\left[ \sum_{i=0}^{n-1}\frac{x_{ij}}{\sigma_i}\left(\frac{y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}}{\sigma_i}\right)\right]=0, +\] +!et +or in a matrix-vector form as +!bt +\[ +\frac{\partial \chi^2(\bm{\beta})}{\partial \bm{\beta}} = 0 = \bm{A}^T\left( \bm{b}-\bm{A}\bm{\beta}\right). +\] +!et +where we have defined the matrix $\bm{A} =\bm{X}/\bm{\Sigma}$ with matrix elements $a_{ij} = x_{ij}/\sigma_i$ and the vector $\bm{b}$ with elements $b_i = y_i/\sigma_i$. + +We can rewrite +!bt +\[ +\frac{\partial \chi^2(\bm{\beta})}{\partial \bm{\beta}} = 0 = \bm{A}^T\left( \bm{b}-\bm{A}\bm{\beta}\right), +\] +!et +as +!bt +\[ +\bm{A}^T\bm{b} = \bm{A}^T\bm{A}\bm{\beta}, +\] +!et +and if the matrix $\bm{A}^T\bm{A}$ is invertible we have the solution +!bt +\[ +\bm{\beta} =\left(\bm{A}^T\bm{A}\right)^{-1}\bm{A}^T\bm{b}. +\] +!et + + +If we then introduce the matrix +!bt +\[ +\bm{H} = \left(\bm{A}^T\bm{A}\right)^{-1}, +\] +!et +we have then the following expression for the parameters $\beta_j$ (the matrix elements of $\bm{H}$ are $h_{ij}$) +!bt +\[ +\beta_j = \sum_{k=0}^{p-1}h_{jk}\sum_{i=0}^{n-1}\frac{y_i}{\sigma_i}\frac{x_{ik}}{\sigma_i} = \sum_{k=0}^{p-1}h_{jk}\sum_{i=0}^{n-1}b_ia_{ik} +\] +!et +We state without proof the expression for the uncertainty in the parameters $\beta_j$ as (we leave this as an exercise) +!bt +\[ +\sigma^2(\beta_j) = \sum_{i=0}^{n-1}\sigma_i^2\left( \frac{\partial \beta_j}{\partial y_i}\right)^2, +\] +!et +resulting in +!bt +\[ +\sigma^2(\beta_j) = \left(\sum_{k=0}^{p-1}h_{jk}\sum_{i=0}^{n-1}a_{ik}\right)\left(\sum_{l=0}^{p-1}h_{jl}\sum_{m=0}^{n-1}a_{ml}\right) = h_{jj}! +\] +!et + +The first step here is to approximate the function $y$ with a first-order polynomial, that is we write +!bt +\[ +y=y(x) \rightarrow y(x_i) \approx \beta_0+\beta_1 x_i. +\] +!et +By computing the derivatives of $\chi^2$ with respect to $\beta_0$ and $\beta_1$ show that these are given by +!bt +\[ +\frac{\partial \chi^2(\bm{\beta})}{\partial \beta_0} = -2\left[ \frac{1}{n}\sum_{i=0}^{n-1}\left(\frac{y_i-\beta_0-\beta_1x_{i}}{\sigma_i^2}\right)\right]=0, +\] +!et +and +!bt +\[ +\frac{\partial \chi^2(\bm{\beta})}{\partial \beta_1} = -\frac{2}{n}\left[ \sum_{i=0}^{n-1}x_i\left(\frac{y_i-\beta_0-\beta_1x_{i}}{\sigma_i^2}\right)\right]=0. +\] +!et + +For a linear fit (a first-order polynomial) we don't need to invert a matrix!! +Defining +!bt +\[ +\gamma = \sum_{i=0}^{n-1}\frac{1}{\sigma_i^2}, +\] +!et + +!bt +\[ +\gamma_x = \sum_{i=0}^{n-1}\frac{x_{i}}{\sigma_i^2}, +\] +!et + +!bt +\[ +\gamma_y = \sum_{i=0}^{n-1}\left(\frac{y_i}{\sigma_i^2}\right), +\] +!et + +!bt +\[ +\gamma_{xx} = \sum_{i=0}^{n-1}\frac{x_ix_{i}}{\sigma_i^2}, +\] +!et + +!bt +\[ +\gamma_{xy} = \sum_{i=0}^{n-1}\frac{y_ix_{i}}{\sigma_i^2}, +\] +!et + +we obtain + +!bt +\[ +\beta_0 = \frac{\gamma_{xx}\gamma_y-\gamma_x\gamma_y}{\gamma\gamma_{xx}-\gamma_x^2}, +\] +!et + +!bt +\[ +\beta_1 = \frac{\gamma_{xy}\gamma-\gamma_x\gamma_y}{\gamma\gamma_{xx}-\gamma_x^2}. +\] +!et + +This approach (different linear and non-linear regression) suffers +often from both being underdetermined and overdetermined in the +unknown coefficients $\beta_i$. A better approach is to use the +Singular Value Decomposition (SVD) method discussed below. Or using +Lasso and Ridge regression. See below. + + +=== Fitting an Equation of State for Dense Nuclear Matter === + +Before we continue, let us introduce yet another example. We are going to fit the +nuclear equation of state using results from many-body calculations. +The equation of state we have made available here, as function of +density, has been derived using modern nucleon-nucleon potentials with +"the addition of three-body +forces":"https://www.sciencedirect.com/science/article/pii/S0370157399001106". This +time the file is presented as a standard _csv_ file. + +The beginning of the Python code here is similar to what you have seen +before, with the same initializations and declarations. We use also +_pandas_ again, rather extensively in order to organize our data. + +The difference now is that we use _Scikit-Learn's_ regression tools +instead of our own matrix inversion implementation. Furthermore, we +sneak in _Ridge_ regression (to be discussed below) which includes a +hyperparameter $\lambda$, also to be explained below. + + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.pyplot as plt +import sklearn.linear_model as skl +from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("EoS.csv"),'r') + +# Read the EoS data as csv file and organize the data into two arrays with density and energies +EoS = pd.read_csv(infile, names=('Density', 'Energy')) +EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce') +EoS = EoS.dropna() +Energies = EoS['Energy'] +Density = EoS['Density'] +# The design matrix now as function of various polytrops +X = np.zeros((len(Density),4)) +X[:,3] = Density**(4.0/3.0) +X[:,2] = Density +X[:,1] = Density**(2.0/3.0) +X[:,0] = 1 + +# We use now Scikit-Learn's linear regressor and ridge regressor +# OLS part +clf = skl.LinearRegression().fit(X, Energies) +ytilde = clf.predict(X) +EoS['Eols'] = ytilde +# The mean squared error +print("Mean squared error: %.2f" % mean_squared_error(Energies, ytilde)) +# Explained variance score: 1 is perfect prediction +print('Variance score: %.2f' % r2_score(Energies, ytilde)) +# Mean absolute error +print('Mean absolute error: %.2f' % mean_absolute_error(Energies, ytilde)) +print(clf.coef_, clf.intercept_) + +# The Ridge regression with a hyperparameter lambda = 0.1 +_lambda = 0.1 +clf_ridge = skl.Ridge(alpha=_lambda).fit(X, Energies) +yridge = clf_ridge.predict(X) +EoS['Eridge'] = yridge +# The mean squared error +print("Mean squared error: %.2f" % mean_squared_error(Energies, yridge)) +# Explained variance score: 1 is perfect prediction +print('Variance score: %.2f' % r2_score(Energies, yridge)) +# Mean absolute error +print('Mean absolute error: %.2f' % mean_absolute_error(Energies, yridge)) +print(clf_ridge.coef_, clf_ridge.intercept_) + +fig, ax = plt.subplots() +ax.set_xlabel(r'$\rho[\mathrm{fm}^{-3}]$') +ax.set_ylabel(r'Energy per particle') +ax.plot(EoS['Density'], EoS['Energy'], alpha=0.7, lw=2, + label='Theoretical data') +ax.plot(EoS['Density'], EoS['Eols'], alpha=0.7, lw=2, c='m', + label='OLS') +ax.plot(EoS['Density'], EoS['Eridge'], alpha=0.7, lw=2, c='g', + label='Ridge $\lambda = 0.1$') +ax.legend() +save_fig("EoSfitting") +plt.show() +!ec + +The above simple polynomial in density $\rho$ gives an excellent fit +to the data. + +We note also that there is a small deviation between the +standard OLS and the Ridge regression at higher densities. We discuss this in more detail +below. + + +===== Splitting our Data in Training and Test data ===== + +It is normal in essentially all Machine Learning studies to split the +data in a training set and a test set (sometimes also an additional +validation set). _Scikit-Learn_ has an own function for this. There +is no explicit recipe for how much data should be included as training +data and say test data. An accepted rule of thumb is to use +approximately $2/3$ to $4/5$ of the data as training data. We will +postpone a discussion of this splitting to the end of these notes and +our discussion of the so-called _bias-variance_ tradeoff. Here we +limit ourselves to repeat the above equation of state fitting example +but now splitting the data into a training set and a test set. + +!bc pycod +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +def R2(y_data, y_model): + return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2) +def MSE(y_data,y_model): + n = np.size(y_model) + return np.sum((y_data-y_model)**2)/n + +infile = open(data_path("EoS.csv"),'r') + +# Read the EoS data as csv file and organized into two arrays with density and energies +EoS = pd.read_csv(infile, names=('Density', 'Energy')) +EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce') +EoS = EoS.dropna() +Energies = EoS['Energy'] +Density = EoS['Density'] +# The design matrix now as function of various polytrops +X = np.zeros((len(Density),5)) +X[:,0] = 1 +X[:,1] = Density**(2.0/3.0) +X[:,2] = Density +X[:,3] = Density**(4.0/3.0) +X[:,4] = Density**(5.0/3.0) +# We split the data in test and training data +X_train, X_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2) +# matrix inversion to find beta +beta = np.linalg.inv(X_train.T.dot(X_train)).dot(X_train.T).dot(y_train) +# and then make the prediction +ytilde = X_train @ beta +print("Training R2") +print(R2(y_train,ytilde)) +print("Training MSE") +print(MSE(y_train,ytilde)) +ypredict = X_test @ beta +print("Test R2") +print(R2(y_test,ypredict)) +print("Test MSE") +print(MSE(y_test,ypredict)) +!ec + + +===== The Boston housing data example ===== + +The Boston housing +data set was originally a part of UCI Machine Learning Repository +and has been removed now. The data set is now included in _Scikit-Learn_'s +library. There are 506 samples and 13 feature (predictor) variables +in this data set. The objective is to predict the value of prices of +the house using the features (predictors) listed here. + +The features/predictors are + o CRIM: Per capita crime rate by town + o ZN: Proportion of residential land zoned for lots over 25000 square feet + o INDUS: Proportion of non-retail business acres per town + o CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise) + o NOX: Nitric oxide concentration (parts per 10 million) + o RM: Average number of rooms per dwelling + o AGE: Proportion of owner-occupied units built prior to 1940 + o DIS: Weighted distances to five Boston employment centers + o RAD: Index of accessibility to radial highways + o TAX: Full-value property tax rate per USD10000 + o B: $1000(Bk - 0.63)^2$, where $Bk$ is the proportion of [people of African American descent] by town + o LSTAT: Percentage of lower status of the population + o MEDV: Median value of owner-occupied homes in USD 1000s + +!split +===== Housing data, the code ===== +We start by importing the libraries +!bc pycod +import numpy as np +import matplotlib.pyplot as plt + +import pandas as pd +import seaborn as sns +!ec +and load the Boston Housing DataSet from _Scikit-Learn_ + + +!bc pycod +from sklearn.datasets import load_boston + +boston_dataset = load_boston() + +# boston_dataset is a dictionary +# let's check what it contains +boston_dataset.keys() +!ec +Then we invoke Pandas +!bc pycod +boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names) +boston.head() +boston['MEDV'] = boston_dataset.target +!ec +and preprocess the data +!bc pycod +# check for missing values in all the columns +boston.isnull().sum() +!ec +We can then visualize the data +!bc pycod +# set the size of the figure +sns.set(rc={'figure.figsize':(11.7,8.27)}) + +# plot a histogram showing the distribution of the target values +sns.distplot(boston['MEDV'], bins=30) +plt.show() +!ec + +It is now useful to look at the correlation matrix +!bc pycod +# compute the pair wise correlation for all columns +correlation_matrix = boston.corr().round(2) +# use the heatmap function from seaborn to plot the correlation matrix +# annot = True to print the values inside the square +sns.heatmap(data=correlation_matrix, annot=True) +!ec +From the above coorelation plot we can see that _MEDV_ is strongly correlated to _LSTAT_ and _RM_. We see also that _RAD_ and _TAX_ are stronly correlated, but we don't include this in our features together to avoid multi-colinearity + +!bc pycod +plt.figure(figsize=(20, 5)) + +features = ['LSTAT', 'RM'] +target = boston['MEDV'] + +for i, col in enumerate(features): + plt.subplot(1, len(features) , i+1) + x = boston[col] + y = target + plt.scatter(x, y, marker='o') + plt.title(col) + plt.xlabel(col) + plt.ylabel('MEDV') +!ec +Now we start training our model +!bc pycod +X = pd.DataFrame(np.c_[boston['LSTAT'], boston['RM']], columns = ['LSTAT','RM']) +Y = boston['MEDV'] +!ec +We split the data into training and test sets + +!bc pycod +from sklearn.model_selection import train_test_split + +# splits the training and test data set in 80% : 20% +# assign random_state to any value.This ensures consistency. +X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state=5) +print(X_train.shape) +print(X_test.shape) +print(Y_train.shape) +print(Y_test.shape) +!ec +Then we use the linear regression functionality from _Scikit-Learn_ +!bc pycod +from sklearn.linear_model import LinearRegression +from sklearn.metrics import mean_squared_error, r2_score + +lin_model = LinearRegression() +lin_model.fit(X_train, Y_train) + +# model evaluation for training set + +y_train_predict = lin_model.predict(X_train) +rmse = (np.sqrt(mean_squared_error(Y_train, y_train_predict))) +r2 = r2_score(Y_train, y_train_predict) + +print("The model performance for training set") +print("--------------------------------------") +print('RMSE is {}'.format(rmse)) +print('R2 score is {}'.format(r2)) +print("\n") + +# model evaluation for testing set + +y_test_predict = lin_model.predict(X_test) +# root mean square error of the model +rmse = (np.sqrt(mean_squared_error(Y_test, y_test_predict))) + +# r-squared score of the model +r2 = r2_score(Y_test, y_test_predict) + +print("The model performance for testing set") +print("--------------------------------------") +print('RMSE is {}'.format(rmse)) +print('R2 score is {}'.format(r2)) +!ec + +!bc pycod +# plotting the y_test vs y_pred +# ideally should have been a straight line +plt.scatter(Y_test, y_test_predict) +plt.show() +!ec + + +===== Reducing the number of degrees of freedom, overarching view ===== + +Many Machine Learning problems involve thousands or even millions of +features for each training instance. Not only does this make training +extremely slow, it can also make it much harder to find a good +solution, as we will see. This problem is often referred to as the +curse of dimensionality. Fortunately, in real-world problems, it is +often possible to reduce the number of features considerably, turning +an intractable problem into a tractable one. + +Later we will discuss some of the most popular dimensionality reduction +techniques: the principal component analysis (PCA), Kernel PCA, and +Locally Linear Embedding (LLE). + + +Principal component analysis and its various variants deal with the +problem of fitting a low-dimensional "affine +subspace":"https://en.wikipedia.org/wiki/Affine_space" to a set of of +data points in a high-dimensional space. With its family of methods it +is one of the most used tools in data modeling, compression and +visualization. + + +Before we proceed however, we will discuss how to preprocess our +data. Till now and in connection with our previous examples we have +not met so many cases where we are too sensitive to the scaling of our +data. Normally the data may need a rescaling and/or may be sensitive +to extreme values. Scaling the data renders our inputs much more +suitable for the algorithms we want to employ. + +_Scikit-Learn_ has several functions which allow us to rescale the +data, normally resulting in much better results in terms of various +accuracy scores. The _StandardScaler_ function in _Scikit-Learn_ +ensures that for each feature/predictor we study the mean value is +zero and the variance is one (every column in the design/feature +matrix). This scaling has the drawback that it does not ensure that +we have a particular maximum or minimum in our data set. Another +function included in _Scikit-Learn_ is the _MinMaxScaler_ which +ensures that all features are exactly between $0$ and $1$. The + + +The _Normalizer_ scales each data +point such that the feature vector has a euclidean length of one. In other words, it +projects a data point on the circle (or sphere in the case of higher dimensions) with a +radius of 1. This means every data point is scaled by a different number (by the +inverse of it’s length). +This normalization is often used when only the direction (or angle) of the data matters, +not the length of the feature vector. + +The _RobustScaler_ works similarly to the StandardScaler in that it +ensures statistical properties for each feature that guarantee that +they are on the same scale. However, the RobustScaler uses the median +and quartiles, instead of mean and variance. This makes the +RobustScaler ignore data points that are very different from the rest +(like measurement errors). These odd data points are also called +outliers, and might often lead to trouble for other scaling +techniques. + + +=== Simple preprocessing examples, Franke function and regression === + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import sklearn.linear_model as skl +from sklearn.metrics import mean_squared_error +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + + +def FrankeFunction(x,y): + term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2)) + term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1)) + term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2)) + term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2) + return term1 + term2 + term3 + term4 + + +def create_X(x, y, n ): + if len(x.shape) > 1: + x = np.ravel(x) + y = np.ravel(y) + + N = len(x) + l = int((n+1)*(n+2)/2) # Number of elements in beta + X = np.ones((N,l)) + + for i in range(1,n+1): + q = int((i)*(i+1)/2) + for k in range(i+1): + X[:,q+k] = (x**(i-k))*(y**k) + + return X + + +# Making meshgrid of datapoints and compute Franke's function +n = 5 +N = 1000 +x = np.sort(np.random.uniform(0, 1, N)) +y = np.sort(np.random.uniform(0, 1, N)) +z = FrankeFunction(x, y) +X = create_X(x, y, n=n) +# split in training and test data +X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2) + + +clf = skl.LinearRegression().fit(X_train, y_train) + +# The mean squared error and R2 score +print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test))) +print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test))) + +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) + +print("Feature min values before scaling:\n {}".format(X_train.min(axis=0))) +print("Feature max values before scaling:\n {}".format(X_train.max(axis=0))) + +print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0))) +print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0))) + +clf = skl.LinearRegression().fit(X_train_scaled, y_train) + + +print("MSE after scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test))) +print("R2 score for scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test))) + +!ec + + diff --git a/doc/BookChapters/chapter1.do.txt~ b/doc/BookChapters/chapter1.do.txt~ new file mode 100644 index 000000000..6d41bf736 --- /dev/null +++ b/doc/BookChapters/chapter1.do.txt~ @@ -0,0 +1,1557 @@ +TITLE: Week 34: Introduction to the course, Logistics and Practicalities +AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo & Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University +DATE: today + + + +!split +===== Overview of first week ===== + +!bblock + * Thursday August 20: First lecture: Presentation of the course, aims and content + * Thursday: Second Lecture: Start with simple linear regression and repetition of linear algebra and elements of statistics + * Friday August 21: Linear regression + * Computer lab: Wednesdays, 8am-6pm. First time: Wednesday August 26. +!eblock + + +!split +===== Thursday August 20 ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/zoom_0.mp4?vrtx=view-as-webpage". + + +!split +===== Lectures and ComputerLab ===== + +!bblock + * Lectures: Thursday (12.15pm-2pm and Friday (12.15pm-2pm). Due to the present COVID-19 situation all lectures will be online. They will be recorded and posted online at the official UiO "website":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/index.html". + * Weekly reading assignments and videos needed to solve projects and exercises. + * Weekly exercises when not working on projects. You can hand in exercises if you want. + * Detailed lecture notes, exercises, all programs presented, projects etc can be found at the homepage of the course. + * Weekly plans and all other information are on the official webpage. + * No final exam, three projects that are graded and have to be approved. +!eblock + + + + +!split +===== Course Format ===== + +!bblock + * Three compulsory projects. Electronic reports only using "Canvas":"https://www.uio.no/english/services/it/education/canvas/" to hand in projects and "git":"https://git-scm.com/" as version control software and "GitHub":"https://github.com/" for repository (or "GitLab":"https://about.gitlab.com/") of all your material. + * Evaluation and grading: The three projects are graded and each counts 1/3 of the final mark. No final written or oral exam. + o For the last project each group/participant submits a proposal or works with suggested (by us) proposals for the project. + o If possible, we would like to organize the last project as a workshop where each group makes a poster and presents this to all other participants of the course + o Poster session where all participants can study and discuss the other proposals. + o Based on feedback etc, each group finalizes the report and submits for grading. + * Python is the default programming language, but feel free to use C/C++ and/or Fortran or other programming languages. All source codes discussed during the lectures can be found at the webpage and "github address":"https://github.com/CompPhysics/MachineLearning/tree/master/doc/Programs" of the course. +!eblock + + + + +!split +===== Teachers ===== + +!bblock + +_Teachers :_ +* Morten Hjorth-Jensen, morten.hjorth-jensen@fys.uio.no + * _Phone_: +47-48257387 + * _Office_: Department of Physics, University of Oslo, Eastern wing, room FØ470 + * _Office hours_: *Anytime*! In Fall Semester 2020 (FS20), as a rule of thumb office hours are planned via computer or telephone. Individual or group office hours will be performed via zoom. Feel free to send an email for planning. In person meetings may also be possible if allowed by the University of Oslo's COVID-19 instructions. + +* Øyvind Sigmundson Schøyen, oyvinssc@student.matnat.uio.no + * _Office_: Department of Physics, University of Oslo, Eastern wing, room FØ452 +* Michael Bitney, m.s.bitney@fys.uio.no +* Kristian Wold, kriswold@student.matnat.uio.no +* Nicolai Haug, nicoha@student.matnat.uio.no +* Per-Dimitri Sønsteland, perdimitri.bs@gmail.com + + +!eblock + +!split +===== Deadlines for projects (tentative) ===== + +!bblock + +o Project 1: September 28 (graded with feedback) +o Project 2: November 2 (graded with feedback) +o Project 3: December 7 (graded with feedback) + +Projects are handed in using _Canvas_. We use Github as repository for codes, benchmark calculations etc. Comments and feedback on projects only via _Canvas_. + +!eblock + + +!split +===== Recommended textbooks ===== + +* "Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer":"https://www.springer.com/gp/book/9780387848570" +* "Aurelien Geron, Hands-On Machine Learning with Scikit-Learn, Keras, and TensorFlow, 2nd Edition":"https://www.oreilly.com/library/view/hands-on-machine-learning/9781492032632/" + +!split +===== Prerequisites ===== + +Basic knowledge in programming and mathematics, with an emphasis on +linear algebra. Knowledge of Python or/and C++ as programming +languages is strongly recommended and experience with Jupiter notebook +is recommended. Required courses are the equivalents to the University +of Oslo mathematics courses MAT1100, MAT1110, MAT1120 and at least one +of the corresponding computing and programming courses INF1000/INF1110 +or MAT-INF1100/MAT-INF1100L/BIOS1100/KJM-INF1100. Most universities +offer nowadays a basic programming course (often compulsory) where +Python is the recurring programming language. + + + +!split +===== Learning outcomes ===== + +!bblock + + +This course aims at giving you insights and knowledge about many of the central algorithms used in Data Analysis and Machine Learning. The course is project based and through various numerical projects, normally three, you will be exposed to fundamental research problems in these fields, with the aim to reproduce state of the art scientific results. Both supervised and unsupervised methods will be covered. The emphasis is on a frequentist approach, although we will try to link it with a Bayesian approach as well. You will learn to develop and structure large codes for studying different cases where Machine Learning is applied to, get acquainted with computing facilities and learn to handle large scientific projects. A good scientific and ethical conduct is emphasized throughout the course. More specifically, after this course you will + +* Learn about basic data analysis, statistical analysis, Bayesian statistics, Monte Carlo sampling, data optimization and machine learning; +* Be capable of extending the acquired knowledge to other systems and cases; +* Have an understanding of central algorithms used in data analysis and machine learning; +* Understand linear methods for regression and classification, from ordinary least squares, via Lasso and Ridge to Logistic regression; +* Learn about neural networks and deep learning methods for supervised and unsupervised learning. Emphasis on feed forward neural networks, convolutional and recurrent neural networks; +* Learn about about decision trees, random forests, bagging and boosting methods; +* Learn about support vector machines and kernel transformations; +* Reduction of data sets, from PCA to clustering; +* Autoencoders and Reinforcement Learning; +* Work on numerical projects to illustrate the theory. The projects play a central role and you are expected to know modern programming languages like Python or C++ and/or Fortran (Fortran2003 or later). + +!eblock + +!split +===== Topics covered in this course: Statistical analysis and optimization of data ===== + +The course has two central parts + +o Statistical analysis and optimization of data +o Machine learning + +These topics will be scattered thorughout the course and may not necessarily be taught separately. Rather, we will often take an approach (during the lectures and project/exercise sessions) where say elements from statistical data analysis are mixed with specific Machine Learning algorithms + +!bblock Statistical analysis and optimization of data + +The following topics will be covered +* Basic concepts, expectation values, variance, covariance, correlation functions and errors; +* Simpler models, binomial distribution, the Poisson distribution, simple and multivariate normal distributions; +* Central elements of Bayesian statistics and modeling; +* Gradient methods for data optimization, +* Monte Carlo methods, Markov chains, Gibbs sampling and Metropolis-Hastings sampling; +* Estimation of errors and resampling techniques such as the cross-validation, blocking, bootstrapping and jackknife methods; +* Principal Component Analysis (PCA) and its mathematical foundation + +!eblock + + +!split +===== Topics covered in this course: Machine Learning ===== + +!bblock +The following topics will be covered +* Linear Regression and Logistic Regression; +* Neural networks and deep learning, including convolutional and recurrent neural networks +* Decisions trees, Random Forests, Bagging and Boosting +* Support vector machines +* Bayesian linear and logistic regression +* Boltzmann Machines +* Unsupervised learning Dimensionality reduction, from PCA to cluster models + +Hands-on demonstrations, exercises and projects aim at deepening your understanding of these topics. + +!eblock + + +!split +===== Extremely useful tools, strongly recommended ===== + +!bblock and discussed at the lab sessions + * GIT for version control, and GitHub or GitLab as repositories, highly recommended. This will be discussed during the first exercise session + * Anaconda and other Python environments, see intro slides and first exercise session +!eblock + + + + + +!split +===== Other courses on Data science and Machine Learning at UiO ===== + +The link here URL:"https://www.mn.uio.no/english/research/about/centre-focus/innovation/data-science/studies/" gives an excellent overview of courses on Machine learning at UiO. + +o "STK2100 Machine learning and statistical methods for prediction and classification":"http://www.uio.no/studier/emner/matnat/math/STK2100/index-eng.html". +o "IN3050 Introduction to Artificial Intelligence and Machine Learning":"https://www.uio.no/studier/emner/matnat/ifi/IN3050/index-eng.html". Introductory course in machine learning and AI with an algorithmic approach. +o "STK-INF3000/4000 Selected Topics in Data Science":"http://www.uio.no/studier/emner/matnat/math/STK-INF3000/index-eng.html". The course provides insight into selected contemporary relevant topics within Data Science. +o "IN4080 Natural Language Processing":"https://www.uio.no/studier/emner/matnat/ifi/IN4080/index.html". Probabilistic and machine learning techniques applied to natural language processing. +o "STK-IN4300 Statistical learning methods in Data Science":"https://www.uio.no/studier/emner/matnat/math/STK-IN4300/index-eng.html". An advanced introduction to statistical and machine learning. For students with a good mathematics and statistics background. +o "INF4490 Biologically Inspired Computing":"http://www.uio.no/studier/emner/matnat/ifi/INF4490/". An introduction to self-adapting methods also called artificial intelligence or machine learning. +o "IN-STK5000 Adaptive Methods for Data-Based Decision Making":"https://www.uio.no/studier/emner/matnat/ifi/IN-STK5000/index-eng.html". Methods for adaptive collection and processing of data based on machine learning techniques. +o "IN5400/INF5860 Machine Learning for Image Analysis":"https://www.uio.no/studier/emner/matnat/ifi/IN5400/". An introduction to deep learning with particular emphasis on applications within Image analysis, but useful for other application areas too. +o "TEK5040 Deep learning for autonomous systems":"https://www.uio.no/studier/emner/matnat/its/TEK5040/". The course addresses advanced algorithms and architectures for deep learning with neural networks. The course provides an introduction to how deep-learning techniques can be used in the construction of key parts of advanced autonomous systems that exist in physical environments and cyber environments. +o "STK4051 Computational Statistics":"https://www.uio.no/studier/emner/matnat/math/STK4051/index-eng.html" +o "STK4021 Applied Bayesian Analysis and Numerical Methods":"https://www.uio.no/studier/emner/matnat/math/STK4021/index-eng.html" + + + +!split +===== Introduction ===== + +Our emphasis throughout this series of lectures +is on understanding the mathematical aspects of +different algorithms used in the fields of data analysis and machine learning. + +However, where possible we will emphasize the +importance of using available software. We start thus with a hands-on +and top-down approach to machine learning. The aim is thus to start with +relevant data or data we have produced +and use these to introduce statistical data analysis +concepts and machine learning algorithms before we delve into the +algorithms themselves. The examples we will use in the beginning, start with simple +polynomials with random noise added. We will use the Python +software package "Scikit-Learn":"http://scikit-learn.org/stable/" and +introduce various machine learning algorithms to make fits of +the data and predictions. We move thereafter to more interesting +cases such as data from say experiments (below we will look at experimental nuclear binding energies as an example). +These are examples where we can easily set up the data and +then use machine learning algorithms included in for example +_Scikit-Learn_. + +These examples will serve us the purpose of getting +started. Furthermore, they allow us to catch more than two birds with +a stone. They will allow us to bring in some programming specific +topics and tools as well as showing the power of various Python +libraries for machine learning and statistical data analysis. + +Here, we will mainly focus on two +specific Python packages for Machine Learning, Scikit-Learn and +Tensorflow (see below for links etc). Moreover, the examples we +introduce will serve as inputs to many of our discussions later, as +well as allowing you to set up models and produce your own data and +get started with programming. + + +!split +===== What is Machine Learning? ===== + +Statistics, data science and machine learning form important fields of +research in modern science. They describe how to learn and make +predictions from data, as well as allowing us to extract important +correlations about physical process and the underlying laws of motion +in large data sets. The latter, big data sets, appear frequently in +essentially all disciplines, from the traditional Science, Technology, +Mathematics and Engineering fields to Life Science, Law, education +research, the Humanities and the Social Sciences. + +It has become more +and more common to see research projects on big data in for example +the Social Sciences where extracting patterns from complicated survey +data is one of many research directions. Having a solid grasp of data +analysis and machine learning is thus becoming central to scientific +computing in many fields, and competences and skills within the fields +of machine learning and scientific computing are nowadays strongly +requested by many potential employers. The latter cannot be +overstated, familiarity with machine learning has almost become a +prerequisite for many of the most exciting employment opportunities, +whether they are in bioinformatics, life science, physics or finance, +in the private or the public sector. This author has had several +students or met students who have been hired recently based on their +skills and competences in scientific computing and data science, often +with marginal knowledge of machine learning. + +Machine learning is a subfield of computer science, and is closely +related to computational statistics. It evolved from the study of +pattern recognition in artificial intelligence (AI) research, and has +made contributions to AI tasks like computer vision, natural language +processing and speech recognition. Many of the methods we will study are also +strongly rooted in basic mathematics and physics research. + +Ideally, machine learning represents the science of giving computers +the ability to learn without being explicitly programmed. The idea is +that there exist generic algorithms which can be used to find patterns +in a broad class of data sets without having to write code +specifically for each problem. The algorithm will build its own logic +based on the data. You should however always keep in mind that +machines and algorithms are to a large extent developed by humans. The +insights and knowledge we have about a specific system, play a central +role when we develop a specific machine learning algorithm. + +Machine learning is an extremely rich field, in spite of its young +age. The increases we have seen during the last three decades in +computational capabilities have been followed by developments of +methods and techniques for analyzing and handling large date sets, +relying heavily on statistics, computer science and mathematics. The +field is rather new and developing rapidly. Popular software packages +written in Python for machine learning like +"Scikit-learn":"http://scikit-learn.org/stable/", +"Tensorflow":"https://www.tensorflow.org/", +"PyTorch":"http://pytorch.org/" and "Keras":"https://keras.io/", all +freely available at their respective GitHub sites, encompass +communities of developers in the thousands or more. And the number of +code developers and contributors keeps increasing. Not all the +algorithms and methods can be given a rigorous mathematical +justification, opening up thereby large rooms for experimenting and +trial and error and thereby exciting new developments. However, a +solid command of linear algebra, multivariate theory, probability +theory, statistical data analysis, understanding errors and Monte +Carlo methods are central elements in a proper understanding of many +of algorithms and methods we will discuss. + + +!split +===== Types of Machine Learning ===== + + +The approaches to machine learning are many, but are often split into +two main categories. In *supervised learning* we know the answer to a +problem, and let the computer deduce the logic behind it. On the other +hand, *unsupervised learning* is a method for finding patterns and +relationship in data sets without any prior knowledge of the system. +Some authours also operate with a third category, namely +*reinforcement learning*. This is a paradigm of learning inspired by +behavioral psychology, where learning is achieved by trial-and-error, +solely from rewards and punishment. + +Another way to categorize machine learning tasks is to consider the +desired output of a system. Some of the most common tasks are: + + * Classification: Outputs are divided into two or more classes. The goal is to produce a model that assigns inputs into one of these classes. An example is to identify digits based on pictures of hand-written ones. Classification is typically supervised learning. + + * Regression: Finding a functional relationship between an input data set and a reference data set. The goal is to construct a function that maps input data to continuous output values. + + * Clustering: Data are divided into groups with certain common traits, without knowing the different groups beforehand. It is thus a form of unsupervised learning. + + +The methods we cover have three main topics in common, irrespective of +whether we deal with supervised or unsupervised learning. The first +ingredient is normally our data set (which can be subdivided into +training and test data), the second item is a model which is normally a +function of some parameters. The model reflects our knowledge of the system (or lack thereof). As an example, if we know that our data show a behavior similar to what would be predicted by a polynomial, fitting our data to a polynomial of some degree would then determin our model. + +The last ingredient is a so-called _cost_ +function which allows us to present an estimate on how good our model +is in reproducing the data it is supposed to train. +At the heart of basically all ML algorithms there are so-called minimization algorithms, often we end up with various variants of _gradient_ methods. + + + + + + +!split +===== Software and needed installations ===== + +We will make extensive use of Python as programming language and its +myriad of available libraries. You will find +Jupyter notebooks invaluable in your work. You can run _R_ +codes in the Jupyter/IPython notebooks, with the immediate benefit of +visualizing your data. You can also use compiled languages like C++, +Rust, Julia, Fortran etc if you prefer. The focus in these lectures will be +on Python. + + +If you have Python installed (we strongly recommend Python3) and you feel +pretty familiar with installing different packages, we recommend that +you install the following Python packages via _pip_ as + +o pip install numpy scipy matplotlib ipython scikit-learn mglearn sympy pandas pillow + +For Python3, replace _pip_ with _pip3_. + +For OSX users we recommend, after having installed Xcode, to +install _brew_. Brew allows for a seamless installation of additional +software via for example + +o brew install python3 + +For Linux users, with its variety of distributions like for example the widely popular Ubuntu distribution, +you can use _pip_ as well and simply install Python as + +o sudo apt-get install python3 (or python for pyhton2.7) + +etc etc. + + +!split +===== Python installers ===== + +If you don't want to perform these operations separately and venture +into the hassle of exploring how to set up dependencies and paths, we +recommend two widely used distrubutions which set up all relevant +dependencies for Python, namely + +* "Anaconda":"https://docs.anaconda.com/", + +which is an open source +distribution of the Python and R programming languages for large-scale +data processing, predictive analytics, and scientific computing, that +aims to simplify package management and deployment. Package versions +are managed by the package management system _conda_. + +* "Enthought canopy":"https://www.enthought.com/product/canopy/" + +is a Python +distribution for scientific and analytic computing distribution and +analysis environment, available for free and under a commercial +license. + +Furthermore, "Google's Colab":"https://colab.research.google.com/notebooks/welcome.ipynb" is a free Jupyter notebook environment that requires +no setup and runs entirely in the cloud. Try it out! + +!split +===== Useful Python libraries ===== +Here we list several useful Python libraries we strongly recommend (if you use anaconda many of these are already there) + +* "NumPy":"https://www.numpy.org/" is a highly popular library for large, multi-dimensional arrays and matrices, along with a large collection of high-level mathematical functions to operate on these arrays +* "The pandas":"https://pandas.pydata.org/" library provides high-performance, easy-to-use data structures and data analysis tools +* "Xarray":"http://xarray.pydata.org/en/stable/" is a Python package that makes working with labelled multi-dimensional arrays simple, efficient, and fun! +* "Scipy":"https://www.scipy.org/" (pronounced “Sigh Pie”) is a Python-based ecosystem of open-source software for mathematics, science, and engineering. +* "Matplotlib":"https://matplotlib.org/" is a Python 2D plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms. +* "Autograd":"https://github.com/HIPS/autograd" can automatically differentiate native Python and Numpy code. It can handle a large subset of Python's features, including loops, ifs, recursion and closures, and it can even take derivatives of derivatives of derivatives +* "SymPy":"https://www.sympy.org/en/index.html" is a Python library for symbolic mathematics. +* "scikit-learn":"https://scikit-learn.org/stable/" has simple and efficient tools for machine learning, data mining and data analysis +* "TensorFlow":"https://www.tensorflow.org/" is a Python library for fast numerical computing created and released by Google +* "Keras":"https://keras.io/" is a high-level neural networks API, written in Python and capable of running on top of TensorFlow, CNTK, or Theano +* And many more such as "pytorch":"https://pytorch.org/", "Theano":"https://pypi.org/project/Theano/" etc + +!split +===== Installing R, C++, cython or Julia ===== + +You will also find it convenient to utilize _R_. We will mainly +use Python during our lectures and in various projects and exercises. +Those of you +already familiar with _R_ should feel free to continue using _R_, keeping +however an eye on the parallel Python set ups. Similarly, if you are a +Python afecionado, feel free to explore _R_ as well. Jupyter/Ipython +notebook allows you to run _R_ codes interactively in your +browser. The software library _R_ is really tailored for statistical data analysis +and allows for an easy usage of the tools and algorithms we will discuss in these +lectures. + +To install _R_ with Jupyter notebook +"follow the link here":"https://mpacer.org/maths/r-kernel-for-ipython-notebook" + + + +!split +===== Installing R, C++, cython, Numba etc ===== + + +For the C++ aficionados, Jupyter/IPython notebook allows you also to +install C++ and run codes written in this language interactively in +the browser. Since we will emphasize writing many of the algorithms +yourself, you can thus opt for either Python or C++ (or Fortran or other compiled languages) as programming +languages. + +To add more entropy, _cython_ can also be used when running your +notebooks. It means that Python with the jupyter notebook +setup allows you to integrate widely popular softwares and tools for +scientific computing. Similarly, the +"Numba Python package":"https://numba.pydata.org/" delivers increased performance +capabilities with minimal rewrites of your codes. With its +versatility, including symbolic operations, Python offers a unique +computational environment. Your jupyter notebook can easily be +converted into a nicely rendered _PDF_ file or a Latex file for +further processing. For example, convert to latex as + +!bc +pycod jupyter nbconvert filename.ipynb --to latex +!ec + +And to add more versatility, the Python package "SymPy":"http://www.sympy.org/en/index.html" is a Python library for symbolic mathematics. It aims to become a full-featured computer algebra system (CAS) and is entirely written in Python. + +Finally, if you wish to use the light mark-up language +"doconce":"https://github.com/hplgit/doconce" you can convert a standard ascii text file into various HTML +formats, ipython notebooks, latex files, pdf files etc with minimal edits. These lectures were generated using _doconce_. + + +!split +===== Numpy examples and Important Matrix and vector handling packages ===== + +There are several central software libraries for linear algebra and eigenvalue problems. Several of the more +popular ones have been wrapped into ofter software packages like those from the widely used text _Numerical Recipes_. The original source codes in many of the available packages are often taken from the widely used +software package LAPACK, which follows two other popular packages +developed in the 1970s, namely EISPACK and LINPACK. We describe them shortly here. + + * LINPACK: package for linear equations and least square problems. + * LAPACK:package for solving symmetric, unsymmetric and generalized eigenvalue problems. From LAPACK's website URL: "http://www.netlib.org" it is possible to download for free all source codes from this library. Both C/C++ and Fortran versions are available. + * BLAS (I, II and III): (Basic Linear Algebra Subprograms) are routines that provide standard building blocks for performing basic vector and matrix operations. Blas I is vector operations, II vector-matrix operations and III matrix-matrix operations. Highly parallelized and efficient codes, all available for download from URL: "http://www.netlib.org". + +!split +===== Basic Matrix Features ===== + +!bblock Matrix properties reminder +!bt +\[ + \mathbf{A} = + \begin{bmatrix} a_{11} & a_{12} & a_{13} & a_{14} \\ + a_{21} & a_{22} & a_{23} & a_{24} \\ + a_{31} & a_{32} & a_{33} & a_{34} \\ + a_{41} & a_{42} & a_{43} & a_{44} + \end{bmatrix}\qquad +\mathbf{I} = + \begin{bmatrix} 1 & 0 & 0 & 0 \\ + 0 & 1 & 0 & 0 \\ + 0 & 0 & 1 & 0 \\ + 0 & 0 & 0 & 1 + \end{bmatrix} +\] +!et + + + +The inverse of a matrix is defined by + +!bt +\[ +\mathbf{A}^{-1} \cdot \mathbf{A} = I +\] +!et + + +|----------------------------------------------------------------------| +| Relations | Name | matrix elements | +|----------------------------------------------------------------------| +| $A = A^{T}$ | symmetric | $a_{ij} = a_{ji}$ | +| $A = \left (A^{T} \right )^{-1}$ | real orthogonal | $\sum_k a_{ik} a_{jk} = \sum_k a_{ki} a_{kj} = \delta_{ij}$ | +| $A = A^{ * }$ | real matrix | $a_{ij} = a_{ij}^{ * }$ | +| $A = A^{\dagger}$ | hermitian | $a_{ij} = a_{ji}^{ * }$ | +| $A = \left (A^{\dagger} \right )^{-1}$ | unitary | $\sum_k a_{ik} a_{jk}^{ * } = \sum_k a_{ki}^{ * } a_{kj} = \delta_{ij}$ | +|----------------------------------------------------------------------| + +!eblock + +!split +=== Some famous Matrices === + + * Diagonal if $a_{ij}=0$ for $i\ne j$ + * Upper triangular if $a_{ij}=0$ for $i > j$ + * Lower triangular if $a_{ij}=0$ for $i < j$ + * Upper Hessenberg if $a_{ij}=0$ for $i > j+1$ + * Lower Hessenberg if $a_{ij}=0$ for $i < j+1$ + * Tridiagonal if $a_{ij}=0$ for $|i -j| > 1$ + * Lower banded with bandwidth $p$: $a_{ij}=0$ for $i > j+p$ + * Upper banded with bandwidth $p$: $a_{ij}=0$ for $i < j+p$ + * Banded, block upper triangular, block lower triangular.... + + +!split +=== More Basic Matrix Features === + +!bblock Some Equivalent Statements +For an $N\times N$ matrix $\mathbf{A}$ the following properties are all equivalent + + * If the inverse of $\mathbf{A}$ exists, $\mathbf{A}$ is nonsingular. + * The equation $\mathbf{Ax}=0$ implies $\mathbf{x}=0$. + * The rows of $\mathbf{A}$ form a basis of $R^N$. + * The columns of $\mathbf{A}$ form a basis of $R^N$. + * $\mathbf{A}$ is a product of elementary matrices. + * $0$ is not eigenvalue of $\mathbf{A}$. +!eblock + +!split +===== Numpy and arrays ===== +"Numpy":"http://www.numpy.org/" provides an easy way to handle arrays in Python. The standard way to import this library is as + +!bc pycod +import numpy as np +!ec +Here follows a simple example where we set up an array of ten elements, all determined by random numbers drawn according to the normal distribution, +!bc pycod +n = 10 +x = np.random.normal(size=n) +print(x) +!ec +We defined a vector $x$ with $n=10$ elements with its values given by the Normal distribution $N(0,1)$. +Another alternative is to declare a vector as follows +!bc pycod +import numpy as np +x = np.array([1, 2, 3]) +print(x) +!ec +Here we have defined a vector with three elements, with $x_0=1$, $x_1=2$ and $x_2=3$. Note that both Python and C++ +start numbering array elements from $0$ and on. This means that a vector with $n$ elements has a sequence of entities $x_0, x_1, x_2, \dots, x_{n-1}$. We could also let (recommended) Numpy to compute the logarithms of a specific array as +!bc pycod +import numpy as np +x = np.log(np.array([4, 7, 8])) +print(x) +!ec + +In the last example we used Numpy's unary function $np.log$. This function is +highly tuned to compute array elements since the code is vectorized +and does not require looping. We normaly recommend that you use the +Numpy intrinsic functions instead of the corresponding _log_ function +from Python's _math_ module. The looping is done explicitely by the +_np.log_ function. The alternative, and slower way to compute the +logarithms of a vector would be to write + +!bc pycod +import numpy as np +from math import log +x = np.array([4, 7, 8]) +for i in range(0, len(x)): + x[i] = log(x[i]) +print(x) +!ec +We note that our code is much longer already and we need to import the _log_ function from the _math_ module. +The attentive reader will also notice that the output is $[1, 1, 2]$. Python interprets automagically our numbers as integers (like the _automatic_ keyword in C++). To change this we could define our array elements to be double precision numbers as +!bc pycod +import numpy as np +x = np.log(np.array([4, 7, 8], dtype = np.float64)) +print(x) +!ec +or simply write them as double precision numbers (Python uses 64 bits as default for floating point type variables), that is +!bc pycod +import numpy as np +x = np.log(np.array([4.0, 7.0, 8.0]) +print(x) +!ec +To check the number of bytes (remember that one byte contains eight bits for double precision variables), you can use simple use the _itemsize_ functionality (the array $x$ is actually an object which inherits the functionalities defined in Numpy) as +!bc pycod +import numpy as np +x = np.log(np.array([4.0, 7.0, 8.0]) +print(x.itemsize) +!ec + +!split +===== Matrices in Python ===== + +Having defined vectors, we are now ready to try out matrices. We can +define a $3 \times 3 $ real matrix $\hat{A}$ as (recall that we user +lowercase letters for vectors and uppercase letters for matrices) + +!bc pycod +import numpy as np +A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ])) +print(A) +!ec +If we use the _shape_ function we would get $(3, 3)$ as output, that is verifying that our matrix is a $3\times 3$ matrix. We can slice the matrix and print for example the first column (Python organized matrix elements in a row-major order, see below) as +!bc pycod +import numpy as np +A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ])) +# print the first column, row-major order and elements start with 0 +print(A[:,0]) +!ec +We can continue this was by printing out other columns or rows. The example here prints out the second column +!bc pycod +import numpy as np +A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ])) +# print the first column, row-major order and elements start with 0 +print(A[1,:]) +!ec +Numpy contains many other functionalities that allow us to slice, subdivide etc etc arrays. We strongly recommend that you look up the "Numpy website for more details":"http://www.numpy.org/". Useful functions when defining a matrix are the _np.zeros_ function which declares a matrix of a given dimension and sets all elements to zero +!bc pycod +import numpy as np +n = 10 +# define a matrix of dimension 10 x 10 and set all elements to zero +A = np.zeros( (n, n) ) +print(A) +!ec +or initializing all elements to +!bc pycod +import numpy as np +n = 10 +# define a matrix of dimension 10 x 10 and set all elements to one +A = np.ones( (n, n) ) +print(A) +!ec +or as unitarily distributed random numbers (see the material on random number generators in the statistics part) +!bc pycod +import numpy as np +n = 10 +# define a matrix of dimension 10 x 10 and set all elements to random numbers with x \in [0, 1] +A = np.random.rand(n, n) +print(A) +!ec + +As we will see throughout these lectures, there are several extremely useful functionalities in Numpy. +As an example, consider the discussion of the covariance matrix. Suppose we have defined three vectors +$\hat{x}, \hat{y}, \hat{z}$ with $n$ elements each. The covariance matrix is defined as +!bt +\[ +\hat{\Sigma} = \begin{bmatrix} \sigma_{xx} & \sigma_{xy} & \sigma_{xz} \\ + \sigma_{yx} & \sigma_{yy} & \sigma_{yz} \\ + \sigma_{zx} & \sigma_{zy} & \sigma_{zz} + \end{bmatrix}, +\] +!et +where for example +!bt +\[ +\sigma_{xy} =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\] +!et +The Numpy function _np.cov_ calculates the covariance elements using the factor $1/(n-1)$ instead of $1/n$ since it assumes we do not have the exact mean values. +The following simple function uses the _np.vstack_ function which takes each vector of dimension $1\times n$ and produces a $3\times n$ matrix $\hat{W}$ +!bt +\[ +\hat{W} = \begin{bmatrix} x_0 & y_0 & z_0 \\ + x_1 & y_1 & z_1 \\ + x_2 & y_2 & z_2 \\ + \dots & \dots & \dots \\ + x_{n-2} & y_{n-2} & z_{n-2} \\ + x_{n-1} & y_{n-1} & z_{n-1} + \end{bmatrix}, +\] +!et + +which in turn is converted into into the $3\times 3$ covariance matrix +$\hat{\Sigma}$ via the Numpy function _np.cov()_. We note that we can also calculate +the mean value of each set of samples $\hat{x}$ etc using the Numpy +function _np.mean(x)_. We can also extract the eigenvalues of the +covariance matrix through the _np.linalg.eig()_ function. + +!bc pycod +# Importing various packages +import numpy as np + +n = 100 +x = np.random.normal(size=n) +print(np.mean(x)) +y = 4+3*x+np.random.normal(size=n) +print(np.mean(y)) +z = x**3+np.random.normal(size=n) +print(np.mean(z)) +W = np.vstack((x, y, z)) +Sigma = np.cov(W) +print(Sigma) +Eigvals, Eigvecs = np.linalg.eig(Sigma) +print(Eigvals) +!ec + + +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from scipy import sparse +eye = np.eye(4) +print(eye) +sparse_mtx = sparse.csr_matrix(eye) +print(sparse_mtx) +x = np.linspace(-10,10,100) +y = np.sin(x) +plt.plot(x,y,marker='x') +plt.show() +!ec + +!split +===== Meet the Pandas ===== + + +FIGURE: [fig/pandas.jpg, width=600 frac=0.8] + +Another useful Python package is +"pandas":"https://pandas.pydata.org/", which is an open source library +providing high-performance, easy-to-use data structures and data +analysis tools for Python. _pandas_ stands for panel data, a term borrowed from econometrics and is an efficient library for data analysis with an emphasis on tabular data. +_pandas_ has two major classes, the _DataFrame_ class with two-dimensional data objects and tabular data organized in columns and the class _Series_ with a focus on one-dimensional data objects. Both classes allow you to index data easily as we will see in the examples below. +_pandas_ allows you also to perform mathematical operations on the data, spanning from simple reshapings of vectors and matrices to statistical operations. + +The following simple example shows how we can, in an easy way make tables of our data. Here we define a data set which includes names, place of birth and date of birth, and displays the data in an easy to read way. We will see repeated use of _pandas_, in particular in connection with classification of data. + +!bc pycod +import pandas as pd +from IPython.display import display +data = {'First Name': ["Frodo", "Bilbo", "Aragorn II", "Samwise"], + 'Last Name': ["Baggins", "Baggins","Elessar","Gamgee"], + 'Place of birth': ["Shire", "Shire", "Eriador", "Shire"], + 'Date of Birth T.A.': [2968, 2890, 2931, 2980] + } +data_pandas = pd.DataFrame(data) +display(data_pandas) +!ec + +In the above we have imported _pandas_ with the shorthand _pd_, the latter has become the standard way we import _pandas_. We make then a list of various variables +and reorganize the aboves lists into a _DataFrame_ and then print out a neat table with specific column labels as *Name*, *place of birth* and *date of birth*. +Displaying these results, we see that the indices are given by the default numbers from zero to three. +_pandas_ is extremely flexible and we can easily change the above indices by defining a new type of indexing as +!bc pycod +data_pandas = pd.DataFrame(data,index=['Frodo','Bilbo','Aragorn','Sam']) +display(data_pandas) +!ec +Thereafter we display the content of the row which begins with the index _Aragorn_ +!bc pycod +display(data_pandas.loc['Aragorn']) +!ec + +We can easily append data to this, for example +!bc pycod +new_hobbit = {'First Name': ["Peregrin"], + 'Last Name': ["Took"], + 'Place of birth': ["Shire"], + 'Date of Birth T.A.': [2990] + } +data_pandas=data_pandas.append(pd.DataFrame(new_hobbit, index=['Pippin'])) +display(data_pandas) +!ec + + +Here are other examples where we use the _DataFrame_ functionality to handle arrays, now with more interesting features for us, namely numbers. We set up a matrix +of dimensionality $10\times 5$ and compute the mean value and standard deviation of each column. Similarly, we can perform mathematial operations like squaring the matrix elements and many other operations. +!bc pycod +import numpy as np +import pandas as pd +from IPython.display import display +np.random.seed(100) +# setting up a 10 x 5 matrix +rows = 10 +cols = 5 +a = np.random.randn(rows,cols) +df = pd.DataFrame(a) +display(df) +print(df.mean()) +print(df.std()) +display(df**2) +!ec + +Thereafter we can select specific columns only and plot final results +!bc pycod +df.columns = ['First', 'Second', 'Third', 'Fourth', 'Fifth'] +df.index = np.arange(10) + +display(df) +print(df['Second'].mean() ) +print(df.info()) +print(df.describe()) + +from pylab import plt, mpl +plt.style.use('seaborn') +mpl.rcParams['font.family'] = 'serif' + +df.cumsum().plot(lw=2.0, figsize=(10,6)) +plt.show() + + +df.plot.bar(figsize=(10,6), rot=15) +plt.show() +!ec +We can produce a $4\times 4$ matrix +!bc pycod +b = np.arange(16).reshape((4,4)) +print(b) +df1 = pd.DataFrame(b) +print(df1) +!ec +and many other operations. + +The _Series_ class is another important class included in +_pandas_. You can view it as a specialization of _DataFrame_ but where +we have just a single column of data. It shares many of the same features as _DataFrame. As with _DataFrame_, +most operations are vectorized, achieving thereby a high performance when dealing with computations of arrays, in particular labeled arrays. +As we will see below it leads also to a very concice code close to the mathematical operations we may be interested in. +For multidimensional arrays, we recommend strongly "xarray":"http://xarray.pydata.org/en/stable/". _xarray_ has much of the same flexibility as _pandas_, but allows for the extension to higher dimensions than two. We will see examples later of the usage of both _pandas_ and _xarray_. + + +!split +===== Friday August 21 ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureAug21.mp4?vrtx=view-as-webpage" and "Handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesAugust21.pdf" + + + + +!split +===== Reading Data and fitting ===== + +In order to study various Machine Learning algorithms, we need to +access data. Acccessing data is an essential step in all machine +learning algorithms. In particular, setting up the so-called _design +matrix_ (to be defined below) is often the first element we need in +order to perform our calculations. To set up the design matrix means +reading (and later, when the calculations are done, writing) data +in various formats, The formats span from reading files from disk, +loading data from databases and interacting with online sources +like web application programming interfaces (APIs). + +In handling various input formats, as discussed above, we will mainly stay with _pandas_, +a Python package which allows us, in a seamless and painless way, to +deal with a multitude of formats, from standard _csv_ (comma separated +values) files, via _excel_, _html_ to _hdf5_ formats. With _pandas_ +and the _DataFrame_ and _Series_ functionalities we are able to convert text data +into the calculational formats we need for a specific algorithm. And our code is going to be +pretty close the basic mathematical expressions. + +Our first data set is going to be a classic from nuclear physics, namely all +available data on binding energies. Don't be intimidated if you are not familiar with nuclear physics. It serves simply as an example here of a data set. + +We will show some of the +strengths of packages like _Scikit-Learn_ in fitting nuclear binding energies to +specific functions using linear regression first. Then, as a teaser, we will show you how +you can easily implement other algorithms like decision trees and random forests and neural networks. + +But before we really start with nuclear physics data, let's just look at some simpler polynomial fitting cases, such as, +(don't be offended) fitting straight lines! + +!split +===== Friday August 21 ===== + +!split +=== Simple linear regression model using _scikit-learn_ === + +We start with perhaps our simplest possible example, using _Scikit-Learn_ to perform linear regression analysis on a data set produced by us. + +What follows is a simple Python code where we have defined a function +$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries. +The numbers in the vector $\hat{x}$ are given +by random numbers generated with a uniform distribution with entries +$x_i \in [0,1]$ (more about probability distribution functions +later). These values are then used to define a function $y(x)$ +(tabulated again as a vector) with a linear dependence on $x$ plus a +random noise added via the normal distribution. + + +The Numpy functions are imported used the _import numpy as np_ +statement and the random number generator for the uniform distribution +is called using the function _np.random.rand()_, where we specificy +that we want $100$ random variables. Using Numpy we define +automatically an array with the specified number of elements, $100$ in +our case. With the Numpy function _randn()_ we can compute random +numbers with the normal distribution (mean value $\mu$ equal to zero and +variance $\sigma^2$ set to one) and produce the values of $y$ assuming a linear +dependence as function of $x$ + +!bt +\[ +y = 2x+N(0,1), +\] +!et + +where $N(0,1)$ represents random numbers generated by the normal +distribution. From _Scikit-Learn_ we import then the +_LinearRegression_ functionality and make a prediction $\tilde{y} = +\alpha + \beta x$ using the function _fit(x,y)_. We call the set of +data $(\hat{x},\hat{y})$ for our training data. The Python package +_scikit-learn_ has also a functionality which extracts the above +fitting parameters $\alpha$ and $\beta$ (see below). Later we will +distinguish between training data and test data. + +For plotting we use the Python package +"matplotlib":"https://matplotlib.org/" which produces publication +quality figures. Feel free to explore the extensive +"gallery":"https://matplotlib.org/gallery/index.html" of examples. In +this example we plot our original values of $x$ and $y$ as well as the +prediction _ypredict_ ($\tilde{y}$), which attempts at fitting our +data with a straight line. + +The Python code follows here. +!bc pycod +# Importing various packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression + +x = np.random.rand(100,1) +y = 2*x+np.random.randn(100,1) +linreg = LinearRegression() +linreg.fit(x,y) +xnew = np.array([[0],[1]]) +ypredict = linreg.predict(xnew) + +plt.plot(xnew, ypredict, "r-") +plt.plot(x, y ,'ro') +plt.axis([0,1.0,0, 5.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Simple Linear Regression') +plt.show() +!ec + +This example serves several aims. It allows us to demonstrate several +aspects of data analysis and later machine learning algorithms. The +immediate visualization shows that our linear fit is not +impressive. It goes through the data points, but there are many +outliers which are not reproduced by our linear regression. We could +now play around with this small program and change for example the +factor in front of $x$ and the normal distribution. Try to change the +function $y$ to + +!bt +\[ +y = 10x+0.01 \times N(0,1), +\] +!et + +where $x$ is defined as before. Does the fit look better? Indeed, by +reducing the role of the noise given by the normal distribution we see immediately that +our linear prediction seemingly reproduces better the training +set. However, this testing 'by the eye' is obviouly not satisfactory in the +long run. Here we have only defined the training data and our model, and +have not discussed a more rigorous approach to the _cost_ function. + +We need more rigorous criteria in defining whether we have succeeded or +not in modeling our training data. You will be surprised to see that +many scientists seldomly venture beyond this 'by the eye' approach. A +standard approach for the *cost* function is the so-called $\chi^2$ +function (a variant of the mean-squared error (MSE)) + +!bt +\[ \chi^2 = \frac{1}{n} +\sum_{i=0}^{n-1}\frac{(y_i-\tilde{y}_i)^2}{\sigma_i^2}, +\] +!et + +where $\sigma_i^2$ is the variance (to be defined later) of the entry +$y_i$. We may not know the explicit value of $\sigma_i^2$, it serves +however the aim of scaling the equations and make the cost function +dimensionless. + +Minimizing the cost function is a central aspect of +our discussions to come. Finding its minima as function of the model +parameters ($\alpha$ and $\beta$ in our case) will be a recurring +theme in these series of lectures. Essentially all machine learning +algorithms we will discuss center around the minimization of the +chosen cost function. This depends in turn on our specific +model for describing the data, a typical situation in supervised +learning. Automatizing the search for the minima of the cost function is a +central ingredient in all algorithms. Typical methods which are +employed are various variants of _gradient_ methods. These will be +discussed in more detail later. Again, you'll be surprised to hear that +many practitioners minimize the above function ''by the eye', popularly dubbed as +'chi by the eye'. That is, change a parameter and see (visually and numerically) that +the $\chi^2$ function becomes smaller. + +There are many ways to define the cost function. A simpler approach is to look at the relative difference between the training data and the predicted data, that is we define +the relative error (why would we prefer the MSE instead of the relative error?) as + +!bt +\[ +\epsilon_{\mathrm{relative}}= \frac{\vert \hat{y} -\hat{\tilde{y}}\vert}{\vert \hat{y}\vert}. +\] +!et + +The squared cost function results in an arithmetic mean-unbiased +estimator, and the absolute-value cost function results in a +median-unbiased estimator (in the one-dimensional case, and a +geometric median-unbiased estimator for the multi-dimensional +case). The squared cost function has the disadvantage that it has the tendency +to be dominated by outliers. + +We can modify easily the above Python code and plot the relative error instead +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression + +x = np.random.rand(100,1) +y = 5*x+0.01*np.random.randn(100,1) +linreg = LinearRegression() +linreg.fit(x,y) +ypredict = linreg.predict(x) + +plt.plot(x, np.abs(ypredict-y)/abs(y), "ro") +plt.axis([0,1.0,0.0, 0.5]) +plt.xlabel(r'$x$') +plt.ylabel(r'$\epsilon_{\mathrm{relative}}$') +plt.title(r'Relative error') +plt.show() +!ec + +Depending on the parameter in front of the normal distribution, we may +have a small or larger relative error. Try to play around with +different training data sets and study (graphically) the value of the +relative error. + +As mentioned above, _Scikit-Learn_ has an impressive functionality. +We can for example extract the values of $\alpha$ and $\beta$ and +their error estimates, or the variance and standard deviation and many +other properties from the statistical data analysis. + +Here we show an +example of the functionality of _Scikit-Learn_. +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression +from sklearn.metrics import mean_squared_error, r2_score, mean_squared_log_error, mean_absolute_error + +x = np.random.rand(100,1) +y = 2.0+ 5*x+0.5*np.random.randn(100,1) +linreg = LinearRegression() +linreg.fit(x,y) +ypredict = linreg.predict(x) +print('The intercept alpha: \n', linreg.intercept_) +print('Coefficient beta : \n', linreg.coef_) +# The mean squared error +print("Mean squared error: %.2f" % mean_squared_error(y, ypredict)) +# Explained variance score: 1 is perfect prediction +print('Variance score: %.2f' % r2_score(y, ypredict)) +# Mean squared log error +print('Mean squared log error: %.2f' % mean_squared_log_error(y, ypredict) ) +# Mean absolute error +print('Mean absolute error: %.2f' % mean_absolute_error(y, ypredict)) +plt.plot(x, ypredict, "r-") +plt.plot(x, y ,'ro') +plt.axis([0.0,1.0,1.5, 7.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Linear Regression fit ') +plt.show() + +!ec +The function _coef_ gives us the parameter $\beta$ of our fit while _intercept_ yields +$\alpha$. Depending on the constant in front of the normal distribution, we get values near or far from $alpha =2$ and $\beta =5$. Try to play around with different parameters in front of the normal distribution. The function _meansquarederror_ gives us the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error or loss defined as +!bt +\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n} +\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2, +\] +!et + +The smaller the value, the better the fit. Ideally we would like to +have an MSE equal zero. The attentive reader has probably recognized +this function as being similar to the $\chi^2$ function defined above. + +The _r2score_ function computes $R^2$, the coefficient of +determination. It provides a measure of how well future samples are +likely to be predicted by the model. Best possible score is 1.0 and it +can be negative (because the model can be arbitrarily worse). A +constant model that always predicts the expected value of $\hat{y}$, +disregarding the input features, would get a $R^2$ score of $0.0$. + +If $\tilde{\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as +!bt +\[ +R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2}, +\] +!et +where we have defined the mean value of $\hat{y}$ as +!bt +\[ +\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i. +\] +!et +Another quantity taht we will meet again in our discussions of regression analysis is + the mean absolute error (MAE), a risk metric corresponding to the expected value of the absolute error loss or what we call the $l1$-norm loss. In our discussion above we presented the relative error. +The MAE is defined as follows +!bt +\[ +\text{MAE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n-1} \left| y_i - \tilde{y}_i \right|. +\] +!et +We present the +squared logarithmic (quadratic) error +!bt +\[ +\text{MSLE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n - 1} (\log_e (1 + y_i) - \log_e (1 + \tilde{y}_i) )^2, +\] +!et + +where $\log_e (x)$ stands for the natural logarithm of $x$. This error +estimate is best to use when targets having exponential growth, such +as population counts, average sales of a commodity over a span of +years etc. + + +Finally, another cost function is the Huber cost function used in robust regression. + +The rationale behind this possible cost function is its reduced +sensitivity to outliers in the data set. In our discussions on +dimensionality reduction and normalization of data we will meet other +ways of dealing with outliers. + +The Huber cost function is defined as +!bt +\[ +H_{\delta}(a)={\begin{cases}{\frac {1}{2}}{a^{2}}&{\text{for }}|a|\leq \delta ,\\\delta (|a|-{\frac {1}{2}}\delta ),&{\text{otherwise.}}\end{cases}}}. +\] +!et +Here $a=\bm{y} - \bm{\tilde{y}}$. +We will discuss in more +detail these and other functions in the various lectures. We conclude this part with another example. Instead of +a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn. + +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +import random +from sklearn.linear_model import Ridge +from sklearn.preprocessing import PolynomialFeatures +from sklearn.pipeline import make_pipeline +from sklearn.linear_model import LinearRegression + +x=np.linspace(0.02,0.98,200) +noise = np.asarray(random.sample((range(200)),200)) +y=x**3*noise +yn=x**3*100 +poly3 = PolynomialFeatures(degree=3) +X = poly3.fit_transform(x[:,np.newaxis]) +clf3 = LinearRegression() +clf3.fit(X,y) + +Xplot=poly3.fit_transform(x[:,np.newaxis]) +poly3_plot=plt.plot(x, clf3.predict(Xplot), label='Cubic Fit') +plt.plot(x,yn, color='red', label="True Cubic") +plt.scatter(x, y, label='Data', color='orange', s=15) +plt.legend() +plt.show() + +def error(a): + for i in y: + err=(y-yn)/yn + return abs(np.sum(err))/len(err) + +print (error(y)) +!ec + + + + +=== To our real data: nuclear binding energies. Brief reminder on masses and binding energies === + +Let us now dive into nuclear physics and remind ourselves briefly about some basic features about binding +energies. A basic quantity which can be measured for the ground +states of nuclei is the atomic mass $M(N, Z)$ of the neutral atom with +atomic mass number $A$ and charge $Z$. The number of neutrons is $N$. There are indeed several sophisticated experiments worldwide which allow us to measure this quantity to high precision (parts per million even). + +Atomic masses are usually tabulated in terms of the mass excess defined by +!bt +\[ +\Delta M(N, Z) = M(N, Z) - uA, +\] +!et +where $u$ is the Atomic Mass Unit +!bt +\[ +u = M(^{12}\mathrm{C})/12 = 931.4940954(57) \hspace{0.1cm} \mathrm{MeV}/c^2. +\] +!et +The nucleon masses are +!bt +\[ +m_p = 1.00727646693(9)u, +\] +!et +and +!bt +\[ +m_n = 939.56536(8)\hspace{0.1cm} \mathrm{MeV}/c^2 = 1.0086649156(6)u. +\] +!et + +In the "2016 mass evaluation of by W.J.Huang, G.Audi, M.Wang, F.G.Kondev, S.Naimi and X.Xu":"http://nuclearmasses.org/resources_folder/Wang_2017_Chinese_Phys_C_41_030003.pdf" +there are data on masses and decays of 3437 nuclei. + +The nuclear binding energy is defined as the energy required to break +up a given nucleus into its constituent parts of $N$ neutrons and $Z$ +protons. In terms of the atomic masses $M(N, Z)$ the binding energy is +defined by + + +!bt +\[ +BE(N, Z) = ZM_H c^2 + Nm_n c^2 - M(N, Z)c^2 , +\] +!et +where $M_H$ is the mass of the hydrogen atom and $m_n$ is the mass of the neutron. +In terms of the mass excess the binding energy is given by +!bt +\[ +BE(N, Z) = Z\Delta_H c^2 + N\Delta_n c^2 -\Delta(N, Z)c^2 , +\] +!et +where $\Delta_H c^2 = 7.2890$ MeV and $\Delta_n c^2 = 8.0713$ MeV. + + +A popular and physically intuitive model which can be used to parametrize +the experimental binding energies as function of $A$, is the so-called +_liquid drop model_. The ansatz is based on the following expression + +!bt +\[ +BE(N,Z) = a_1A-a_2A^{2/3}-a_3\frac{Z^2}{A^{1/3}}-a_4\frac{(N-Z)^2}{A}, +\] +!et + +where $A$ stands for the number of nucleons and the $a_i$s are parameters which are determined by a fit +to the experimental data. + + + + +To arrive at the above expression we have assumed that we can make the following assumptions: + + * There is a volume term $a_1A$ proportional with the number of nucleons (the energy is also an extensive quantity). When an assembly of nucleons of the same size is packed together into the smallest volume, each interior nucleon has a certain number of other nucleons in contact with it. This contribution is proportional to the volume. + + * There is a surface energy term $a_2A^{2/3}$. The assumption here is that a nucleon at the surface of a nucleus interacts with fewer other nucleons than one in the interior of the nucleus and hence its binding energy is less. This surface energy term takes that into account and is therefore negative and is proportional to the surface area. + + + * There is a Coulomb energy term $a_3\frac{Z^2}{A^{1/3}}$. The electric repulsion between each pair of protons in a nucleus yields less binding. + + * There is an asymmetry term $a_4\frac{(N-Z)^2}{A}$. This term is associated with the Pauli exclusion principle and reflects the fact that the proton-neutron interaction is more attractive on the average than the neutron-neutron and proton-proton interactions. + +We could also add a so-called pairing term, which is a correction term that +arises from the tendency of proton pairs and neutron pairs to +occur. An even number of particles is more stable than an odd number. + + +=== Organizing our data === + +Let us start with reading and organizing our data. +We start with the compilation of masses and binding energies from 2016. +After having downloaded this file to our own computer, we are now ready to read the file and start structuring our data. + + +We start with preparing folders for storing our calculations and the data file over masses and binding energies. We import also various modules that we will find useful in order to present various Machine Learning methods. Here we focus mainly on the functionality of _scikit-learn_. +!bc pycod +# Common imports +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import sklearn.linear_model as skl +from sklearn.model_selection import train_test_split +from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error +import os + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("MassEval2016.dat"),'r') +!ec + + +Before we proceed, we define also a function for making our plots. You can obviously avoid this and simply set up various _matplotlib_ commands every time you need them. You may however find it convenient to collect all such commands in one function and simply call this function. +!bc pycod +from pylab import plt, mpl +plt.style.use('seaborn') +mpl.rcParams['font.family'] = 'serif' + +def MakePlot(x,y, styles, labels, axlabels): + plt.figure(figsize=(10,6)) + for i in range(len(x)): + plt.plot(x[i], y[i], styles[i], label = labels[i]) + plt.xlabel(axlabels[0]) + plt.ylabel(axlabels[1]) + plt.legend(loc=0) +!ec + +Our next step is to read the data on experimental binding energies and +reorganize them as functions of the mass number $A$, the number of +protons $Z$ and neutrons $N$ using _pandas_. Before we do this it is +always useful (unless you have a binary file or other types of compressed +data) to actually open the file and simply take a look at it! + + +In particular, the program that outputs the final nuclear masses is written in Fortran with a specific format. It means that we need to figure out the format and which columns contain the data we are interested in. Pandas comes with a function that reads formatted output. After having admired the file, we are now ready to start massaging it with _pandas_. The file begins with some basic format information. +!bc pycod +""" +This is taken from the data file of the mass 2016 evaluation. +All files are 3436 lines long with 124 character per line. + Headers are 39 lines long. + col 1 : Fortran character control: 1 = page feed 0 = line feed + format : a1,i3,i5,i5,i5,1x,a3,a4,1x,f13.5,f11.5,f11.3,f9.3,1x,a2,f11.3,f9.3,1x,i3,1x,f12.5,f11.5 + These formats are reflected in the pandas widths variable below, see the statement + widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1), + Pandas has also a variable header, with length 39 in this case. +""" +!ec + +The data we are interested in are in columns 2, 3, 4 and 11, giving us +the number of neutrons, protons, mass numbers and binding energies, +respectively. We add also for the sake of completeness the element name. The data are in fixed-width formatted lines and we will +covert them into the _pandas_ DataFrame structure. + +!bc pycod +# Read the experimental data with Pandas +Masses = pd.read_fwf(infile, usecols=(2,3,4,6,11), + names=('N', 'Z', 'A', 'Element', 'Ebinding'), + widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1), + header=39, + index_col=False) + +# Extrapolated values are indicated by '#' in place of the decimal place, so +# the Ebinding column won't be numeric. Coerce to float and drop these entries. +Masses['Ebinding'] = pd.to_numeric(Masses['Ebinding'], errors='coerce') +Masses = Masses.dropna() +# Convert from keV to MeV. +Masses['Ebinding'] /= 1000 + +# Group the DataFrame by nucleon number, A. +Masses = Masses.groupby('A') +# Find the rows of the grouped DataFrame with the maximum binding energy. +Masses = Masses.apply(lambda t: t[t.Ebinding==t.Ebinding.max()]) +!ec + +We have now read in the data, grouped them according to the variables we are interested in. +We see how easy it is to reorganize the data using _pandas_. If we +were to do these operations in C/C++ or Fortran, we would have had to +write various functions/subroutines which perform the above +reorganizations for us. Having reorganized the data, we can now start +to make some simple fits using both the functionalities in _numpy_ and +_Scikit-Learn_ afterwards. + +Now we define five variables which contain +the number of nucleons $A$, the number of protons $Z$ and the number of neutrons $N$, the element name and finally the energies themselves. +!bc pycod +A = Masses['A'] +Z = Masses['Z'] +N = Masses['N'] +Element = Masses['Element'] +Energies = Masses['Ebinding'] +print(Masses) +!ec +The next step, and we will define this mathematically later, is to set up the so-called _design matrix_. We will throughout call this matrix $\bm{X}$. +It has dimensionality $p\times n$, where $n$ is the number of data points and $p$ are the so-called predictors. In our case here they are given by the number of polynomials in $A$ we wish to include in the fit. +!bc pycod +# Now we set up the design matrix X +X = np.zeros((len(A),5)) +X[:,0] = 1 +X[:,1] = A +X[:,2] = A**(2.0/3.0) +X[:,3] = A**(-1.0/3.0) +X[:,4] = A**(-1.0) +!ec +With _scikitlearn_ we are now ready to use linear regression and fit our data. +!bc pycod +clf = skl.LinearRegression().fit(X, Energies) +fity = clf.predict(X) +!ec +Pretty simple! +Now we can print measures of how our fit is doing, the coefficients from the fits and plot the final fit together with our data. +!bc pycod +# The mean squared error +print("Mean squared error: %.2f" % mean_squared_error(Energies, fity)) +# Explained variance score: 1 is perfect prediction +print('Variance score: %.2f' % r2_score(Energies, fity)) +# Mean absolute error +print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity)) +print(clf.coef_, clf.intercept_) + +Masses['Eapprox'] = fity +# Generate a plot comparing the experimental with the fitted values values. +fig, ax = plt.subplots() +ax.set_xlabel(r'$A = N + Z$') +ax.set_ylabel(r'$E_\mathrm{bind}\,/\mathrm{MeV}$') +ax.plot(Masses['A'], Masses['Ebinding'], alpha=0.7, lw=2, + label='Ame2016') +ax.plot(Masses['A'], Masses['Eapprox'], alpha=0.7, lw=2, c='m', + label='Fit') +ax.legend() +save_fig("Masses2016") +plt.show() +!ec + + +=== Seeing the wood for the trees === + +As a teaser, let us now see how we can do this with decision trees using _scikit-learn_. Later we will switch to so-called _random forests_! + + +!bc pycod + +#Decision Tree Regression +from sklearn.tree import DecisionTreeRegressor +regr_1=DecisionTreeRegressor(max_depth=5) +regr_2=DecisionTreeRegressor(max_depth=7) +regr_3=DecisionTreeRegressor(max_depth=9) +regr_1.fit(X, Energies) +regr_2.fit(X, Energies) +regr_3.fit(X, Energies) + + +y_1 = regr_1.predict(X) +y_2 = regr_2.predict(X) +y_3=regr_3.predict(X) +Masses['Eapprox'] = y_3 +# Plot the results +plt.figure() +plt.plot(A, Energies, color="blue", label="Data", linewidth=2) +plt.plot(A, y_1, color="red", label="max_depth=5", linewidth=2) +plt.plot(A, y_2, color="green", label="max_depth=7", linewidth=2) +plt.plot(A, y_3, color="m", label="max_depth=9", linewidth=2) + +plt.xlabel("$A$") +plt.ylabel("$E$[MeV]") +plt.title("Decision Tree Regression") +plt.legend() +save_fig("Masses2016Trees") +plt.show() +print(Masses) +print(np.mean( (Energies-y_1)**2)) +!ec + + +=== And what about using neural networks? === +The _seaborn_ package allows us to visualize data in an efficient way. Note that we use _scikit-learn_'s multi-layer perceptron (or feed forward neural network) +functionality. +!bc pycod +from sklearn.neural_network import MLPRegressor +from sklearn.metrics import accuracy_score +import seaborn as sns + +X_train = X +Y_train = Energies +n_hidden_neurons = 100 +epochs = 100 +# store models for later use +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store the models for later use +DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +sns.set() +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = MLPRegressor(hidden_layer_sizes=(n_hidden_neurons), activation='logistic', + alpha=lmbd, learning_rate_init=eta, max_iter=epochs) + dnn.fit(X_train, Y_train) + DNN_scikit[i][j] = dnn + train_accuracy[i][j] = dnn.score(X_train, Y_train) + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + + + +!ec + + + + + + +===== A first summary ===== + +The aim behind these introductory words was to present to you various +Python libraries and their functionalities, in particular libraries like +_numpy_, _pandas_, _xarray_ and _matplotlib_ and other that make our life much easier +in handling various data sets and visualizing data. + +Furthermore, +_Scikit-Learn_ allows us with few lines of code to implement popular +Machine Learning algorithms for supervised learning. Later we will meet _Tensorflow_, a powerful library for deep learning. +Now it is time to dive more into the details of various methods. We will start with linear regression and try to take a deeper look at what it entails. + + + + diff --git a/doc/BookChapters/chapter2.dlog b/doc/BookChapters/chapter2.dlog new file mode 100644 index 000000000..3d924b166 --- /dev/null +++ b/doc/BookChapters/chapter2.dlog @@ -0,0 +1,7 @@ +Translating doconce text in chapter2.do.txt to ipynb +*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax) + +*** warning: latex envir \begin{eqnarray*} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. +Failed to remove ans_at_end environment +Failed to remove sol_at_end environment +output in chapter2.ipynb diff --git a/doc/BookChapters/chapter2.do.txt b/doc/BookChapters/chapter2.do.txt new file mode 100644 index 000000000..fd989f613 --- /dev/null +++ b/doc/BookChapters/chapter2.do.txt @@ -0,0 +1,1075 @@ +======= Resampling Methods ======= + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSept3.mp4?vrtx=view-as-webpage" + + +===== Introduction ===== + +Resampling methods are an indispensable tool in modern +statistics. They involve repeatedly drawing samples from a training +set and refitting a model of interest on each sample in order to +obtain additional information about the fitted model. For example, in +order to estimate the variability of a linear regression fit, we can +repeatedly draw different samples from the training data, fit a linear +regression to each new sample, and then examine the extent to which +the resulting fits differ. Such an approach may allow us to obtain +information that would not be available from fitting the model only +once using the original training sample. + +Two resampling methods are often used in Machine Learning analyses, +o The _bootstrap method_ +o and _Cross-Validation_ + +In addition there are several other methods such as the Jackknife and the Blocking methods. We will discuss in particular +cross-validation and the bootstrap method. + + +Resampling approaches can be computationally expensive, because they +involve fitting the same statistical method multiple times using +different subsets of the training data. However, due to recent +advances in computing power, the computational requirements of +resampling methods generally are not prohibitive. In this chapter, we +discuss two of the most commonly used resampling methods, +cross-validation and the bootstrap. Both methods are important tools +in the practical application of many statistical learning +procedures. For example, cross-validation can be used to estimate the +test error associated with a given statistical learning method in +order to evaluate its performance, or to select the appropriate level +of flexibility. The process of evaluating a model’s performance is +known as model assessment, whereas the process of selecting the proper +level of flexibility for a model is known as model selection. The +bootstrap is widely used. + + +* Our simulations can be treated as *computer experiments*. This is particularly the case for Monte Carlo methods +* The results can be analysed with the same statistical tools as we would use analysing experimental data. +* As in all experiments, we are looking for expectation values and an estimate of how accurate they are, i.e., possible sources for errors. + +===== Reminder on Statistics ===== + + +* As in other experiments, many numerical experiments have two classes of errors: + * Statistical errors + * Systematical errors +* Statistical errors can be estimated using standard tools from statistics +* Systematical errors are method specific and must be treated differently from case to case. + +The +advantage of doing linear regression is that we actually end up with +analytical expressions for several statistical quantities. +Standard least squares and Ridge regression allow us to +derive quantities like the variance and other expectation values in a +rather straightforward way. + + +It is assumed that $\varepsilon_i +\sim \mathcal{N}(0, \sigma^2)$ and the $\varepsilon_{i}$ are +independent, i.e.: +!bt +\begin{align*} +\mbox{Cov}(\varepsilon_{i_1}, +\varepsilon_{i_2}) & = \left\{ \begin{array}{lcc} \sigma^2 & \mbox{if} +& i_1 = i_2, \\ 0 & \mbox{if} & i_1 \not= i_2. \end{array} \right. +\end{align*} +!et +The randomness of $\varepsilon_i$ implies that +$\mathbf{y}_i$ is also a random variable. In particular, +$\mathbf{y}_i$ is normally distributed, because $\varepsilon_i \sim +\mathcal{N}(0, \sigma^2)$ and $\mathbf{X}_{i,\ast} \, \bm{\beta}$ is a +non-random scalar. To specify the parameters of the distribution of +$\mathbf{y}_i$ we need to calculate its first two moments. + +Recall that $\bm{X}$ is a matrix of dimensionality $n\times p$. The +notation above $\mathbf{X}_{i,\ast}$ means that we are looking at the +row number $i$ and perform a sum over all values $p$. + + +The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off) +that there exists a function $f(\bm{x})$ and a normal distributed error $\bm{\varepsilon}\sim \mathcal{N}(0, \sigma^2)$ +which describe our data +!bt +\[ +\bm{y} = f(\bm{x})+\bm{\varepsilon} +\] +!et + +We approximate this function with our model from the solution of the linear regression equations, that is our +function $f$ is approximated by $\bm{\tilde{y}}$ where we want to minimize $(\bm{y}-\bm{\tilde{y}})^2$, our MSE, with +!bt +\[ +\bm{\tilde{y}} = \bm{X}\bm{\beta}. +\] +!et + + +We can calculate the expectation value of $\bm{y}$ for a given element $i$ +!bt +\begin{align*} +\mathbb{E}(y_i) & = +\mathbb{E}(\mathbf{X}_{i, \ast} \, \bm{\beta}) + \mathbb{E}(\varepsilon_i) +\, \, \, = \, \, \, \mathbf{X}_{i, \ast} \, \beta, +\end{align*} +!et +while +its variance is +!bt +\begin{align*} \mbox{Var}(y_i) & = \mathbb{E} \{ [y_i +- \mathbb{E}(y_i)]^2 \} \, \, \, = \, \, \, \mathbb{E} ( y_i^2 ) - +[\mathbb{E}(y_i)]^2 \\ & = \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, +\beta + \varepsilon_i )^2] - ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 \\ & += \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 + 2 \varepsilon_i +\mathbf{X}_{i, \ast} \, \bm{\beta} + \varepsilon_i^2 ] - ( \mathbf{X}_{i, +\ast} \, \beta)^2 \\ & = ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 + 2 +\mathbb{E}(\varepsilon_i) \mathbf{X}_{i, \ast} \, \bm{\beta} + +\mathbb{E}(\varepsilon_i^2 ) - ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 +\\ & = \mathbb{E}(\varepsilon_i^2 ) \, \, \, = \, \, \, +\mbox{Var}(\varepsilon_i) \, \, \, = \, \, \, \sigma^2. +\end{align*} +!et +Hence, $y_i \sim \mathcal{N}( \mathbf{X}_{i, \ast} \, \bm{\beta}, \sigma^2)$, that is $\bm{y}$ follows a normal distribution with +mean value $\bm{X}\bm{\beta}$ and variance $\sigma^2$ (not be confused with the singular values of the SVD). + + +With the OLS expressions for the parameters $\bm{\beta}$ we can evaluate the expectation value +!bt +\[ +\mathbb{E}(\bm{\beta}) = \mathbb{E}[ (\mathbf{X}^{\top} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbb{E}[ \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1} \mathbf{X}^{T}\mathbf{X}\bm{\beta}=\bm{\beta}. +\] +!et +This means that the estimator of the regression parameters is unbiased. + +We can also calculate the variance + +The variance of $\bm{\beta}$ is +!bt +\begin{eqnarray*} +\mbox{Var}(\bm{\beta}) & = & \mathbb{E} \{ [\bm{\beta} - \mathbb{E}(\bm{\beta})] [\bm{\beta} - \mathbb{E}(\bm{\beta})]^{T} \} +\\ +& = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \bm{\beta}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \bm{\beta}]^{T} \} +\\ +% & = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}]^{T} \} - \bm{\beta} \, \bm{\beta}^{T} +% \\ +% & = & \mathbb{E} \{ (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} \, \mathbf{Y}^{T} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} \} - \bm{\beta} \, \bm{\beta}^{T} +% \\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \mathbb{E} \{ \mathbf{Y} \, \mathbf{Y}^{T} \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \bm{\beta} \, \bm{\beta}^{T} +\\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \{ \mathbf{X} \, \bm{\beta} \, \bm{\beta}^{T} \, \mathbf{X}^{T} + \sigma^2 \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \bm{\beta} \, \bm{\beta}^{T} +% \\ +% & = & (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, \bm{\beta} \, \bm{\beta}^T \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T % \mathbf{X})^{-1} +% \\ +% & & + \, \, \sigma^2 \, (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T \mathbf{X})^{-1} - \bm{\beta} \bm{\beta}^T +\\ +& = & \bm{\beta} \, \bm{\beta}^{T} + \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \bm{\beta} \, \bm{\beta}^{T} +\, \, \, = \, \, \, \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}, +\end{eqnarray*} +!et + +where we have used that $\mathbb{E} (\mathbf{Y} \mathbf{Y}^{T}) = +\mathbf{X} \, \bm{\beta} \, \bm{\beta}^{T} \, \mathbf{X}^{T} + +\sigma^2 \, \mathbf{I}_{nn}$. From $\mbox{Var}(\bm{\beta}) = \sigma^2 +\, (\mathbf{X}^{T} \mathbf{X})^{-1}$, one obtains an estimate of the +variance of the estimate of the $j$-th regression coefficient: +$\bm{\sigma}^2 (\bm{\beta}_j ) = \bm{\sigma}^2 \sqrt{ +[(\mathbf{X}^{T} \mathbf{X})^{-1}]_{jj} }$. This may be used to +construct a confidence interval for the estimates. + + +In a similar way, we can obtain analytical expressions for say the +expectation values of the parameters $\bm{\beta}$ and their variance +when we employ Ridge regression, allowing us again to define a confidence interval. + +It is rather straightforward to show that +!bt +\[ +\mathbb{E} \big[ \bm{\beta}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\bm{\beta}^{\mathrm{OLS}}. +\] +!et +We see clearly that +$\mathbb{E} \big[ \bm{\beta}^{\mathrm{Ridge}} \big] \not= \bm{\beta}^{\mathrm{OLS}}$ for any $\lambda > 0$. We say then that the ridge estimator is biased. + +We can also compute the variance as + +!bt +\[ +\mbox{Var}[\bm{\beta}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T} \mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, +\] +!et +and it is easy to see that if the parameter $\lambda$ goes to infinity then the variance of Ridge parameters $\bm{\beta}$ goes to zero. + +With this, we can compute the difference + +!bt +\[ +\mbox{Var}[\bm{\beta}^{\mathrm{OLS}}]-\mbox{Var}(\bm{\beta}^{\mathrm{Ridge}})=\sigma^2 [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}[ 2\lambda\mathbf{I} + \lambda^2 (\mathbf{X}^{T} \mathbf{X})^{-1} ] \{ [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}. +\] +!et +The difference is non-negative definite since each component of the +matrix product is non-negative definite. +This means the variance we obtain with the standard OLS will always for $\lambda > 0$ be larger than the variance of $\bm{\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below. + + + +===== Resampling methods ===== + +With all these analytical equations for both the OLS and Ridge +regression, we will now outline how to assess a given model. This will +lead us to a discussion of the so-called bias-variance tradeoff (see +below) and so-called resampling methods. + +One of the quantities we have discussed as a way to measure errors is +the mean-squared error (MSE), mainly used for fitting of continuous +functions. Another choice is the absolute error. + +In the discussions below we will focus on the MSE and in particular since we will split the data into test and training data, +we discuss the +o prediction error or simply the _test error_ $\mathrm{Err_{Test}}$, where we have a fixed training set and the test error is the MSE arising from the data reserved for testing. We discuss also the +o training error $\mathrm{Err_{Train}}$, which is the average loss over the training data. + +As our model becomes more and more complex, more of the training data tends to used. The training may thence adapt to more complicated structures in the data. This may lead to a decrease in the bias (see below for code example) and a slight increase of the variance for the test error. +For a certain level of complexity the test error will reach minimum, before starting to increase again. The +training error reaches a saturation. + + + +Two famous +resampling methods are the _independent bootstrap_ and _the jackknife_. + +The jackknife is a special case of the independent bootstrap. Still, the jackknife was made +popular prior to the independent bootstrap. And as the popularity of +the independent bootstrap soared, new variants, such as _the dependent bootstrap_. + +The Jackknife and independent bootstrap work for +independent, identically distributed random variables. +If these conditions are not +satisfied, the methods will fail. Yet, it should be said that if the data are +independent, identically distributed, and we only want to estimate the +variance of $\overline{X}$ (which often is the case), then there is no +need for bootstrapping. + + +The Jackknife works by making many replicas of the estimator $\widehat{\theta}$. +The jackknife is a resampling method where we systematically leave out one observation from the vector of observed values $\bm{x} = (x_1,x_2,\cdots,X_n)$. +Let $\bm{x}_i$ denote the vector +!bt +\[ +\bm{x}_i = (x_1,x_2,\cdots,x_{i-1},x_{i+1},\cdots,x_n), +\] +!et + +which equals the vector $\bm{x}$ with the exception that observation +number $i$ is left out. Using this notation, define +$\widehat{\theta}_i$ to be the estimator +$\widehat{\theta}$ computed using $\vec{X}_i$. + + + +!bc pycod +from numpy import * +from numpy.random import randint, randn +from time import time + +def jackknife(data, stat): + n = len(data);t = zeros(n); inds = arange(n); t0 = time() + ## 'jackknifing' by leaving out an observation for each i + for i in range(n): + t[i] = stat(delete(data,i) ) + + # analysis + print("Runtime: %g sec" % (time()-t0)); print("Jackknife Statistics :") + print("original bias std. error") + print("%8g %14g %15g" % (stat(data),(n-1)*mean(t)/n, (n*var(t))**.5)) + + return t + + +# Returns mean of data samples +def stat(data): + return mean(data) + + +mu, sigma = 100, 15 +datapoints = 10000 +x = mu + sigma*random.randn(datapoints) +# jackknife returns the data sample +t = jackknife(x, stat) + +!ec + + +=== Bootstrap === + +Bootstrapping is a nonparametric approach to statistical inference +that substitutes computation for more traditional distributional +assumptions and asymptotic results. Bootstrapping offers a number of +advantages: +o The bootstrap is quite general, although there are some cases in which it fails. +o Because it does not require distributional assumptions (such as normally distributed errors), the bootstrap can provide more accurate inferences when the data are not well behaved or when the sample size is small. +o It is possible to apply the bootstrap to statistics with sampling distributions that are difficult to derive, even asymptotically. +o It is relatively simple to apply the bootstrap to complex data-collection plans (such as stratified and clustered samples). + + + +Since $\widehat{\theta} = \widehat{\theta}(\bm{X})$ is a function of random variables, +$\widehat{\theta}$ itself must be a random variable. Thus it has +a pdf, call this function $p(\bm{t})$. The aim of the bootstrap is to +estimate $p(\bm{t})$ by the relative frequency of +$\widehat{\theta}$. You can think of this as using a histogram +in the place of $p(\bm{t})$. If the relative frequency closely +resembles $p(\vec{t})$, then using numerics, it is straight forward to +estimate all the interesting parameters of $p(\bm{t})$ using point +estimators. + + + +In the case that $\widehat{\theta}$ has +more than one component, and the components are independent, we use the +same estimator on each component separately. If the probability +density function of $X_i$, $p(x)$, had been known, then it would have +been straight forward to do this by: +o Drawing lots of numbers from $p(x)$, suppose we call one such set of numbers $(X_1^*, X_2^*, \cdots, X_n^*)$. +o Then using these numbers, we could compute a replica of $\widehat{\theta}$ called $\widehat{\theta}^*$. + +By repeated use of (1) and (2), many +estimates of $\widehat{\theta}$ could have been obtained. The +idea is to use the relative frequency of $\widehat{\theta}^*$ +(think of a histogram) as an estimate of $p(\bm{t})$. + + +But +unless there is enough information available about the process that +generated $X_1,X_2,\cdots,X_n$, $p(x)$ is in general +unknown. Therefore, "Efron in 1979":"https://projecteuclid.org/euclid.aos/1176344552" asked the +question: What if we replace $p(x)$ by the relative frequency +of the observation $X_i$; if we draw observations in accordance with +the relative frequency of the observations, will we obtain the same +result in some asymptotic sense? The answer is yes. + + +Instead of generating the histogram for the relative +frequency of the observation $X_i$, just draw the values +$(X_1^*,X_2^*,\cdots,X_n^*)$ with replacement from the vector +$\bm{X}$. + + +The independent bootstrap works like this: + +o Draw with replacement $n$ numbers for the observed variables $\bm{x} = (x_1,x_2,\cdots,x_n)$. +o Define a vector $\bm{x}^*$ containing the values which were drawn from $\bm{x}$. +o Using the vector $\bm{x}^*$ compute $\widehat{\theta}^*$ by evaluating $\widehat \theta$ under the observations $\bm{x}^*$. +o Repeat this process $k$ times. + +When you are done, you can draw a histogram of the relative frequency +of $\widehat \theta^*$. This is your estimate of the probability +distribution $p(t)$. Using this probability distribution you can +estimate any statistics thereof. In principle you never draw the +histogram of the relative frequency of $\widehat{\theta}^*$. Instead +you use the estimators corresponding to the statistic of interest. For +example, if you are interested in estimating the variance of $\widehat +\theta$, apply the etsimator $\widehat \sigma^2$ to the values +$\widehat \theta ^*$. + + + +The following code starts with a Gaussian distribution with mean value +$\mu =100$ and variance $\sigma=15$. We use this to generate the data +used in the bootstrap analysis. The bootstrap analysis returns a data +set after a given number of bootstrap operations (as many as we have +data points). This data set consists of estimated mean values for each +bootstrap operation. The histogram generated by the bootstrap method +shows that the distribution for these mean values is also a Gaussian, +centered around the mean value $\mu=100$ but with standard deviation +$\sigma/\sqrt{n}$, where $n$ is the number of bootstrap samples (in +this case the same as the number of original data points). The value +of the standard deviation is what we expect from the central limit +theorem. + + +!bc pycod +from numpy import * +from numpy.random import randint, randn +from time import time +import matplotlib.mlab as mlab +import matplotlib.pyplot as plt + +# Returns mean of bootstrap samples +def stat(data): + return mean(data) + +# Bootstrap algorithm +def bootstrap(data, statistic, R): + t = zeros(R); n = len(data); inds = arange(n); t0 = time() + # non-parametric bootstrap + for i in range(R): + t[i] = statistic(data[randint(0,n,n)]) + + # analysis + print("Runtime: %g sec" % (time()-t0)); print("Bootstrap Statistics :") + print("original bias std. error") + print("%8g %8g %14g %15g" % (statistic(data), std(data),mean(t),std(t))) + return t + + +mu, sigma = 100, 15 +datapoints = 10000 +x = mu + sigma*random.randn(datapoints) +# bootstrap returns the data sample +t = bootstrap(x, stat, datapoints) +# the histogram of the bootstrapped data +n, binsboot, patches = plt.hist(t, 50, normed=1, facecolor='red', alpha=0.75) + +# add a 'best fit' line +y = mlab.normpdf( binsboot, mean(t), std(t)) +lt = plt.plot(binsboot, y, 'r--', linewidth=1) +plt.xlabel('Smarts') +plt.ylabel('Probability') +plt.axis([99.5, 100.6, 0, 3.0]) +plt.grid(True) + +plt.show() + +!ec + + + +===== Various steps in cross-validation ===== + +When the repetitive splitting of the data set is done randomly, +samples may accidently end up in a fast majority of the splits in +either training or test set. Such samples may have an unbalanced +influence on either model building or prediction evaluation. To avoid +this $k$-fold cross-validation structures the data splitting. The +samples are divided into $k$ more or less equally sized exhaustive and +mutually exclusive subsets. In turn (at each split) one of these +subsets plays the role of the test set while the union of the +remaining subsets constitutes the training set. Such a splitting +warrants a balanced representation of each sample in both training and +test set over the splits. Still the division into the $k$ subsets +involves a degree of randomness. This may be fully excluded when +choosing $k=n$. This particular case is referred to as leave-one-out +cross-validation (LOOCV). + + +* Define a range of interest for the penalty parameter. + +* Divide the data set into training and test set comprising samples $\{1, \ldots, n\} \setminus i$ and $\{ i \}$, respectively. + +* Fit the linear regression model by means of ridge estimation for each $\lambda$ in the grid using the training set, and the corresponding estimate of the error variance $\bm{\sigma}_{-i}^2(\lambda)$, as +!bt +\begin{align*} +\bm{\beta}_{-i}(\lambda) & = ( \bm{X}_{-i, \ast}^{T} +\bm{X}_{-i, \ast} + \lambda \bm{I}_{pp})^{-1} +\bm{X}_{-i, \ast}^{T} \bm{y}_{-i} +\end{align*} +!et + +* Evaluate the prediction performance of these models on the test set by $\log\{L[y_i, \bm{X}_{i, \ast}; \bm{\beta}_{-i}(\lambda), \bm{\sigma}_{-i}^2(\lambda)]\}$. Or, by the prediction error $|y_i - \bm{X}_{i, \ast} \bm{\beta}_{-i}(\lambda)|$, the relative error, the error squared or the R2 score function. + +* Repeat the first three steps such that each sample plays the role of the test set once. + +* Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as +!bt +\begin{align*} +\frac{1}{n} \sum_{i = 1}^n \log\{L[y_i, \mathbf{X}_{i, \ast}; \bm{\beta}_{-i}(\lambda), \bm{\sigma}_{-i}^2(\lambda)]\}. +\end{align*} +!et + + +For the various values of $k$ + +o shuffle the dataset randomly. +o Split the dataset into $k$ groups. +o For each unique group: + o Decide which group to use as set for test data + o Take the remaining groups as a training data set + o Fit a model on the training set and evaluate it on the test set + o Retain the evaluation score and discard the model +o Summarize the model using the sample of model evaluation scores + + + + +The code here uses Ridge regression with cross-validation (CV) resampling and $k$-fold CV in order to fit a specific polynomial. +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import KFold +from sklearn.linear_model import Ridge +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import PolynomialFeatures + +# A seed just to ensure that the random numbers are the same for every run. +# Useful for eventual debugging. +np.random.seed(3155) + +# Generate the data. +nsamples = 100 +x = np.random.randn(nsamples) +y = 3*x**2 + np.random.randn(nsamples) + +## Cross-validation on Ridge regression using KFold only + +# Decide degree on polynomial to fit +poly = PolynomialFeatures(degree = 6) + +# Decide which values of lambda to use +nlambdas = 500 +lambdas = np.logspace(-3, 5, nlambdas) + +# Initialize a KFold instance +k = 5 +kfold = KFold(n_splits = k) + +# Perform the cross-validation to estimate MSE +scores_KFold = np.zeros((nlambdas, k)) + +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + j = 0 + for train_inds, test_inds in kfold.split(x): + xtrain = x[train_inds] + ytrain = y[train_inds] + + xtest = x[test_inds] + ytest = y[test_inds] + + Xtrain = poly.fit_transform(xtrain[:, np.newaxis]) + ridge.fit(Xtrain, ytrain[:, np.newaxis]) + + Xtest = poly.fit_transform(xtest[:, np.newaxis]) + ypred = ridge.predict(Xtest) + + scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred) + + j += 1 + i += 1 + + +estimated_mse_KFold = np.mean(scores_KFold, axis = 1) + +## Cross-validation using cross_val_score from sklearn along with KFold + +# kfold is an instance initialized above as: +# kfold = KFold(n_splits = k) + +estimated_mse_sklearn = np.zeros(nlambdas) +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + + X = poly.fit_transform(x[:, np.newaxis]) + estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold) + + # cross_val_score return an array containing the estimated negative mse for every fold. + # we have to the the mean of every array in order to get an estimate of the mse of the model + estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds) + + i += 1 + +## Plot and compare the slightly different ways to perform cross-validation + +plt.figure() + +plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score') +plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold') + +plt.xlabel('log10(lambda)') +plt.ylabel('mse') + +plt.legend() + +plt.show() + +!ec + + + +===== The bias-variance tradeoff ===== + + +We will discuss the bias-variance tradeoff in the context of +continuous predictions such as regression. However, many of the +intuitions and ideas discussed here also carry over to classification +tasks. Consider a dataset $\mathcal{L}$ consisting of the data +$\mathbf{X}_\mathcal{L}=\{(y_j, \boldsymbol{x}_j), j=0\ldots n-1\}$. + +Let us assume that the true data is generated from a noisy model + +!bt +\[ +\bm{y}=f(\boldsymbol{x}) + \bm{\epsilon} +\] +!et + +where $\epsilon$ is normally distributed with mean zero and standard deviation $\sigma^2$. + +In our derivation of the ordinary least squares method we defined then +an approximation to the function $f$ in terms of the parameters +$\bm{\beta}$ and the design matrix $\bm{X}$ which embody our model, +that is $\bm{\tilde{y}}=\bm{X}\bm{\beta}$. + +Thereafter we found the parameters $\bm{\beta}$ by optimizing the means squared error via the so-called cost function +!bt +\[ +C(\bm{X},\bm{\beta}) =\frac{1}{n}\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2=\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]. +\] +!et + +We can rewrite this as +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\frac{1}{n}\sum_i(f_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2+\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2+\sigma^2. +\] +!et + +The three terms represent the square of the bias of the learning +method, which can be thought of as the error caused by the simplifying +assumptions built into the method. The second term represents the +variance of the chosen model and finally the last terms is variance of +the error $\bm{\epsilon}$. + +To derive this equation, we need to recall that the variance of $\bm{y}$ and $\bm{\epsilon}$ are both equal to $\sigma^2$. The mean value of $\bm{\epsilon}$ is by definition equal to zero. Furthermore, the function $f$ is not a stochastics variable, idem for $\bm{\tilde{y}}$. +We use a more compact notation in terms of the expectation value +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathbb{E}\left[(\bm{f}+\bm{\epsilon}-\bm{\tilde{y}})^2\right], +\] +!et +and adding and subtracting $\mathbb{E}\left[\bm{\tilde{y}}\right]$ we get +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathbb{E}\left[(\bm{f}+\bm{\epsilon}-\bm{\tilde{y}}+\mathbb{E}\left[\bm{\tilde{y}}\right]-\mathbb{E}\left[\bm{\tilde{y}}\right])^2\right], +\] +!et +which, using the abovementioned expectation values can be rewritten as +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathbb{E}\left[(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right])^2\right]+\mathrm{Var}\left[\bm{\tilde{y}}\right]+\sigma^2, +\] +!et +that is the rewriting in terms of the so-called bias, the variance of the model $\bm{\tilde{y}}$ and the variance of $\bm{\epsilon}$. + + + + +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.preprocessing import PolynomialFeatures +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample + +np.random.seed(2018) + +n = 500 +n_boostraps = 100 +degree = 18 # A quite high value, just to show. +noise = 0.1 + +# Make data set. +x = np.linspace(-1, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1, x.shape) + +# Hold out some test data that is never used in training. +x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +# Combine x transformation and model into one operation. +# Not neccesary, but convenient. +model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False)) + +# The following (m x n_bootstraps) matrix holds the column vectors y_pred +# for each bootstrap iteration. +y_pred = np.empty((y_test.shape[0], n_boostraps)) +for i in range(n_boostraps): + x_, y_ = resample(x_train, y_train) + + # Evaluate the new model on the same test data each time. + y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel() + +# Note: Expectations and variances taken w.r.t. different training +# data sets, hence the axis=1. Subsequent means are taken across the test data +# set in order to obtain a total value, but before this we have error/bias/variance +# calculated per data point in the test set. +# Note 2: The use of keepdims=True is important in the calculation of bias as this +# maintains the column vector form. Dropping this yields very unexpected results. +error = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) +bias = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) +variance = np.mean( np.var(y_pred, axis=1, keepdims=True) ) +print('Error:', error) +print('Bias^2:', bias) +print('Var:', variance) +print('{} >= {} + {} = {}'.format(error, bias, variance, bias+variance)) + +plt.plot(x[::5, :], y[::5, :], label='f(x)') +plt.scatter(x_test, y_test, label='Data points') +plt.scatter(x_test, np.mean(y_pred, axis=1), label='Pred') +plt.legend() +plt.show() + +!ec + + + +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.preprocessing import PolynomialFeatures +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample + +np.random.seed(2018) + +n = 40 +n_boostraps = 100 +maxdegree = 14 + + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +for degree in range(maxdegree): + model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False)) + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(x_train, y_train) + y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel() + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + + +!ec + + + +The bias-variance tradeoff summarizes the fundamental tension in +machine learning, particularly supervised learning, between the +complexity of a model and the amount of training data needed to train +it. Since data is often limited, in practice it is often useful to +use a less-complex model with higher bias, that is a model whose asymptotic +performance is worse than another model because it is easier to +train and less sensitive to sampling noise arising from having a +finite-sized training dataset (smaller variance). + + + +The above equations tell us that in +order to minimize the expected test error, we need to select a +statistical learning method that simultaneously achieves low variance +and low bias. Note that variance is inherently a nonnegative quantity, +and squared bias is also nonnegative. Hence, we see that the expected +test MSE can never lie below $Var(\epsilon)$, the irreducible error. + + +What do we mean by the variance and bias of a statistical learning +method? The variance refers to the amount by which our model would change if we +estimated it using a different training data set. Since the training +data are used to fit the statistical learning method, different +training data sets will result in a different estimate. But ideally the +estimate for our model should not vary too much between training +sets. However, if a method has high variance then small changes in +the training data can result in large changes in the model. In general, more +flexible statistical methods have higher variance. + + +You may also find this recent "article":"https://www.pnas.org/content/116/32/15849" of interest. + + +!bc pycod +""" +============================ +Underfitting vs. Overfitting +============================ + +This example demonstrates the problems of underfitting and overfitting and +how we can use linear regression with polynomial features to approximate +nonlinear functions. The plot shows the function that we want to approximate, +which is a part of the cosine function. In addition, the samples from the +real function and the approximations of different models are displayed. The +models have polynomial features of different degrees. We can see that a +linear function (polynomial with degree 1) is not sufficient to fit the +training samples. This is called **underfitting**. A polynomial of degree 4 +approximates the true function almost perfectly. However, for higher degrees +the model will **overfit** the training data, i.e. it learns the noise of the +training data. +We evaluate quantitatively **overfitting** / **underfitting** by using +cross-validation. We calculate the mean squared error (MSE) on the validation +set, the higher, the less likely the model generalizes correctly from the +training data. +""" + +print(__doc__) + +import numpy as np +import matplotlib.pyplot as plt +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import PolynomialFeatures +from sklearn.linear_model import LinearRegression +from sklearn.model_selection import cross_val_score + + +def true_fun(X): + return np.cos(1.5 * np.pi * X) + +np.random.seed(0) + +n_samples = 30 +degrees = [1, 4, 15] + +X = np.sort(np.random.rand(n_samples)) +y = true_fun(X) + np.random.randn(n_samples) * 0.1 + +plt.figure(figsize=(14, 5)) +for i in range(len(degrees)): + ax = plt.subplot(1, len(degrees), i + 1) + plt.setp(ax, xticks=(), yticks=()) + + polynomial_features = PolynomialFeatures(degree=degrees[i], + include_bias=False) + linear_regression = LinearRegression() + pipeline = Pipeline([("polynomial_features", polynomial_features), + ("linear_regression", linear_regression)]) + pipeline.fit(X[:, np.newaxis], y) + + # Evaluate the models using crossvalidation + scores = cross_val_score(pipeline, X[:, np.newaxis], y, + scoring="neg_mean_squared_error", cv=10) + + X_test = np.linspace(0, 1, 100) + plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model") + plt.plot(X_test, true_fun(X_test), label="True function") + plt.scatter(X, y, edgecolor='b', s=20, label="Samples") + plt.xlabel("x") + plt.ylabel("y") + plt.xlim((0, 1)) + plt.ylim((-2, 2)) + plt.legend(loc="best") + plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format( + degrees[i], -scores.mean(), scores.std())) +plt.show() +!ec + + + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.model_selection import train_test_split +from sklearn.utils import resample +from sklearn.metrics import mean_squared_error +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("EoS.csv"),'r') + +# Read the EoS data as csv file and organize the data into two arrays with density and energies +EoS = pd.read_csv(infile, names=('Density', 'Energy')) +EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce') +EoS = EoS.dropna() +Energies = EoS['Energy'] +Density = EoS['Density'] +# The design matrix now as function of various polytrops + +Maxpolydegree = 30 +X = np.zeros((len(Density),Maxpolydegree)) +X[:,0] = 1.0 +testerror = np.zeros(Maxpolydegree) +trainingerror = np.zeros(Maxpolydegree) +polynomial = np.zeros(Maxpolydegree) + +trials = 100 +for polydegree in range(1, Maxpolydegree): + polynomial[polydegree] = polydegree + for degree in range(polydegree): + X[:,degree] = Density**(degree/3.0) + +# loop over trials in order to estimate the expectation value of the MSE + testerror[polydegree] = 0.0 + trainingerror[polydegree] = 0.0 + for samples in range(trials): + x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2) + model = LinearRegression(fit_intercept=True).fit(x_train, y_train) + ypred = model.predict(x_train) + ytilde = model.predict(x_test) + testerror[polydegree] += mean_squared_error(y_test, ytilde) + trainingerror[polydegree] += mean_squared_error(y_train, ypred) + + testerror[polydegree] /= trials + trainingerror[polydegree] /= trials + print("Degree of polynomial: %3d"% polynomial[polydegree]) + print("Mean squared error on training data: %.8f" % trainingerror[polydegree]) + print("Mean squared error on test data: %.8f" % testerror[polydegree]) + +plt.plot(polynomial, np.log10(trainingerror), label='Training Error') +plt.plot(polynomial, np.log10(testerror), label='Test Error') +plt.xlabel('Polynomial degree') +plt.ylabel('log10[MSE]') +plt.legend() +plt.show() + +!ec + + + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.metrics import mean_squared_error +from sklearn.model_selection import KFold +from sklearn.model_selection import cross_val_score + + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("EoS.csv"),'r') + +# Read the EoS data as csv file and organize the data into two arrays with density and energies +EoS = pd.read_csv(infile, names=('Density', 'Energy')) +EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce') +EoS = EoS.dropna() +Energies = EoS['Energy'] +Density = EoS['Density'] +# The design matrix now as function of various polytrops + +Maxpolydegree = 30 +X = np.zeros((len(Density),Maxpolydegree)) +X[:,0] = 1.0 +estimated_mse_sklearn = np.zeros(Maxpolydegree) +polynomial = np.zeros(Maxpolydegree) +k =5 +kfold = KFold(n_splits = k) + +for polydegree in range(1, Maxpolydegree): + polynomial[polydegree] = polydegree + for degree in range(polydegree): + X[:,degree] = Density**(degree/3.0) + OLS = LinearRegression() +# loop over trials in order to estimate the expectation value of the MSE + estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold) +#[:, np.newaxis] + estimated_mse_sklearn[polydegree] = np.mean(-estimated_mse_folds) + +plt.plot(polynomial, np.log10(estimated_mse_sklearn), label='Test Error') +plt.xlabel('Polynomial degree') +plt.ylabel('log10[MSE]') +plt.legend() +plt.show() + +!ec + + +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import KFold +from sklearn.linear_model import Ridge +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import PolynomialFeatures + +# A seed just to ensure that the random numbers are the same for every run. +np.random.seed(3155) +# Generate the data. +n = 100 +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +# Decide degree on polynomial to fit +poly = PolynomialFeatures(degree = 10) + +# Decide which values of lambda to use +nlambdas = 500 +lambdas = np.logspace(-3, 5, nlambdas) +# Initialize a KFold instance +k = 5 +kfold = KFold(n_splits = k) +estimated_mse_sklearn = np.zeros(nlambdas) +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold) + estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds) + i += 1 +plt.figure() +plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score') +plt.xlabel('log10(lambda)') +plt.ylabel('MSE') +plt.legend() +plt.show() + + +!ec + + diff --git a/doc/BookChapters/chapter2.do.txt~ b/doc/BookChapters/chapter2.do.txt~ new file mode 100644 index 000000000..be6831a85 --- /dev/null +++ b/doc/BookChapters/chapter2.do.txt~ @@ -0,0 +1,1186 @@ +TITLE: Week 36: Resampling techniques and Ordinary Least Square +AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo & Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University +DATE: today + + +!split +===== Plans for week 36 ===== + +* Thursday: Statistics, probability theory and resampling methods +* Friday: Resampling methods and motivation for Ridge Regression + + +!split +===== Thursday September 3 ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSept3.mp4?vrtx=view-as-webpage" and "handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember3.pdf" + +More material will be added here, see handwritten notes also. + + +!split +===== Why resampling methods ===== + +Before we proceed, we need to rethink what we have been doing. In our +eager to fit the data, we have omitted several important elements in +our regression analysis. In what follows we will +o look at statistical properties, including a discussion of mean values, variance and the so-called bias-variance tradeoff +o introduce resampling techniques like cross-validation, bootstrapping and jackknife and more + +This will allow us to link the standard linear algebra methods we have discussed above to a statistical interpretation of the methods. + + + + + +!split +===== Resampling methods ===== +!bblock +Resampling methods are an indispensable tool in modern +statistics. They involve repeatedly drawing samples from a training +set and refitting a model of interest on each sample in order to +obtain additional information about the fitted model. For example, in +order to estimate the variability of a linear regression fit, we can +repeatedly draw different samples from the training data, fit a linear +regression to each new sample, and then examine the extent to which +the resulting fits differ. Such an approach may allow us to obtain +information that would not be available from fitting the model only +once using the original training sample. + +Two resampling methods are often used in Machine Learning analyses, +o The _bootstrap method_ +o and _Cross-Validation_ + +In addition there are several other methods such as the Jackknife and the Blocking methods. We will discuss in particular +cross-validation and the bootstrap method. + + +!eblock + + +!split +===== Resampling approaches can be computationally expensive ===== +!bblock + +Resampling approaches can be computationally expensive, because they +involve fitting the same statistical method multiple times using +different subsets of the training data. However, due to recent +advances in computing power, the computational requirements of +resampling methods generally are not prohibitive. In this chapter, we +discuss two of the most commonly used resampling methods, +cross-validation and the bootstrap. Both methods are important tools +in the practical application of many statistical learning +procedures. For example, cross-validation can be used to estimate the +test error associated with a given statistical learning method in +order to evaluate its performance, or to select the appropriate level +of flexibility. The process of evaluating a model’s performance is +known as model assessment, whereas the process of selecting the proper +level of flexibility for a model is known as model selection. The +bootstrap is widely used. + +!eblock + +!split +===== Why resampling methods ? ===== +!bblock Statistical analysis + +* Our simulations can be treated as *computer experiments*. This is particularly the case for Monte Carlo methods +* The results can be analysed with the same statistical tools as we would use analysing experimental data. +* As in all experiments, we are looking for expectation values and an estimate of how accurate they are, i.e., possible sources for errors. + + +!eblock + +!split +===== Statistical analysis ===== +!bblock + +* As in other experiments, many numerical experiments have two classes of errors: + * Statistical errors + * Systematical errors +* Statistical errors can be estimated using standard tools from statistics +* Systematical errors are method specific and must be treated differently from case to case. +!eblock + + + + +!split +===== Linking the regression analysis with a statistical interpretation ===== + + +The +advantage of doing linear regression is that we actually end up with +analytical expressions for several statistical quantities. +Standard least squares and Ridge regression allow us to +derive quantities like the variance and other expectation values in a +rather straightforward way. + + +It is assumed that $\varepsilon_i +\sim \mathcal{N}(0, \sigma^2)$ and the $\varepsilon_{i}$ are +independent, i.e.: +!bt +\begin{align*} +\mbox{Cov}(\varepsilon_{i_1}, +\varepsilon_{i_2}) & = \left\{ \begin{array}{lcc} \sigma^2 & \mbox{if} +& i_1 = i_2, \\ 0 & \mbox{if} & i_1 \not= i_2. \end{array} \right. +\end{align*} +!et +The randomness of $\varepsilon_i$ implies that +$\mathbf{y}_i$ is also a random variable. In particular, +$\mathbf{y}_i$ is normally distributed, because $\varepsilon_i \sim +\mathcal{N}(0, \sigma^2)$ and $\mathbf{X}_{i,\ast} \, \bm{\beta}$ is a +non-random scalar. To specify the parameters of the distribution of +$\mathbf{y}_i$ we need to calculate its first two moments. + +Recall that $\bm{X}$ is a matrix of dimensionality $n\times p$. The +notation above $\mathbf{X}_{i,\ast}$ means that we are looking at the +row number $i$ and perform a sum over all values $p$. + + +!split +===== Assumptions made ===== + +The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off) +that there exists a function $f(\bm{x})$ and a normal distributed error $\bm{\varepsilon}\sim \mathcal{N}(0, \sigma^2)$ +which describe our data +!bt +\[ +\bm{y} = f(\bm{x})+\bm{\varepsilon} +\] +!et + +We approximate this function with our model from the solution of the linear regression equations, that is our +function $f$ is approximated by $\bm{\tilde{y}}$ where we want to minimize $(\bm{y}-\bm{\tilde{y}})^2$, our MSE, with +!bt +\[ +\bm{\tilde{y}} = \bm{X}\bm{\beta}. +\] +!et + +!split +===== Expectation value and variance ===== + +We can calculate the expectation value of $\bm{y}$ for a given element $i$ +!bt +\begin{align*} +\mathbb{E}(y_i) & = +\mathbb{E}(\mathbf{X}_{i, \ast} \, \bm{\beta}) + \mathbb{E}(\varepsilon_i) +\, \, \, = \, \, \, \mathbf{X}_{i, \ast} \, \beta, +\end{align*} +!et +while +its variance is +!bt +\begin{align*} \mbox{Var}(y_i) & = \mathbb{E} \{ [y_i +- \mathbb{E}(y_i)]^2 \} \, \, \, = \, \, \, \mathbb{E} ( y_i^2 ) - +[\mathbb{E}(y_i)]^2 \\ & = \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, +\beta + \varepsilon_i )^2] - ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 \\ & += \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 + 2 \varepsilon_i +\mathbf{X}_{i, \ast} \, \bm{\beta} + \varepsilon_i^2 ] - ( \mathbf{X}_{i, +\ast} \, \beta)^2 \\ & = ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 + 2 +\mathbb{E}(\varepsilon_i) \mathbf{X}_{i, \ast} \, \bm{\beta} + +\mathbb{E}(\varepsilon_i^2 ) - ( \mathbf{X}_{i, \ast} \, \bm{\beta})^2 +\\ & = \mathbb{E}(\varepsilon_i^2 ) \, \, \, = \, \, \, +\mbox{Var}(\varepsilon_i) \, \, \, = \, \, \, \sigma^2. +\end{align*} +!et +Hence, $y_i \sim \mathcal{N}( \mathbf{X}_{i, \ast} \, \bm{\beta}, \sigma^2)$, that is $\bm{y}$ follows a normal distribution with +mean value $\bm{X}\bm{\beta}$ and variance $\sigma^2$ (not be confused with the singular values of the SVD). + +!split +===== Expectation value and variance for $\bm{\beta}$ ===== + +With the OLS expressions for the parameters $\bm{\beta}$ we can evaluate the expectation value +!bt +\[ +\mathbb{E}(\bm{\beta}) = \mathbb{E}[ (\mathbf{X}^{\top} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbb{E}[ \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1} \mathbf{X}^{T}\mathbf{X}\bm{\beta}=\bm{\beta}. +\] +!et +This means that the estimator of the regression parameters is unbiased. + +We can also calculate the variance + +The variance of $\bm{\beta}$ is +!bt +\begin{eqnarray*} +\mbox{Var}(\bm{\beta}) & = & \mathbb{E} \{ [\bm{\beta} - \mathbb{E}(\bm{\beta})] [\bm{\beta} - \mathbb{E}(\bm{\beta})]^{T} \} +\\ +& = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \bm{\beta}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \bm{\beta}]^{T} \} +\\ +% & = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}]^{T} \} - \bm{\beta} \, \bm{\beta}^{T} +% \\ +% & = & \mathbb{E} \{ (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} \, \mathbf{Y}^{T} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} \} - \bm{\beta} \, \bm{\beta}^{T} +% \\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \mathbb{E} \{ \mathbf{Y} \, \mathbf{Y}^{T} \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \bm{\beta} \, \bm{\beta}^{T} +\\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \{ \mathbf{X} \, \bm{\beta} \, \bm{\beta}^{T} \, \mathbf{X}^{T} + \sigma^2 \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \bm{\beta} \, \bm{\beta}^{T} +% \\ +% & = & (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, \bm{\beta} \, \bm{\beta}^T \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T % \mathbf{X})^{-1} +% \\ +% & & + \, \, \sigma^2 \, (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T \mathbf{X})^{-1} - \bm{\beta} \bm{\beta}^T +\\ +& = & \bm{\beta} \, \bm{\beta}^{T} + \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \bm{\beta} \, \bm{\beta}^{T} +\, \, \, = \, \, \, \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}, +\end{eqnarray*} +!et + +where we have used that $\mathbb{E} (\mathbf{Y} \mathbf{Y}^{T}) = +\mathbf{X} \, \bm{\beta} \, \bm{\beta}^{T} \, \mathbf{X}^{T} + +\sigma^2 \, \mathbf{I}_{nn}$. From $\mbox{Var}(\bm{\beta}) = \sigma^2 +\, (\mathbf{X}^{T} \mathbf{X})^{-1}$, one obtains an estimate of the +variance of the estimate of the $j$-th regression coefficient: +$\bm{\sigma}^2 (\bm{\beta}_j ) = \bm{\sigma}^2 \sqrt{ +[(\mathbf{X}^{T} \mathbf{X})^{-1}]_{jj} }$. This may be used to +construct a confidence interval for the estimates. + + +In a similar way, we can obtain analytical expressions for say the +expectation values of the parameters $\bm{\beta}$ and their variance +when we employ Ridge regression, allowing us again to define a confidence interval. + +It is rather straightforward to show that +!bt +\[ +\mathbb{E} \big[ \bm{\beta}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\bm{\beta}^{\mathrm{OLS}}. +\] +!et +We see clearly that +$\mathbb{E} \big[ \bm{\beta}^{\mathrm{Ridge}} \big] \not= \bm{\beta}^{\mathrm{OLS}}$ for any $\lambda > 0$. We say then that the ridge estimator is biased. + +We can also compute the variance as + +!bt +\[ +\mbox{Var}[\bm{\beta}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T} \mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, +\] +!et +and it is easy to see that if the parameter $\lambda$ goes to infinity then the variance of Ridge parameters $\bm{\beta}$ goes to zero. + +With this, we can compute the difference + +!bt +\[ +\mbox{Var}[\bm{\beta}^{\mathrm{OLS}}]-\mbox{Var}(\bm{\beta}^{\mathrm{Ridge}})=\sigma^2 [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}[ 2\lambda\mathbf{I} + \lambda^2 (\mathbf{X}^{T} \mathbf{X})^{-1} ] \{ [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}. +\] +!et +The difference is non-negative definite since each component of the +matrix product is non-negative definite. +This means the variance we obtain with the standard OLS will always for $\lambda > 0$ be larger than the variance of $\bm{\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below. + + +!split +===== Resampling methods ===== + +With all these analytical equations for both the OLS and Ridge +regression, we will now outline how to assess a given model. This will +lead us to a discussion of the so-called bias-variance tradeoff (see +below) and so-called resampling methods. + +One of the quantities we have discussed as a way to measure errors is +the mean-squared error (MSE), mainly used for fitting of continuous +functions. Another choice is the absolute error. + +In the discussions below we will focus on the MSE and in particular since we will split the data into test and training data, +we discuss the +o prediction error or simply the _test error_ $\mathrm{Err_{Test}}$, where we have a fixed training set and the test error is the MSE arising from the data reserved for testing. We discuss also the +o training error $\mathrm{Err_{Train}}$, which is the average loss over the training data. + +As our model becomes more and more complex, more of the training data tends to used. The training may thence adapt to more complicated structures in the data. This may lead to a decrease in the bias (see below for code example) and a slight increase of the variance for the test error. +For a certain level of complexity the test error will reach minimum, before starting to increase again. The +training error reaches a saturation. + + + + +!split +===== Resampling methods: Jackknife and Bootstrap ===== + +Two famous +resampling methods are the _independent bootstrap_ and _the jackknife_. + +The jackknife is a special case of the independent bootstrap. Still, the jackknife was made +popular prior to the independent bootstrap. And as the popularity of +the independent bootstrap soared, new variants, such as _the dependent bootstrap_. + +The Jackknife and independent bootstrap work for +independent, identically distributed random variables. +If these conditions are not +satisfied, the methods will fail. Yet, it should be said that if the data are +independent, identically distributed, and we only want to estimate the +variance of $\overline{X}$ (which often is the case), then there is no +need for bootstrapping. + +!split +===== Resampling methods: Jackknife ===== + +The Jackknife works by making many replicas of the estimator $\widehat{\theta}$. +The jackknife is a resampling method where we systematically leave out one observation from the vector of observed values $\bm{x} = (x_1,x_2,\cdots,X_n)$. +Let $\bm{x}_i$ denote the vector +!bt +\[ +\bm{x}_i = (x_1,x_2,\cdots,x_{i-1},x_{i+1},\cdots,x_n), +\] +!et + +which equals the vector $\bm{x}$ with the exception that observation +number $i$ is left out. Using this notation, define +$\widehat{\theta}_i$ to be the estimator +$\widehat{\theta}$ computed using $\vec{X}_i$. + + +!split +===== Jackknife code example ===== +!bc pycod +from numpy import * +from numpy.random import randint, randn +from time import time + +def jackknife(data, stat): + n = len(data);t = zeros(n); inds = arange(n); t0 = time() + ## 'jackknifing' by leaving out an observation for each i + for i in range(n): + t[i] = stat(delete(data,i) ) + + # analysis + print("Runtime: %g sec" % (time()-t0)); print("Jackknife Statistics :") + print("original bias std. error") + print("%8g %14g %15g" % (stat(data),(n-1)*mean(t)/n, (n*var(t))**.5)) + + return t + + +# Returns mean of data samples +def stat(data): + return mean(data) + + +mu, sigma = 100, 15 +datapoints = 10000 +x = mu + sigma*random.randn(datapoints) +# jackknife returns the data sample +t = jackknife(x, stat) + +!ec + + +!split +===== Resampling methods: Bootstrap ===== +!bblock +Bootstrapping is a nonparametric approach to statistical inference +that substitutes computation for more traditional distributional +assumptions and asymptotic results. Bootstrapping offers a number of +advantages: +o The bootstrap is quite general, although there are some cases in which it fails. +o Because it does not require distributional assumptions (such as normally distributed errors), the bootstrap can provide more accurate inferences when the data are not well behaved or when the sample size is small. +o It is possible to apply the bootstrap to statistics with sampling distributions that are difficult to derive, even asymptotically. +o It is relatively simple to apply the bootstrap to complex data-collection plans (such as stratified and clustered samples). +!eblock + + +!split +===== Resampling methods: Bootstrap background ===== + +Since $\widehat{\theta} = \widehat{\theta}(\bm{X})$ is a function of random variables, +$\widehat{\theta}$ itself must be a random variable. Thus it has +a pdf, call this function $p(\bm{t})$. The aim of the bootstrap is to +estimate $p(\bm{t})$ by the relative frequency of +$\widehat{\theta}$. You can think of this as using a histogram +in the place of $p(\bm{t})$. If the relative frequency closely +resembles $p(\vec{t})$, then using numerics, it is straight forward to +estimate all the interesting parameters of $p(\bm{t})$ using point +estimators. + + +!split +===== Resampling methods: More Bootstrap background ===== + +In the case that $\widehat{\theta}$ has +more than one component, and the components are independent, we use the +same estimator on each component separately. If the probability +density function of $X_i$, $p(x)$, had been known, then it would have +been straight forward to do this by: +o Drawing lots of numbers from $p(x)$, suppose we call one such set of numbers $(X_1^*, X_2^*, \cdots, X_n^*)$. +o Then using these numbers, we could compute a replica of $\widehat{\theta}$ called $\widehat{\theta}^*$. + +By repeated use of (1) and (2), many +estimates of $\widehat{\theta}$ could have been obtained. The +idea is to use the relative frequency of $\widehat{\theta}^*$ +(think of a histogram) as an estimate of $p(\bm{t})$. + +!split +===== Resampling methods: Bootstrap approach ===== + +But +unless there is enough information available about the process that +generated $X_1,X_2,\cdots,X_n$, $p(x)$ is in general +unknown. Therefore, "Efron in 1979":"https://projecteuclid.org/euclid.aos/1176344552" asked the +question: What if we replace $p(x)$ by the relative frequency +of the observation $X_i$; if we draw observations in accordance with +the relative frequency of the observations, will we obtain the same +result in some asymptotic sense? The answer is yes. + + +Instead of generating the histogram for the relative +frequency of the observation $X_i$, just draw the values +$(X_1^*,X_2^*,\cdots,X_n^*)$ with replacement from the vector +$\bm{X}$. + +!split +===== Resampling methods: Bootstrap steps ===== + +The independent bootstrap works like this: + +o Draw with replacement $n$ numbers for the observed variables $\bm{x} = (x_1,x_2,\cdots,x_n)$. +o Define a vector $\bm{x}^*$ containing the values which were drawn from $\bm{x}$. +o Using the vector $\bm{x}^*$ compute $\widehat{\theta}^*$ by evaluating $\widehat \theta$ under the observations $\bm{x}^*$. +o Repeat this process $k$ times. + +When you are done, you can draw a histogram of the relative frequency +of $\widehat \theta^*$. This is your estimate of the probability +distribution $p(t)$. Using this probability distribution you can +estimate any statistics thereof. In principle you never draw the +histogram of the relative frequency of $\widehat{\theta}^*$. Instead +you use the estimators corresponding to the statistic of interest. For +example, if you are interested in estimating the variance of $\widehat +\theta$, apply the etsimator $\widehat \sigma^2$ to the values +$\widehat \theta ^*$. + + +!split +===== Code example for the Bootstrap method ===== + +The following code starts with a Gaussian distribution with mean value +$\mu =100$ and variance $\sigma=15$. We use this to generate the data +used in the bootstrap analysis. The bootstrap analysis returns a data +set after a given number of bootstrap operations (as many as we have +data points). This data set consists of estimated mean values for each +bootstrap operation. The histogram generated by the bootstrap method +shows that the distribution for these mean values is also a Gaussian, +centered around the mean value $\mu=100$ but with standard deviation +$\sigma/\sqrt{n}$, where $n$ is the number of bootstrap samples (in +this case the same as the number of original data points). The value +of the standard deviation is what we expect from the central limit +theorem. + + +!bc pycod +from numpy import * +from numpy.random import randint, randn +from time import time +import matplotlib.mlab as mlab +import matplotlib.pyplot as plt + +# Returns mean of bootstrap samples +def stat(data): + return mean(data) + +# Bootstrap algorithm +def bootstrap(data, statistic, R): + t = zeros(R); n = len(data); inds = arange(n); t0 = time() + # non-parametric bootstrap + for i in range(R): + t[i] = statistic(data[randint(0,n,n)]) + + # analysis + print("Runtime: %g sec" % (time()-t0)); print("Bootstrap Statistics :") + print("original bias std. error") + print("%8g %8g %14g %15g" % (statistic(data), std(data),mean(t),std(t))) + return t + + +mu, sigma = 100, 15 +datapoints = 10000 +x = mu + sigma*random.randn(datapoints) +# bootstrap returns the data sample +t = bootstrap(x, stat, datapoints) +# the histogram of the bootstrapped data +n, binsboot, patches = plt.hist(t, 50, normed=1, facecolor='red', alpha=0.75) + +# add a 'best fit' line +y = mlab.normpdf( binsboot, mean(t), std(t)) +lt = plt.plot(binsboot, y, 'r--', linewidth=1) +plt.xlabel('Smarts') +plt.ylabel('Probability') +plt.axis([99.5, 100.6, 0, 3.0]) +plt.grid(True) + +plt.show() + +!ec + + +!split +===== Various steps in cross-validation ===== + +When the repetitive splitting of the data set is done randomly, +samples may accidently end up in a fast majority of the splits in +either training or test set. Such samples may have an unbalanced +influence on either model building or prediction evaluation. To avoid +this $k$-fold cross-validation structures the data splitting. The +samples are divided into $k$ more or less equally sized exhaustive and +mutually exclusive subsets. In turn (at each split) one of these +subsets plays the role of the test set while the union of the +remaining subsets constitutes the training set. Such a splitting +warrants a balanced representation of each sample in both training and +test set over the splits. Still the division into the $k$ subsets +involves a degree of randomness. This may be fully excluded when +choosing $k=n$. This particular case is referred to as leave-one-out +cross-validation (LOOCV). + +!split +===== How to set up the cross-validation for Ridge and/or Lasso ===== + +* Define a range of interest for the penalty parameter. + +* Divide the data set into training and test set comprising samples $\{1, \ldots, n\} \setminus i$ and $\{ i \}$, respectively. + +* Fit the linear regression model by means of ridge estimation for each $\lambda$ in the grid using the training set, and the corresponding estimate of the error variance $\bm{\sigma}_{-i}^2(\lambda)$, as +!bt +\begin{align*} +\bm{\beta}_{-i}(\lambda) & = ( \bm{X}_{-i, \ast}^{T} +\bm{X}_{-i, \ast} + \lambda \bm{I}_{pp})^{-1} +\bm{X}_{-i, \ast}^{T} \bm{y}_{-i} +\end{align*} +!et + +* Evaluate the prediction performance of these models on the test set by $\log\{L[y_i, \bm{X}_{i, \ast}; \bm{\beta}_{-i}(\lambda), \bm{\sigma}_{-i}^2(\lambda)]\}$. Or, by the prediction error $|y_i - \bm{X}_{i, \ast} \bm{\beta}_{-i}(\lambda)|$, the relative error, the error squared or the R2 score function. + +* Repeat the first three steps such that each sample plays the role of the test set once. + +* Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as +!bt +\begin{align*} +\frac{1}{n} \sum_{i = 1}^n \log\{L[y_i, \mathbf{X}_{i, \ast}; \bm{\beta}_{-i}(\lambda), \bm{\sigma}_{-i}^2(\lambda)]\}. +\end{align*} +!et + +!split +===== Cross-validation in brief ===== + +For the various values of $k$ + +o shuffle the dataset randomly. +o Split the dataset into $k$ groups. +o For each unique group: + o Decide which group to use as set for test data + o Take the remaining groups as a training data set + o Fit a model on the training set and evaluate it on the test set + o Retain the evaluation score and discard the model +o Summarize the model using the sample of model evaluation scores + + + +!split +===== Code Example for Cross-validation and $k$-fold Cross-validation ===== + +The code here uses Ridge regression with cross-validation (CV) resampling and $k$-fold CV in order to fit a specific polynomial. +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import KFold +from sklearn.linear_model import Ridge +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import PolynomialFeatures + +# A seed just to ensure that the random numbers are the same for every run. +# Useful for eventual debugging. +np.random.seed(3155) + +# Generate the data. +nsamples = 100 +x = np.random.randn(nsamples) +y = 3*x**2 + np.random.randn(nsamples) + +## Cross-validation on Ridge regression using KFold only + +# Decide degree on polynomial to fit +poly = PolynomialFeatures(degree = 6) + +# Decide which values of lambda to use +nlambdas = 500 +lambdas = np.logspace(-3, 5, nlambdas) + +# Initialize a KFold instance +k = 5 +kfold = KFold(n_splits = k) + +# Perform the cross-validation to estimate MSE +scores_KFold = np.zeros((nlambdas, k)) + +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + j = 0 + for train_inds, test_inds in kfold.split(x): + xtrain = x[train_inds] + ytrain = y[train_inds] + + xtest = x[test_inds] + ytest = y[test_inds] + + Xtrain = poly.fit_transform(xtrain[:, np.newaxis]) + ridge.fit(Xtrain, ytrain[:, np.newaxis]) + + Xtest = poly.fit_transform(xtest[:, np.newaxis]) + ypred = ridge.predict(Xtest) + + scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred) + + j += 1 + i += 1 + + +estimated_mse_KFold = np.mean(scores_KFold, axis = 1) + +## Cross-validation using cross_val_score from sklearn along with KFold + +# kfold is an instance initialized above as: +# kfold = KFold(n_splits = k) + +estimated_mse_sklearn = np.zeros(nlambdas) +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + + X = poly.fit_transform(x[:, np.newaxis]) + estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold) + + # cross_val_score return an array containing the estimated negative mse for every fold. + # we have to the the mean of every array in order to get an estimate of the mse of the model + estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds) + + i += 1 + +## Plot and compare the slightly different ways to perform cross-validation + +plt.figure() + +plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score') +plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold') + +plt.xlabel('log10(lambda)') +plt.ylabel('mse') + +plt.legend() + +plt.show() + +!ec + + +!split +===== The bias-variance tradeoff ===== + + +We will discuss the bias-variance tradeoff in the context of +continuous predictions such as regression. However, many of the +intuitions and ideas discussed here also carry over to classification +tasks. Consider a dataset $\mathcal{L}$ consisting of the data +$\mathbf{X}_\mathcal{L}=\{(y_j, \boldsymbol{x}_j), j=0\ldots n-1\}$. + +Let us assume that the true data is generated from a noisy model + +!bt +\[ +\bm{y}=f(\boldsymbol{x}) + \bm{\epsilon} +\] +!et + +where $\epsilon$ is normally distributed with mean zero and standard deviation $\sigma^2$. + +In our derivation of the ordinary least squares method we defined then +an approximation to the function $f$ in terms of the parameters +$\bm{\beta}$ and the design matrix $\bm{X}$ which embody our model, +that is $\bm{\tilde{y}}=\bm{X}\bm{\beta}$. + +Thereafter we found the parameters $\bm{\beta}$ by optimizing the means squared error via the so-called cost function +!bt +\[ +C(\bm{X},\bm{\beta}) =\frac{1}{n}\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2=\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]. +\] +!et + +We can rewrite this as +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\frac{1}{n}\sum_i(f_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2+\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\bm{\tilde{y}}\right])^2+\sigma^2. +\] +!et + +The three terms represent the square of the bias of the learning +method, which can be thought of as the error caused by the simplifying +assumptions built into the method. The second term represents the +variance of the chosen model and finally the last terms is variance of +the error $\bm{\epsilon}$. + +To derive this equation, we need to recall that the variance of $\bm{y}$ and $\bm{\epsilon}$ are both equal to $\sigma^2$. The mean value of $\bm{\epsilon}$ is by definition equal to zero. Furthermore, the function $f$ is not a stochastics variable, idem for $\bm{\tilde{y}}$. +We use a more compact notation in terms of the expectation value +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathbb{E}\left[(\bm{f}+\bm{\epsilon}-\bm{\tilde{y}})^2\right], +\] +!et +and adding and subtracting $\mathbb{E}\left[\bm{\tilde{y}}\right]$ we get +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathbb{E}\left[(\bm{f}+\bm{\epsilon}-\bm{\tilde{y}}+\mathbb{E}\left[\bm{\tilde{y}}\right]-\mathbb{E}\left[\bm{\tilde{y}}\right])^2\right], +\] +!et +which, using the abovementioned expectation values can be rewritten as +!bt +\[ +\mathbb{E}\left[(\bm{y}-\bm{\tilde{y}})^2\right]=\mathbb{E}\left[(\bm{y}-\mathbb{E}\left[\bm{\tilde{y}}\right])^2\right]+\mathrm{Var}\left[\bm{\tilde{y}}\right]+\sigma^2, +\] +!et +that is the rewriting in terms of the so-called bias, the variance of the model $\bm{\tilde{y}}$ and the variance of $\bm{\epsilon}$. + + + + + +!split +===== Example code for Bias-Variance tradeoff ===== +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.preprocessing import PolynomialFeatures +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample + +np.random.seed(2018) + +n = 500 +n_boostraps = 100 +degree = 18 # A quite high value, just to show. +noise = 0.1 + +# Make data set. +x = np.linspace(-1, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1, x.shape) + +# Hold out some test data that is never used in training. +x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +# Combine x transformation and model into one operation. +# Not neccesary, but convenient. +model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False)) + +# The following (m x n_bootstraps) matrix holds the column vectors y_pred +# for each bootstrap iteration. +y_pred = np.empty((y_test.shape[0], n_boostraps)) +for i in range(n_boostraps): + x_, y_ = resample(x_train, y_train) + + # Evaluate the new model on the same test data each time. + y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel() + +# Note: Expectations and variances taken w.r.t. different training +# data sets, hence the axis=1. Subsequent means are taken across the test data +# set in order to obtain a total value, but before this we have error/bias/variance +# calculated per data point in the test set. +# Note 2: The use of keepdims=True is important in the calculation of bias as this +# maintains the column vector form. Dropping this yields very unexpected results. +error = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) +bias = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) +variance = np.mean( np.var(y_pred, axis=1, keepdims=True) ) +print('Error:', error) +print('Bias^2:', bias) +print('Var:', variance) +print('{} >= {} + {} = {}'.format(error, bias, variance, bias+variance)) + +plt.plot(x[::5, :], y[::5, :], label='f(x)') +plt.scatter(x_test, y_test, label='Data points') +plt.scatter(x_test, np.mean(y_pred, axis=1), label='Pred') +plt.legend() +plt.show() + +!ec + + +!split +===== Understanding what happens ===== +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.preprocessing import PolynomialFeatures +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample + +np.random.seed(2018) + +n = 40 +n_boostraps = 100 +maxdegree = 14 + + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +for degree in range(maxdegree): + model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False)) + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(x_train, y_train) + y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel() + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + + +!ec + +!split +===== Summing up ===== + + + + +The bias-variance tradeoff summarizes the fundamental tension in +machine learning, particularly supervised learning, between the +complexity of a model and the amount of training data needed to train +it. Since data is often limited, in practice it is often useful to +use a less-complex model with higher bias, that is a model whose asymptotic +performance is worse than another model because it is easier to +train and less sensitive to sampling noise arising from having a +finite-sized training dataset (smaller variance). + + + +The above equations tell us that in +order to minimize the expected test error, we need to select a +statistical learning method that simultaneously achieves low variance +and low bias. Note that variance is inherently a nonnegative quantity, +and squared bias is also nonnegative. Hence, we see that the expected +test MSE can never lie below $Var(\epsilon)$, the irreducible error. + + +What do we mean by the variance and bias of a statistical learning +method? The variance refers to the amount by which our model would change if we +estimated it using a different training data set. Since the training +data are used to fit the statistical learning method, different +training data sets will result in a different estimate. But ideally the +estimate for our model should not vary too much between training +sets. However, if a method has high variance then small changes in +the training data can result in large changes in the model. In general, more +flexible statistical methods have higher variance. + + +You may also find this recent "article":"https://www.pnas.org/content/116/32/15849" of interest. + +!split +===== Another Example from Scikit-Learn's Repository ===== +!bc pycod +""" +============================ +Underfitting vs. Overfitting +============================ + +This example demonstrates the problems of underfitting and overfitting and +how we can use linear regression with polynomial features to approximate +nonlinear functions. The plot shows the function that we want to approximate, +which is a part of the cosine function. In addition, the samples from the +real function and the approximations of different models are displayed. The +models have polynomial features of different degrees. We can see that a +linear function (polynomial with degree 1) is not sufficient to fit the +training samples. This is called **underfitting**. A polynomial of degree 4 +approximates the true function almost perfectly. However, for higher degrees +the model will **overfit** the training data, i.e. it learns the noise of the +training data. +We evaluate quantitatively **overfitting** / **underfitting** by using +cross-validation. We calculate the mean squared error (MSE) on the validation +set, the higher, the less likely the model generalizes correctly from the +training data. +""" + +print(__doc__) + +import numpy as np +import matplotlib.pyplot as plt +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import PolynomialFeatures +from sklearn.linear_model import LinearRegression +from sklearn.model_selection import cross_val_score + + +def true_fun(X): + return np.cos(1.5 * np.pi * X) + +np.random.seed(0) + +n_samples = 30 +degrees = [1, 4, 15] + +X = np.sort(np.random.rand(n_samples)) +y = true_fun(X) + np.random.randn(n_samples) * 0.1 + +plt.figure(figsize=(14, 5)) +for i in range(len(degrees)): + ax = plt.subplot(1, len(degrees), i + 1) + plt.setp(ax, xticks=(), yticks=()) + + polynomial_features = PolynomialFeatures(degree=degrees[i], + include_bias=False) + linear_regression = LinearRegression() + pipeline = Pipeline([("polynomial_features", polynomial_features), + ("linear_regression", linear_regression)]) + pipeline.fit(X[:, np.newaxis], y) + + # Evaluate the models using crossvalidation + scores = cross_val_score(pipeline, X[:, np.newaxis], y, + scoring="neg_mean_squared_error", cv=10) + + X_test = np.linspace(0, 1, 100) + plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model") + plt.plot(X_test, true_fun(X_test), label="True function") + plt.scatter(X, y, edgecolor='b', s=20, label="Samples") + plt.xlabel("x") + plt.ylabel("y") + plt.xlim((0, 1)) + plt.ylim((-2, 2)) + plt.legend(loc="best") + plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format( + degrees[i], -scores.mean(), scores.std())) +plt.show() +!ec + + +!split +===== More examples on bootstrap and cross-validation and errors ===== + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.model_selection import train_test_split +from sklearn.utils import resample +from sklearn.metrics import mean_squared_error +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("EoS.csv"),'r') + +# Read the EoS data as csv file and organize the data into two arrays with density and energies +EoS = pd.read_csv(infile, names=('Density', 'Energy')) +EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce') +EoS = EoS.dropna() +Energies = EoS['Energy'] +Density = EoS['Density'] +# The design matrix now as function of various polytrops + +Maxpolydegree = 30 +X = np.zeros((len(Density),Maxpolydegree)) +X[:,0] = 1.0 +testerror = np.zeros(Maxpolydegree) +trainingerror = np.zeros(Maxpolydegree) +polynomial = np.zeros(Maxpolydegree) + +trials = 100 +for polydegree in range(1, Maxpolydegree): + polynomial[polydegree] = polydegree + for degree in range(polydegree): + X[:,degree] = Density**(degree/3.0) + +# loop over trials in order to estimate the expectation value of the MSE + testerror[polydegree] = 0.0 + trainingerror[polydegree] = 0.0 + for samples in range(trials): + x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2) + model = LinearRegression(fit_intercept=True).fit(x_train, y_train) + ypred = model.predict(x_train) + ytilde = model.predict(x_test) + testerror[polydegree] += mean_squared_error(y_test, ytilde) + trainingerror[polydegree] += mean_squared_error(y_train, ypred) + + testerror[polydegree] /= trials + trainingerror[polydegree] /= trials + print("Degree of polynomial: %3d"% polynomial[polydegree]) + print("Mean squared error on training data: %.8f" % trainingerror[polydegree]) + print("Mean squared error on test data: %.8f" % testerror[polydegree]) + +plt.plot(polynomial, np.log10(trainingerror), label='Training Error') +plt.plot(polynomial, np.log10(testerror), label='Test Error') +plt.xlabel('Polynomial degree') +plt.ylabel('log10[MSE]') +plt.legend() +plt.show() + +!ec + + +!split +===== The same example but now with cross-validation ===== + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.metrics import mean_squared_error +from sklearn.model_selection import KFold +from sklearn.model_selection import cross_val_score + + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("EoS.csv"),'r') + +# Read the EoS data as csv file and organize the data into two arrays with density and energies +EoS = pd.read_csv(infile, names=('Density', 'Energy')) +EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce') +EoS = EoS.dropna() +Energies = EoS['Energy'] +Density = EoS['Density'] +# The design matrix now as function of various polytrops + +Maxpolydegree = 30 +X = np.zeros((len(Density),Maxpolydegree)) +X[:,0] = 1.0 +estimated_mse_sklearn = np.zeros(Maxpolydegree) +polynomial = np.zeros(Maxpolydegree) +k =5 +kfold = KFold(n_splits = k) + +for polydegree in range(1, Maxpolydegree): + polynomial[polydegree] = polydegree + for degree in range(polydegree): + X[:,degree] = Density**(degree/3.0) + OLS = LinearRegression() +# loop over trials in order to estimate the expectation value of the MSE + estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold) +#[:, np.newaxis] + estimated_mse_sklearn[polydegree] = np.mean(-estimated_mse_folds) + +plt.plot(polynomial, np.log10(estimated_mse_sklearn), label='Test Error') +plt.xlabel('Polynomial degree') +plt.ylabel('log10[MSE]') +plt.legend() +plt.show() + +!ec + +!split +===== Cross-validation with Ridge ===== +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import KFold +from sklearn.linear_model import Ridge +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import PolynomialFeatures + +# A seed just to ensure that the random numbers are the same for every run. +np.random.seed(3155) +# Generate the data. +n = 100 +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +# Decide degree on polynomial to fit +poly = PolynomialFeatures(degree = 10) + +# Decide which values of lambda to use +nlambdas = 500 +lambdas = np.logspace(-3, 5, nlambdas) +# Initialize a KFold instance +k = 5 +kfold = KFold(n_splits = k) +estimated_mse_sklearn = np.zeros(nlambdas) +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold) + estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds) + i += 1 +plt.figure() +plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score') +plt.xlabel('log10(lambda)') +plt.ylabel('MSE') +plt.legend() +plt.show() + + +!ec + + +!split +===== Friday September 4 ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember4.mp4?vrtx=view-as-webpage" and "handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember4.pdf" + +More material will be added here, see handwritten notes also. + + + + diff --git a/doc/BookChapters/chapter3.dlog b/doc/BookChapters/chapter3.dlog new file mode 100644 index 000000000..1d754e8e2 --- /dev/null +++ b/doc/BookChapters/chapter3.dlog @@ -0,0 +1,40 @@ +*** error: file has a mako construction ${\bf X}' + but seemingly no definition in <%...%>' + (it is not a command-line given mako variable either). + However, if this is a variable in a Makefile or Bash script + run with --no_mako - and you cannot use mako and Makefile or Bash variables + in the same document! + +Translating doconce text in chapter3.do.txt to ipynb +*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax) + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. + +*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments. +Failed to remove ans_at_end environment +Failed to remove sol_at_end environment +output in chapter3.ipynb diff --git a/doc/BookChapters/chapter3.do.txt b/doc/BookChapters/chapter3.do.txt new file mode 100644 index 000000000..823e50767 --- /dev/null +++ b/doc/BookChapters/chapter3.do.txt @@ -0,0 +1,848 @@ +======= Ridge and Lasso Regression ======= + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember10.mp4?vrtx=view-as-webpage" + + +===== The singular value decomposition ===== + +The examples we have looked at so far are cases where we normally can +invert the matrix $\bm{X}^T\bm{X}$. Using a polynomial expansion as we +did both for the masses and the fitting of the equation of state, +leads to row vectors of the design matrix which are essentially +orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition. + + + +This may +however not the be case in general and a standard matrix inversion +algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below. + +There is however a way to partially circumvent this problem and also gain some insights about the ordinary least squares approach, and later shrinkage methods like Ridge and Lasso regressions. + +This is given by the _Singular Value Decomposition_ algorithm, perhaps +the most powerful linear algebra algorithm. Let us look at a +different example where we may have problems with the standard matrix +inversion algorithm. Thereafter we dive into the math of the SVD. + + + +One of the typical problems we encounter with linear regression, in particular +when the matrix $\bm{X}$ (our so-called design matrix) is high-dimensional, +are problems with near singular or singular matrices. The column vectors of $\bm{X}$ +may be linearly dependent, normally referred to as super-collinearity. +This means that the matrix may be rank deficient and it is basically impossible to +to model the data using linear regression. As an example, consider the matrix +!bt +\begin{align*} +\mathbf{X} & = \left[ +\begin{array}{rrr} +1 & -1 & 2 +\\ +1 & 0 & 1 +\\ +1 & 2 & -1 +\\ +1 & 1 & 0 +\end{array} \right] +\end{align*} +!et + +The columns of $\bm{X}$ are linearly dependent. We see this easily since the +the first column is the row-wise sum of the other two columns. The rank (more correct, +the column rank) of a matrix is the dimension of the space spanned by the +column vectors. Hence, the rank of $\mathbf{X}$ is equal to the number +of linearly independent columns. In this particular case the matrix has rank 2. + +Super-collinearity of an $(n \times p)$-dimensional design matrix $\mathbf{X}$ implies +that the inverse of the matrix $\bm{X}^T\bm{X}$ (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this +!bt +\begin{align*} +\bm{X} & = \left[ +\begin{array}{rr} +1 & -1 +\\ +1 & -1 +\end{array} \right]. +\end{align*} +!et +We see easily that $\mbox{det}(\bm{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0$. Hence, $\mathbf{X}$ is singular and its inverse is undefined. +This is equivalent to saying that the matrix $\bm{X}$ has at least an eigenvalue which is zero. + + +If our design matrix $\bm{X}$ which enters the linear regression problem +!bt +\begin{align} +\bm{\beta} & = (\bm{X}^{T} \bm{X})^{-1} \bm{X}^{T} \bm{y}, +\end{align} +!et +has linearly dependent column vectors, we will not be able to compute the inverse +of $\bm{X}^T\bm{X}$ and we cannot find the parameters (estimators) $\beta_i$. +The estimators are only well-defined if $(\bm{X}^{T}\bm{X})^{-1}$ exits. +This is more likely to happen when the matrix $\bm{X}$ is high-dimensional. In this case it is likely to encounter a situation where +the regression parameters $\beta_i$ cannot be estimated. + +A cheap *ad hoc* approach is simply to add a small diagonal component to the matrix to invert, that is we change +!bt +\[ +\bm{X}^{T} \bm{X} \rightarrow \bm{X}^{T} \bm{X}+\lambda \bm{I}, +\] +!et +where $\bm{I}$ is the identity matrix. When we discuss _Ridge_ regression this is actually what we end up evaluating. The parameter $\lambda$ is called a hyperparameter. More about this later. + + + + + +From standard linear algebra we know that a square matrix $\bm{X}$ can be diagonalized if and only it is +a so-called "normal matrix":"https://en.wikipedia.org/wiki/Normal_matrix", that is if $\bm{X}\in {\mathbb{R}}^{n\times n}$ +we have $\bm{X}\bm{X}^T=\bm{X}^T\bm{X}$ or if $\bm{X}\in {\mathbb{C}}^{n\times n}$ we have $\bm{X}\bm{X}^{\dagger}=\bm{X}^{\dagger}\bm{X}$. +The matrix has then a set of eigenpairs + +!bt +\[ +(\lambda_1,\bm{u}_1),\dots, (\lambda_n,\bm{u}_n), +!et +and the eigenvalues are given by the diagonal matrix +!bt +\[ +\bm{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n). +\] +!et +The matrix $\bm{X}$ can be written in terms of an orthogonal/unitary transformation $\bm{U}$ +!bt +\[ +\bm{X} = \bm{U}\bm{\Sigma}\bm{V}^T, +\] +!et +with $\bm{U}\bm{U}^T=\bm{I}$ or $\bm{U}\bm{U}^{\dagger}=\bm{I}$. + +Not all square matrices are diagonalizable. A matrix like the one discussed above +!bt +\[ +\bm{X} = \begin{bmatrix} +1& -1 \\ +1& -1\\ +\end{bmatrix} +\] +!et +is not diagonalizable, it is a so-called "defective matrix":"https://en.wikipedia.org/wiki/Defective_matrix". It is easy to see that the condition +$\bm{X}\bm{X}^T=\bm{X}^T\bm{X}$ is not fulfilled. + + + +===== The SVD, a Fantastic Algorithm ===== + + +However, and this is the strength of the SVD algorithm, any general +matrix $\bm{X}$ can be decomposed in terms of a diagonal matrix and +two orthogonal/unitary matrices. The "Singular Value Decompostion +(SVD) theorem":"https://en.wikipedia.org/wiki/Singular_value_decomposition" +states that a general $m\times n$ matrix $\bm{X}$ can be written in +terms of a diagonal matrix $\bm{\Sigma}$ of dimensionality $m\times n$ +and two orthognal matrices $\bm{U}$ and $\bm{V}$, where the first has +dimensionality $m \times m$ and the last dimensionality $n\times n$. +We have then + +!bt +\[ +\bm{X} = \bm{U}\bm{\Sigma}\bm{V}^T +\] +!et + +As an example, the above defective matrix can be decomposed as + +!bt +\[ +\bm{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\bm{U}\bm{\Sigma}\bm{V}^T, +\] +!et + +with eigenvalues $\sigma_1=2$ and $\sigma_2=0$. +The SVD exits always! + +The SVD +decomposition (singular values) gives eigenvalues +$\sigma_i\geq\sigma_{i+1}$ for all $i$ and for dimensions larger than $i=p$, the +eigenvalues (singular values) are zero. + +In the general case, where our design matrix $\bm{X}$ has dimension +$n\times p$, the matrix is thus decomposed into an $n\times n$ +orthogonal matrix $\bm{U}$, a $p\times p$ orthogonal matrix $\bm{V}$ +and a diagonal matrix $\bm{\Sigma}$ with $r=\mathrm{min}(n,p)$ +singular values $\sigma_i\geq 0$ on the main diagonal and zeros filling +the rest of the matrix. There are at most $p$ singular values +assuming that $n > p$. In our regression examples for the nuclear +masses and the equation of state this is indeed the case, while for +the Ising model we have $p > n$. These are often cases that lead to +near singular or singular matrices. + +The columns of $\bm{U}$ are called the left singular vectors while the columns of $\bm{V}$ are the right singular vectors. + +===== Economy-size SVD ===== + +If we assume that $n > p$, then our matrix $\bm{U}$ has dimension $n +\times n$. The last $n-p$ columns of $\bm{U}$ become however +irrelevant in our calculations since they are multiplied with the +zeros in $\bm{\Sigma}$. + +The economy-size decomposition removes extra rows or columns of zeros +from the diagonal matrix of singular values, $\bm{\Sigma}$, along with the columns +in either $\bm{U}$ or $\bm{V}$ that multiply those zeros in the expression. +Removing these zeros and columns can improve execution time +and reduce storage requirements without compromising the accuracy of +the decomposition. + +If $n > p$, we keep only the first $p$ columns of $\bm{U}$ and $\bm{\Sigma}$ has dimension $p\times p$. +If $p > n$, then only the first $n$ columns of $\bm{V}$ are computed and $\bm{\Sigma}$ has dimension $n\times n$. +The $n=p$ case is obvious, we retain the full SVD. +In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy. + + +!bc pycod +import numpy as np +# SVD inversion +def SVDinv(A): + ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD). + SVD is numerically more stable than the inversion algorithms provided by + numpy and scipy.linalg at the cost of being slower. + ''' + U, s, VT = np.linalg.svd(A) +# print('test U') +# print( (np.transpose(U) @ U - U @np.transpose(U))) +# print('test VT') +# print( (np.transpose(VT) @ VT - VT @np.transpose(VT))) + print(U) + print(s) + print(VT) + + D = np.zeros((len(U),len(VT))) + for i in range(0,len(VT)): + D[i,i]=s[i] + UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D) + return np.matmul(V,np.matmul(invD,UT)) + + +X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ]) +print(X) +A = np.transpose(X) @ X +print(A) +# Brute force inversion of super-collinear matrix +#B = np.linalg.inv(A) +#print(B) +C = SVDinv(A) +print(C) + +!ec + +The matrix $\bm{X}$ has columns that are linearly dependent. The first +column is the row-wise sum of the other two columns. The rank of a +matrix (the column rank) is the dimension of space spanned by the +column vectors. The rank of the matrix is the number of linearly +independent columns, in this case just $2$. We see this from the +singular values when running the above code. Running the standard +inversion algorithm for matrix inversion with $\bm{X}^T\bm{X}$ results +in the program terminating due to a singular matrix. + + + + +There are several interesting mathematical properties which will be +relevant when we are going to discuss the differences between say +ordinary least squares (OLS) and _Ridge_ regression. + +We have from OLS that the parameters of the linear approximation are given by +!bt +\[ +\bm{\tilde{y}} = \bm{X}\bm{\beta} = \bm{X}\left(\bm{X}^T\bm{X}\right)^{-1}\bm{X}^T\bm{y}. +\] +!et + +The matrix to invert can be rewritten in terms of our SVD decomposition as + +!bt +\[ +\bm{X}^T\bm{X} = \bm{V}\bm{\Sigma}^T\bm{U}^T\bm{U}\bm{\Sigma}\bm{V}^T. +\] +!et +Using the orthogonality properties of $\bm{U}$ we have + +!bt +\[ +\bm{X}^T\bm{X} = \bm{V}\bm{\Sigma}^T\bm{\Sigma}\bm{V}^T = \bm{V}\bm{D}\bm{V}^T, +\] +!et +with $\bm{D}$ being a diagonal matrix with values along the diagonal given by the singular values squared. + +This means that +!bt +\[ +(\bm{X}^T\bm{X})\bm{V} = \bm{V}\bm{D}, +\] +!et +that is the eigenvectors of $(\bm{X}^T\bm{X})$ are given by the columns of the right singular matrix of $\bm{X}$ and the eigenvalues are the squared singular values. It is easy to show (show this) that +!bt +\[ +(\bm{X}\bm{X}^T)\bm{U} = \bm{U}\bm{D}, +\] +!et +that is, the eigenvectors of $(\bm{X}\bm{X})^T$ are the columns of the left singular matrix and the eigenvalues are the same. + +Going back to our OLS equation we have +!bt +\[ +\bm{X}\bm{\beta} = \bm{X}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}\bm{X}^T\bm{y}=\bm{U\Sigma V^T}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}(\bm{U\Sigma V^T})^T\bm{y}=\bm{U}\bm{U}^T\bm{y}. +\] +!et +We will come back to this expression when we discuss Ridge regression. + + +$$ \tilde{y}^{OLS}=\bm{X}\hat{\beta}^{OLS}=\sum_{j=1}^p \bm{u}_j\bm{u}_j^T\bm{y}$$ and for Ridge we have  + +$$ \tilde{y}^{Ridge}=\bm{X}\hat{\beta}^{Ridge}=\sum_{j=1}^p \bm{u}_j\frac{\sigma_j^2}{\sigma_j^2+\lambda}\bm{u}_j^T\bm{y}$$ .  + +It is indeed the economy-sized SVD, note the summation runs up tp $$p$$ only and not $$n$$.  + +Here we have that $$\bm{X} = \bm{U}\bm{\Sigma}\bm{V}^T$$, with $$\Sigma$$ being an $$ n\times p$$ matrix and $$\bm{V}$$ being a $$ p\times p$$ matrix. We also have assumed here that $$ n > p$$.  + + + + + + + + +===== Ridge and LASSO Regression ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" + +Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is +our optimization problem is +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\bm{y}-\bm{X}\bm{\beta}\right)^T\left(\bm{y}-\bm{X}\bm{\beta}\right)\right\}. +\] +!et +or we can state it as +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2, +\] +!et +where we have used the definition of a norm-2 vector, that is +!bt +\[ +\vert\vert \bm{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}. +\] +!et + +By minimizing the above equation with respect to the parameters +$\bm{\beta}$ we could then obtain an analytical expression for the +parameters $\bm{\beta}$. We can add a regularization parameter $\lambda$ by +defining a new cost function to be optimized, that is + +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_2^2 +\] +!et + +which leads to the Ridge regression minimization problem where we +require that $\vert\vert \bm{\beta}\vert\vert_2^2\le t$, where $t$ is +a finite number larger than zero. By defining + +!bt +\[ +C(\bm{X},\bm{\beta})=\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_1, +\] +!et + +we have a new optimization equation +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_1 +\] +!et +which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. + +Here we have defined the norm-1 as +!bt +\[ +\vert\vert \bm{x}\vert\vert_1 = \sum_i \vert x_i\vert. +\] +!et + + + +Using the matrix-vector expression for Ridge regression, + +!bt +\[ +C(\bm{X},\bm{\beta})=\frac{1}{n}\left\{(\bm{y}-\bm{X}\bm{\beta})^T(\bm{y}-\bm{X}\bm{\beta})\right\}+\lambda\bm{\beta}^T\bm{\beta}, +\] +!et + +by taking the derivatives with respect to $\bm{\beta}$ we obtain then +a slightly modified matrix inversion problem which for finite values +of $\lambda$ does not suffer from singularity problems. We obtain + +!bt +\[ +\bm{\beta}^{\mathrm{Ridge}} = \left(\bm{X}^T\bm{X}+\lambda\bm{I}\right)^{-1}\bm{X}^T\bm{y}, +\] +!et + +with $\bm{I}$ being a $p\times p$ identity matrix with the constraint that + +!bt +\[ +\sum_{i=0}^{p-1} \beta_i^2 \leq t, +\] +!et + +with $t$ a finite positive number. + +We see that Ridge regression is nothing but the standard +OLS with a modified diagonal term added to $\bm{X}^T\bm{X}$. The +consequences, in particular for our discussion of the bias-variance tradeoff +are rather interesting. + +Furthermore, if we use the result above in terms of the SVD decomposition (our analysis was done for the OLS method), we had +!bt +\[ +(\bm{X}\bm{X}^T)\bm{U} = \bm{U}\bm{D}. +\] +!et + +We can analyse the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix $\bm{U}$ as +!bt +\[ +\bm{X}\bm{\beta} = \bm{X}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}\bm{X}^T\bm{y}=\bm{U\Sigma V^T}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}(\bm{U\Sigma V^T})^T\bm{y}=\bm{U}\bm{U}^T\bm{y} +\] +!et + + +For Ridge regression this becomes + +!bt +\[ +\bm{X}\bm{\beta}^{\mathrm{Ridge}} = \bm{U\Sigma V^T}\left(\bm{V}\bm{D}\bm{V}^T+\lambda\bm{I} \right)^{-1}(\bm{U\Sigma V^T})^T\bm{y}=\sum_{j=0}^{p-1}\bm{u}_j\bm{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\bm{y}, +\] +!et + +with the vectors $\bm{u}_j$ being the columns of $\bm{U}$. + + +Since $\lambda \geq 0$, it means that compared to OLS, we have + +!bt +\[ +\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1. +\] +!et + +Ridge regression finds the coordinates of $\bm{y}$ with respect to the +orthonormal basis $\bm{U}$, it then shrinks the coordinates by +$\frac{\sigma_j^2}{\sigma_j^2+\lambda}$. Recall that the SVD has +eigenvalues ordered in a descending way, that is $\sigma_i \geq +\sigma_{i+1}$. + +For small eigenvalues $\sigma_i$ it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. +Actually, calculating the variance of $\bm{X}\bm{v}_j$ shows that this quantity is equal to $\sigma_j^2/n$. +With a parameter $\lambda$ we can thus shrink the role of specific parameters. + + + +For the sake of simplicity, let us assume that the design matrix is orthonormal, that is + +!bt +\[ +\bm{X}^T\bm{X}=(\bm{X}^T\bm{X})^{-1} =\bm{I}. +\] +!et + +In this case the standard OLS results in +!bt +\[ +\bm{\beta}^{\mathrm{OLS}} = \bm{X}^T\bm{y}=\sum_{i=0}^{p-1}\bm{u}_j\bm{u}_j^T\bm{y}, +\] +!et + +and + +!bt +\[ +\bm{\beta}^{\mathrm{Ridge}} = \left(\bm{I}+\lambda\bm{I}\right)^{-1}\bm{X}^T\bm{y}=\left(1+\lambda\right)^{-1}\bm{\beta}^{\mathrm{OLS}}, +\] +!et + +that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\lambda$, and +the Ridge estimator converges to zero when the hyperparameter goes to +infinity. + +We will come back to more interpreations after we have gone through some of the statistical analysis part. + +For more discussions of Ridge and Lasso regression, "Wessel van Wieringen's":"https://arxiv.org/abs/1509.09169" article is highly recommended. +Similarly, "Mehta et al's article":"https://arxiv.org/abs/1803.08823" is also recommended. + + + +===== A better understanding of regularization ===== + +The parameter $\lambda$ that we have introduced in the Ridge (and +Lasso as well) regression is often called a regularization parameter +or shrinkage parameter. It is common to call it a hyperparameter. What does it mean mathemtically? + +Here we will first look at how to analyze the difference between the +standard OLS equations and the Ridge expressions in terms of a linear +algebra analysis using the SVD algorithm. Thereafter, we will link +(see the material on the bias-variance tradeoff below) these +observation to the statisical analysis of the results. In particular +we consider how the variance of the parameters $\bm{\beta}$ is +affected by changing the parameter $\lambda$. + + +We have our design matrix + $\bm{X}\in {\mathbb{R}}^{n\times p}$. With the SVD we decompose it as + +!bt +\[ +\bm{X} = \bm{U\Sigma V^T}, +\] +!et + +with $\bm{U}\in {\mathbb{R}}^{n\times n}$, $\bm{\Sigma}\in {\mathbb{R}}^{n\times p}$ +and $\bm{V}\in {\mathbb{R}}^{p\times p}$. + +The matrices $\bm{U}$ and $\bm{V}$ are unitary/orthonormal matrices, that is in case the matrices are real we have $\bm{U}^T\bm{U}=\bm{U}\bm{U}^T=\bm{I}$ and $\bm{V}^T\bm{V}=\bm{V}\bm{V}^T=\bm{I}$. + + + +===== Introducing the Covariance and Correlation functions ===== + +Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about +the definition of the covariance and the correlation function. These are quantities + +Suppose we have defined two vectors +$\hat{x}$ and $\hat{y}$ with $n$ elements each. The covariance matrix $\bm{C}$ is defined as +!bt +\[ +\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} \mathrm{cov}[\bm{x},\bm{x}] & \mathrm{cov}[\bm{x},\bm{y}] \\ + \mathrm{cov}[\bm{y},\bm{x}] & \mathrm{cov}[\bm{y},\bm{y}] \\ + \end{bmatrix}, +\] +!et +where for example +!bt +\[ +\mathrm{cov}[\bm{x},\bm{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\] +!et +With this definition and recalling that the variance is defined as +!bt +\[ +\mathrm{var}[\bm{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +\] +!et +we can rewrite the covariance matrix as +!bt +\[ +\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} \mathrm{var}[\bm{x}] & \mathrm{cov}[\bm{x},\bm{y}] \\ + \mathrm{cov}[\bm{x},\bm{y}] & \mathrm{var}[\bm{y}] \\ + \end{bmatrix}. +\] +!et + +The covariance takes values between zero and infinity and may thus +lead to problems with loss of numerical precision for particularly +large values. It is common to scale the covariance matrix by +introducing instead the correlation matrix defined via the so-called +correlation function + +!bt +\[ +\mathrm{corr}[\bm{x},\bm{y}]=\frac{\mathrm{cov}[\bm{x},\bm{y}]}{\sqrt{\mathrm{var}[\bm{x}] \mathrm{var}[\bm{y}]}}. +\] +!et + +The correlation function is then given by values $\mathrm{corr}[\bm{x},\bm{y}] +\in [-1,1]$. This avoids eventual problems with too large values. We +can then define the correlation matrix for the two vectors $\bm{x}$ +and $\bm{y}$ as + +!bt +\[ +\bm{K}[\bm{x},\bm{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\bm{x},\bm{y}] \\ + \mathrm{corr}[\bm{y},\bm{x}] & 1 \\ + \end{bmatrix}, +\] +!et + +In the above example this is the function we constructed using _pandas_. + + + +In our derivation of the various regression algorithms like _Ordinary Least Squares_ or _Ridge regression_ +we defined the design/feature matrix $\bm{X}$ as + +!bt +\[ +\bm{X}=\begin{bmatrix} +x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ +x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ +x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ +\dots & \dots & \dots & \dots \dots & \dots \\ +x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ +x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ +\end{bmatrix}, +\] +!et +with $\bm{X}\in {\mathbb{R}}^{n\times p}$, with the predictors/features $p$ refering to the column numbers and the +entries $n$ being the row elements. +We can rewrite the design/feature matrix in terms of its column vectors as +!bt +\[ +\bm{X}=\begin{bmatrix} \bm{x}_0 & \bm{x}_1 & \bm{x}_2 & \dots & \dots & \bm{x}_{p-1}\end{bmatrix}, +\] +!et +with a given vector +!bt +\[ +\bm{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}. +\] +!et + +With these definitions, we can now rewrite our $2\times 2$ +correaltion/covariance matrix in terms of a moe general design/feature +matrix $\bm{X}\in {\mathbb{R}}^{n\times p}$. This leads to a $p\times p$ +covariance matrix for the vectors $\bm{x}_i$ with $i=0,1,\dots,p-1$ + +!bt +\[ +\bm{C}[\bm{x}] = \begin{bmatrix} +\mathrm{var}[\bm{x}_0] & \mathrm{cov}[\bm{x}_0,\bm{x}_1] & \mathrm{cov}[\bm{x}_0,\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_0,\bm{x}_{p-1}]\\ +\mathrm{cov}[\bm{x}_1,\bm{x}_0] & \mathrm{var}[\bm{x}_1] & \mathrm{cov}[\bm{x}_1,\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_1,\bm{x}_{p-1}]\\ +\mathrm{cov}[\bm{x}_2,\bm{x}_0] & \mathrm{cov}[\bm{x}_2,\bm{x}_1] & \mathrm{var}[\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_2,\bm{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{cov}[\bm{x}_{p-1},\bm{x}_0] & \mathrm{cov}[\bm{x}_{p-1},\bm{x}_1] & \mathrm{cov}[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & \mathrm{var}[\bm{x}_{p-1}]\\ +\end{bmatrix}, +\] +!et +and the correlation matrix +!bt +\[ +\bm{K}[\bm{x}] = \begin{bmatrix} +1 & \mathrm{corr}[\bm{x}_0,\bm{x}_1] & \mathrm{corr}[\bm{x}_0,\bm{x}_2] & \dots & \dots & \mathrm{corr}[\bm{x}_0,\bm{x}_{p-1}]\\ +\mathrm{corr}[\bm{x}_1,\bm{x}_0] & 1 & \mathrm{corr}[\bm{x}_1,\bm{x}_2] & \dots & \dots & \mathrm{corr}[\bm{x}_1,\bm{x}_{p-1}]\\ +\mathrm{corr}[\bm{x}_2,\bm{x}_0] & \mathrm{corr}[\bm{x}_2,\bm{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\bm{x}_2,\bm{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{corr}[\bm{x}_{p-1},\bm{x}_0] & \mathrm{corr}[\bm{x}_{p-1},\bm{x}_1] & \mathrm{corr}[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & 1\\ +\end{bmatrix}, +\] +!et + + + +The Numpy function _np.cov_ calculates the covariance elements using +the factor $1/(n-1)$ instead of $1/n$ since it assumes we do not have +the exact mean values. The following simple function uses the +_np.vstack_ function which takes each vector of dimension $1\times n$ +and produces a $2\times n$ matrix $\bm{W}$ + + +!bt +\[ +\bm{W} = \begin{bmatrix} x_0 & y_0 \\ + x_1 & y_1 \\ + x_2 & y_2\\ + \dots & \dots \\ + x_{n-2} & y_{n-2}\\ + x_{n-1} & y_{n-1} & + \end{bmatrix}, +\] +!et + +which in turn is converted into into the $2\times 2$ covariance matrix +$\bm{C}$ via the Numpy function _np.cov()_. We note that we can also calculate +the mean value of each set of samples $\bm{x}$ etc using the Numpy +function _np.mean(x)_. We can also extract the eigenvalues of the +covariance matrix through the _np.linalg.eig()_ function. + +!bc pycod +# Importing various packages +import numpy as np +n = 100 +x = np.random.normal(size=n) +print(np.mean(x)) +y = 4+3*x+np.random.normal(size=n) +print(np.mean(y)) +W = np.vstack((x, y)) +C = np.cov(W) +print(C) +!ec + + +The previous example can be converted into the correlation matrix by +simply scaling the matrix elements with the variances. We should also +subtract the mean values for each column. This leads to the following +code which sets up the correlations matrix for the previous example in +a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the $2\times 2$ correlation matrix (since we have only two vectors). + +!bc pycod +import numpy as np +n = 100 +# define two vectors +x = np.random.random(size=n) +y = 4+3*x+np.random.normal(size=n) +#scaling the x and y vectors +x = x - np.mean(x) +y = y - np.mean(y) +variance_x = np.sum(x@x)/n +variance_y = np.sum(y@y)/n +print(variance_x) +print(variance_y) +cov_xy = np.sum(x@y)/n +cov_xx = np.sum(x@x)/n +cov_yy = np.sum(y@y)/n +C = np.zeros((2,2)) +C[0,0]= cov_xx/variance_x +C[1,1]= cov_yy/variance_y +C[0,1]= cov_xy/np.sqrt(variance_y*variance_x) +C[1,0]= C[0,1] +print(C) +!ec + +We see that the matrix elements along the diagonal are one as they +should be and that the matrix is symmetric. Furthermore, diagonalizing +this matrix we easily see that it is a positive definite matrix. + +The above procedure with _numpy_ can be made more compact if we use _pandas_. + + +We whow here how we can set up the correlation matrix using _pandas_, as done in this simple code +!bc pycod +import numpy as np +import pandas as pd +n = 10 +x = np.random.normal(size=n) +x = x - np.mean(x) +y = 4+3*x+np.random.normal(size=n) +y = y - np.mean(y) +X = (np.vstack((x, y))).T +print(X) +Xpd = pd.DataFrame(X) +print(Xpd) +correlation_matrix = Xpd.corr() +print(correlation_matrix) +!ec + + +We expand this model to the Franke function discussed above. + + + +!bc pycod +# Common imports +import numpy as np +import pandas as pd + + +def FrankeFunction(x,y): + term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2)) + term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1)) + term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2)) + term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2) + return term1 + term2 + term3 + term4 + + +def create_X(x, y, n ): + if len(x.shape) > 1: + x = np.ravel(x) + y = np.ravel(y) + + N = len(x) + l = int((n+1)*(n+2)/2) # Number of elements in beta + X = np.ones((N,l)) + + for i in range(1,n+1): + q = int((i)*(i+1)/2) + for k in range(i+1): + X[:,q+k] = (x**(i-k))*(y**k) + + return X + + +# Making meshgrid of datapoints and compute Franke's function +n = 4 +N = 100 +x = np.sort(np.random.uniform(0, 1, N)) +y = np.sort(np.random.uniform(0, 1, N)) +z = FrankeFunction(x, y) +X = create_X(x, y, n=n) + +Xpd = pd.DataFrame(X) +# subtract the mean values and set up the covariance matrix +Xpd = Xpd - Xpd.mean() +covariance_matrix = Xpd.cov() +print(covariance_matrix) +!ec + +We note here that the covariance is zero for the first rows and +columns since all matrix elements in the design matrix were set to one +(we are fitting the function in terms of a polynomial of degree $n$). + +This means that the variance for these elements will be zero and will +cause problems when we set up the correlation matrix. We can simply +drop these elements and construct a correlation +matrix without these elements. + + + + +We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix $\bm{X}$ as +!bt +\[ +\bm{C}[\bm{x}] = \frac{1}{n}\bm{X}^T\bm{X}= \mathbb{E}[\bm{X}^T\bm{X}]. +\] +!et + +To see this let us simply look at a design matrix $\bm{X}\in {\mathbb{R}}^{2\times 2}$ +!bt +\[ +\bm{X}=\begin{bmatrix} +x_{00} & x_{01}\\ +x_{10} & x_{11}\\ +\end{bmatrix}=\begin{bmatrix} +\bm{x}_{0} & \bm{x}_{1}\\ +\end{bmatrix}. +\] +!et + +If we then compute the expectation value +!bt +\[ +\mathbb{E}[\bm{X}^T\bm{X}] = \frac{1}{n}\bm{X}^T\bm{X}=\begin{bmatrix} +x_{00}^2+x_{01}^2 & x_{00}x_{10}+x_{01}x_{11}\\ +x_{10}x_{00}+x_{11}x_{01} & x_{10}^2+x_{11}^2\\ +\end{bmatrix}, +\] +!et +which is just +!bt +\[ +\bm{C}[\bm{x}_0,\bm{x}_1] = \bm{C}[\bm{x}]=\begin{bmatrix} \mathrm{var}[\bm{x}_0] & \mathrm{cov}[\bm{x}_0,\bm{x}_1] \\ + \mathrm{cov}[\bm{x}_1,\bm{x}_0] & \mathrm{var}[\bm{x}_1] \\ + \end{bmatrix}, +\] +!et +where we wrote $$\bm{C}[\bm{x}_0,\bm{x}_1] = \bm{C}[\bm{x}]$$ to indicate that this the covariance of the vectors $\bm{x}$ of the design/feature matrix $\bm{X}$. + +It is easy to generalize this to a matrix $\bm{X}\in {\mathbb{R}}^{n\times p}$. + + +===== Linking with SVD ===== + diff --git a/doc/BookChapters/chapter3.do.txt~ b/doc/BookChapters/chapter3.do.txt~ new file mode 100644 index 000000000..5129e8ad5 --- /dev/null +++ b/doc/BookChapters/chapter3.do.txt~ @@ -0,0 +1,848 @@ +======= Ridge and Lasso Regression ======= + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember10.mp4?vrtx=view-as-webpage" + + +===== The singular value decomposition ===== + +The examples we have looked at so far are cases where we normally can +invert the matrix $\bm{X}^T\bm{X}$. Using a polynomial expansion as we +did both for the masses and the fitting of the equation of state, +leads to row vectors of the design matrix which are essentially +orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition. + + + +This may +however not the be case in general and a standard matrix inversion +algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below. + +There is however a way to partially circumvent this problem and also gain some insights about the ordinary least squares approach, and later shrinkage methods like Ridge and Lasso regressions. + +This is given by the _Singular Value Decomposition_ algorithm, perhaps +the most powerful linear algebra algorithm. Let us look at a +different example where we may have problems with the standard matrix +inversion algorithm. Thereafter we dive into the math of the SVD. + + + +One of the typical problems we encounter with linear regression, in particular +when the matrix $\bm{X}$ (our so-called design matrix) is high-dimensional, +are problems with near singular or singular matrices. The column vectors of $\bm{X}$ +may be linearly dependent, normally referred to as super-collinearity. +This means that the matrix may be rank deficient and it is basically impossible to +to model the data using linear regression. As an example, consider the matrix +!bt +\begin{align*} +\mathbf{X} & = \left[ +\begin{array}{rrr} +1 & -1 & 2 +\\ +1 & 0 & 1 +\\ +1 & 2 & -1 +\\ +1 & 1 & 0 +\end{array} \right] +\end{align*} +!et + +The columns of $\bm{X}$ are linearly dependent. We see this easily since the +the first column is the row-wise sum of the other two columns. The rank (more correct, +the column rank) of a matrix is the dimension of the space spanned by the +column vectors. Hence, the rank of $\mathbf{X}$ is equal to the number +of linearly independent columns. In this particular case the matrix has rank 2. + +Super-collinearity of an $(n \times p)$-dimensional design matrix $\mathbf{X}$ implies +that the inverse of the matrix $\bm{X}^T\bm{X}$ (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this +!bt +\begin{align*} +\bm{X} & = \left[ +\begin{array}{rr} +1 & -1 +\\ +1 & -1 +\end{array} \right]. +\end{align*} +!et +We see easily that $\mbox{det}(\bm{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0$. Hence, $\mathbf{X}$ is singular and its inverse is undefined. +This is equivalent to saying that the matrix $\bm{X}$ has at least an eigenvalue which is zero. + + +If our design matrix $\bm{X}$ which enters the linear regression problem +!bt +\begin{align} +\bm{\beta} & = (\bm{X}^{T} \bm{X})^{-1} \bm{X}^{T} \bm{y}, +\end{align} +!et +has linearly dependent column vectors, we will not be able to compute the inverse +of $\bm{X}^T\bm{X}$ and we cannot find the parameters (estimators) $\beta_i$. +The estimators are only well-defined if $(\bm{X}^{T}\bm{X})^{-1}$ exits. +This is more likely to happen when the matrix $\bm{X}$ is high-dimensional. In this case it is likely to encounter a situation where +the regression parameters $\beta_i$ cannot be estimated. + +A cheap *ad hoc* approach is simply to add a small diagonal component to the matrix to invert, that is we change +!bt +\[ +\bm{X}^{T} \bm{X} \rightarrow \bm{X}^{T} \bm{X}+\lambda \bm{I}, +\] +!et +where $\bm{I}$ is the identity matrix. When we discuss _Ridge_ regression this is actually what we end up evaluating. The parameter $\lambda$ is called a hyperparameter. More about this later. + + + + + +From standard linear algebra we know that a square matrix $\bm{X}$ can be diagonalized if and only it is +a so-called "normal matrix":"https://en.wikipedia.org/wiki/Normal_matrix", that is if $\bm{X}\in {\mathbb{R}}^{n\times n}$ +we have $\bm{X}\bm{X}^T=\bm{X}^T\bm{X}$ or if $\bm{X}\in {\mathbb{C}}^{n\times n}$ we have $\bm{X}\bm{X}^{\dagger}=\bm{X}^{\dagger}\bm{X}$. +The matrix has then a set of eigenpairs + +!bt +\[ +(\lambda_1,\bm{u}_1),\dots, (\lambda_n,\bm{u}_n), +!et +and the eigenvalues are given by the diagonal matrix +!bt +\[ +\bm{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n). +\] +!et +The matrix $\bm{X}$ can be written in terms of an orthogonal/unitary transformation $\bm{U}$ +!bt +\[ +\bm{X} = \bm{U}\bm{\Sigma}\bm{V}^T, +\] +!et +with $\bm{U}\bm{U}^T=\bm{I}$ or $\bm{U}\bm{U}^{\dagger}=\bm{I}$. + +Not all square matrices are diagonalizable. A matrix like the one discussed above +!bt +\[ +\bm{X} = \begin{bmatrix} +1& -1 \\ +1& -1\\ +\end{bmatrix} +\] +!et +is not diagonalizable, it is a so-called "defective matrix":"https://en.wikipedia.org/wiki/Defective_matrix". It is easy to see that the condition +$\bm{X}\bm{X}^T=\bm{X}^T\bm{X}$ is not fulfilled. + + + +===== The SVD, a Fantastic Algorithm ===== + + +However, and this is the strength of the SVD algorithm, any general +matrix $\bm{X}$ can be decomposed in terms of a diagonal matrix and +two orthogonal/unitary matrices. The "Singular Value Decompostion +(SVD) theorem":"https://en.wikipedia.org/wiki/Singular_value_decomposition" +states that a general $m\times n$ matrix $\bm{X}$ can be written in +terms of a diagonal matrix $\bm{\Sigma}$ of dimensionality $m\times n$ +and two orthognal matrices $\bm{U}$ and $\bm{V}$, where the first has +dimensionality $m \times m$ and the last dimensionality $n\times n$. +We have then + +!bt +\[ +\bm{X} = \bm{U}\bm{\Sigma}\bm{V}^T +\] +!et + +As an example, the above defective matrix can be decomposed as + +!bt +\[ +\bm{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\bm{U}\bm{\Sigma}\bm{V}^T, +\] +!et + +with eigenvalues $\sigma_1=2$ and $\sigma_2=0$. +The SVD exits always! + +The SVD +decomposition (singular values) gives eigenvalues +$\sigma_i\geq\sigma_{i+1}$ for all $i$ and for dimensions larger than $i=p$, the +eigenvalues (singular values) are zero. + +In the general case, where our design matrix $\bm{X}$ has dimension +$n\times p$, the matrix is thus decomposed into an $n\times n$ +orthogonal matrix $\bm{U}$, a $p\times p$ orthogonal matrix $\bm{V}$ +and a diagonal matrix $\bm{\Sigma}$ with $r=\mathrm{min}(n,p)$ +singular values $\sigma_i\geq 0$ on the main diagonal and zeros filling +the rest of the matrix. There are at most $p$ singular values +assuming that $n > p$. In our regression examples for the nuclear +masses and the equation of state this is indeed the case, while for +the Ising model we have $p > n$. These are often cases that lead to +near singular or singular matrices. + +The columns of $\bm{U}$ are called the left singular vectors while the columns of $\bm{V}$ are the right singular vectors. + +===== Economy-size SVD ===== + +If we assume that $n > p$, then our matrix $\bm{U}$ has dimension $n +\times n$. The last $n-p$ columns of $\bm{U}$ become however +irrelevant in our calculations since they are multiplied with the +zeros in $\bm{\Sigma}$. + +The economy-size decomposition removes extra rows or columns of zeros +from the diagonal matrix of singular values, $\bm{\Sigma}$, along with the columns +in either $\bm{U}$ or $\bm{V}$ that multiply those zeros in the expression. +Removing these zeros and columns can improve execution time +and reduce storage requirements without compromising the accuracy of +the decomposition. + +If $n > p$, we keep only the first $p$ columns of $\bm{U}$ and $\bm{\Sigma}$ has dimension $p\times p$. +If $p > n$, then only the first $n$ columns of $\bm{V}$ are computed and $\bm{\Sigma}$ has dimension $n\times n$. +The $n=p$ case is obvious, we retain the full SVD. +In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy. + + +!bc pycod +import numpy as np +# SVD inversion +def SVDinv(A): + ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD). + SVD is numerically more stable than the inversion algorithms provided by + numpy and scipy.linalg at the cost of being slower. + ''' + U, s, VT = np.linalg.svd(A) +# print('test U') +# print( (np.transpose(U) @ U - U @np.transpose(U))) +# print('test VT') +# print( (np.transpose(VT) @ VT - VT @np.transpose(VT))) + print(U) + print(s) + print(VT) + + D = np.zeros((len(U),len(VT))) + for i in range(0,len(VT)): + D[i,i]=s[i] + UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D) + return np.matmul(V,np.matmul(invD,UT)) + + +X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ]) +print(X) +A = np.transpose(X) @ X +print(A) +# Brute force inversion of super-collinear matrix +#B = np.linalg.inv(A) +#print(B) +C = SVDinv(A) +print(C) + +!ec + +The matrix $\bm{X}$ has columns that are linearly dependent. The first +column is the row-wise sum of the other two columns. The rank of a +matrix (the column rank) is the dimension of space spanned by the +column vectors. The rank of the matrix is the number of linearly +independent columns, in this case just $2$. We see this from the +singular values when running the above code. Running the standard +inversion algorithm for matrix inversion with $\bm{X}^T\bm{X}$ results +in the program terminating due to a singular matrix. + + + + +There are several interesting mathematical properties which will be +relevant when we are going to discuss the differences between say +ordinary least squares (OLS) and _Ridge_ regression. + +We have from OLS that the parameters of the linear approximation are given by +!bt +\[ +\bm{\tilde{y}} = \bm{X}\bm{\beta} = \bm{X}\left(\bm{X}^T\bm{X}\right)^{-1}\bm{X}^T\bm{y}. +\] +!et + +The matrix to invert can be rewritten in terms of our SVD decomposition as + +!bt +\[ +\bm{X}^T\bm{X} = \bm{V}\bm{\Sigma}^T\bm{U}^T\bm{U}\bm{\Sigma}\bm{V}^T. +\] +!et +Using the orthogonality properties of $\bm{U}$ we have + +!bt +\[ +\bm{X}^T\bm{X} = \bm{V}\bm{\Sigma}^T\bm{\Sigma}\bm{V}^T = \bm{V}\bm{D}\bm{V}^T, +\] +!et +with $\bm{D}$ being a diagonal matrix with values along the diagonal given by the singular values squared. + +This means that +!bt +\[ +(\bm{X}^T\bm{X})\bm{V} = \bm{V}\bm{D}, +\] +!et +that is the eigenvectors of $(\bm{X}^T\bm{X})$ are given by the columns of the right singular matrix of $\bm{X}$ and the eigenvalues are the squared singular values. It is easy to show (show this) that +!bt +\[ +(\bm{X}\bm{X}^T)\bm{U} = \bm{U}\bm{D}, +\] +!et +that is, the eigenvectors of $(\bm{X}\bm{X})^T$ are the columns of the left singular matrix and the eigenvalues are the same. + +Going back to our OLS equation we have +!bt +\[ +\bm{X}\bm{\beta} = \bm{X}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}\bm{X}^T\bm{y}=\bm{U\Sigma V^T}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}(\bm{U\Sigma V^T})^T\bm{y}=\bm{U}\bm{U}^T\bm{y}. +\] +!et +We will come back to this expression when we discuss Ridge regression. + + +$$ \tilde{y}^{OLS}={\bf X}\hat{\beta}^{OLS}=\sum_{j=1}^p {\bf u}_j{\bf u}_j^T{\bf y}$$ and for Ridge we have  + +$$ \tilde{y}^{Ridge}={\bf X}\hat{\beta}^{Ridge}=\sum_{j=1}^p {\bf u}_j\frac{\sigma_j^2}{\sigma_j^2+\lambda}{\bf u}_j^T{\bf y}$$ .  + +It is indeed the economy-sized SVD, note the summation runs up tp $$p$$ only and not $$n$$.  + +Here we have that $${\bf X} = {\bf U}{\bf \Sigma}{\bf V}^T$$, with $$\Sigma$$ being an $$ n\times p$$ matrix and $${\bf V}$$ being a $$ p\times p$$ matrix. We also have assumed here that $$ n > p$$.  + + + + + + + + +===== Ridge and LASSO Regression ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage" + +Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is +our optimization problem is +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\bm{y}-\bm{X}\bm{\beta}\right)^T\left(\bm{y}-\bm{X}\bm{\beta}\right)\right\}. +\] +!et +or we can state it as +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2, +\] +!et +where we have used the definition of a norm-2 vector, that is +!bt +\[ +\vert\vert \bm{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}. +\] +!et + +By minimizing the above equation with respect to the parameters +$\bm{\beta}$ we could then obtain an analytical expression for the +parameters $\bm{\beta}$. We can add a regularization parameter $\lambda$ by +defining a new cost function to be optimized, that is + +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_2^2 +\] +!et + +which leads to the Ridge regression minimization problem where we +require that $\vert\vert \bm{\beta}\vert\vert_2^2\le t$, where $t$ is +a finite number larger than zero. By defining + +!bt +\[ +C(\bm{X},\bm{\beta})=\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_1, +\] +!et + +we have a new optimization equation +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_1 +\] +!et +which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. + +Here we have defined the norm-1 as +!bt +\[ +\vert\vert \bm{x}\vert\vert_1 = \sum_i \vert x_i\vert. +\] +!et + + + +Using the matrix-vector expression for Ridge regression, + +!bt +\[ +C(\bm{X},\bm{\beta})=\frac{1}{n}\left\{(\bm{y}-\bm{X}\bm{\beta})^T(\bm{y}-\bm{X}\bm{\beta})\right\}+\lambda\bm{\beta}^T\bm{\beta}, +\] +!et + +by taking the derivatives with respect to $\bm{\beta}$ we obtain then +a slightly modified matrix inversion problem which for finite values +of $\lambda$ does not suffer from singularity problems. We obtain + +!bt +\[ +\bm{\beta}^{\mathrm{Ridge}} = \left(\bm{X}^T\bm{X}+\lambda\bm{I}\right)^{-1}\bm{X}^T\bm{y}, +\] +!et + +with $\bm{I}$ being a $p\times p$ identity matrix with the constraint that + +!bt +\[ +\sum_{i=0}^{p-1} \beta_i^2 \leq t, +\] +!et + +with $t$ a finite positive number. + +We see that Ridge regression is nothing but the standard +OLS with a modified diagonal term added to $\bm{X}^T\bm{X}$. The +consequences, in particular for our discussion of the bias-variance tradeoff +are rather interesting. + +Furthermore, if we use the result above in terms of the SVD decomposition (our analysis was done for the OLS method), we had +!bt +\[ +(\bm{X}\bm{X}^T)\bm{U} = \bm{U}\bm{D}. +\] +!et + +We can analyse the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix $\bm{U}$ as +!bt +\[ +\bm{X}\bm{\beta} = \bm{X}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}\bm{X}^T\bm{y}=\bm{U\Sigma V^T}\left(\bm{V}\bm{D}\bm{V}^T \right)^{-1}(\bm{U\Sigma V^T})^T\bm{y}=\bm{U}\bm{U}^T\bm{y} +\] +!et + + +For Ridge regression this becomes + +!bt +\[ +\bm{X}\bm{\beta}^{\mathrm{Ridge}} = \bm{U\Sigma V^T}\left(\bm{V}\bm{D}\bm{V}^T+\lambda\bm{I} \right)^{-1}(\bm{U\Sigma V^T})^T\bm{y}=\sum_{j=0}^{p-1}\bm{u}_j\bm{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\bm{y}, +\] +!et + +with the vectors $\bm{u}_j$ being the columns of $\bm{U}$. + + +Since $\lambda \geq 0$, it means that compared to OLS, we have + +!bt +\[ +\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1. +\] +!et + +Ridge regression finds the coordinates of $\bm{y}$ with respect to the +orthonormal basis $\bm{U}$, it then shrinks the coordinates by +$\frac{\sigma_j^2}{\sigma_j^2+\lambda}$. Recall that the SVD has +eigenvalues ordered in a descending way, that is $\sigma_i \geq +\sigma_{i+1}$. + +For small eigenvalues $\sigma_i$ it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. +Actually, calculating the variance of $\bm{X}\bm{v}_j$ shows that this quantity is equal to $\sigma_j^2/n$. +With a parameter $\lambda$ we can thus shrink the role of specific parameters. + + + +For the sake of simplicity, let us assume that the design matrix is orthonormal, that is + +!bt +\[ +\bm{X}^T\bm{X}=(\bm{X}^T\bm{X})^{-1} =\bm{I}. +\] +!et + +In this case the standard OLS results in +!bt +\[ +\bm{\beta}^{\mathrm{OLS}} = \bm{X}^T\bm{y}=\sum_{i=0}^{p-1}\bm{u}_j\bm{u}_j^T\bm{y}, +\] +!et + +and + +!bt +\[ +\bm{\beta}^{\mathrm{Ridge}} = \left(\bm{I}+\lambda\bm{I}\right)^{-1}\bm{X}^T\bm{y}=\left(1+\lambda\right)^{-1}\bm{\beta}^{\mathrm{OLS}}, +\] +!et + +that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\lambda$, and +the Ridge estimator converges to zero when the hyperparameter goes to +infinity. + +We will come back to more interpreations after we have gone through some of the statistical analysis part. + +For more discussions of Ridge and Lasso regression, "Wessel van Wieringen's":"https://arxiv.org/abs/1509.09169" article is highly recommended. +Similarly, "Mehta et al's article":"https://arxiv.org/abs/1803.08823" is also recommended. + + + +===== A better understanding of regularization ===== + +The parameter $\lambda$ that we have introduced in the Ridge (and +Lasso as well) regression is often called a regularization parameter +or shrinkage parameter. It is common to call it a hyperparameter. What does it mean mathemtically? + +Here we will first look at how to analyze the difference between the +standard OLS equations and the Ridge expressions in terms of a linear +algebra analysis using the SVD algorithm. Thereafter, we will link +(see the material on the bias-variance tradeoff below) these +observation to the statisical analysis of the results. In particular +we consider how the variance of the parameters $\bm{\beta}$ is +affected by changing the parameter $\lambda$. + + +We have our design matrix + $\bm{X}\in {\mathbb{R}}^{n\times p}$. With the SVD we decompose it as + +!bt +\[ +\bm{X} = \bm{U\Sigma V^T}, +\] +!et + +with $\bm{U}\in {\mathbb{R}}^{n\times n}$, $\bm{\Sigma}\in {\mathbb{R}}^{n\times p}$ +and $\bm{V}\in {\mathbb{R}}^{p\times p}$. + +The matrices $\bm{U}$ and $\bm{V}$ are unitary/orthonormal matrices, that is in case the matrices are real we have $\bm{U}^T\bm{U}=\bm{U}\bm{U}^T=\bm{I}$ and $\bm{V}^T\bm{V}=\bm{V}\bm{V}^T=\bm{I}$. + + + +===== Introducing the Covariance and Correlation functions ===== + +Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about +the definition of the covariance and the correlation function. These are quantities + +Suppose we have defined two vectors +$\hat{x}$ and $\hat{y}$ with $n$ elements each. The covariance matrix $\bm{C}$ is defined as +!bt +\[ +\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} \mathrm{cov}[\bm{x},\bm{x}] & \mathrm{cov}[\bm{x},\bm{y}] \\ + \mathrm{cov}[\bm{y},\bm{x}] & \mathrm{cov}[\bm{y},\bm{y}] \\ + \end{bmatrix}, +\] +!et +where for example +!bt +\[ +\mathrm{cov}[\bm{x},\bm{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +\] +!et +With this definition and recalling that the variance is defined as +!bt +\[ +\mathrm{var}[\bm{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +\] +!et +we can rewrite the covariance matrix as +!bt +\[ +\bm{C}[\bm{x},\bm{y}] = \begin{bmatrix} \mathrm{var}[\bm{x}] & \mathrm{cov}[\bm{x},\bm{y}] \\ + \mathrm{cov}[\bm{x},\bm{y}] & \mathrm{var}[\bm{y}] \\ + \end{bmatrix}. +\] +!et + +The covariance takes values between zero and infinity and may thus +lead to problems with loss of numerical precision for particularly +large values. It is common to scale the covariance matrix by +introducing instead the correlation matrix defined via the so-called +correlation function + +!bt +\[ +\mathrm{corr}[\bm{x},\bm{y}]=\frac{\mathrm{cov}[\bm{x},\bm{y}]}{\sqrt{\mathrm{var}[\bm{x}] \mathrm{var}[\bm{y}]}}. +\] +!et + +The correlation function is then given by values $\mathrm{corr}[\bm{x},\bm{y}] +\in [-1,1]$. This avoids eventual problems with too large values. We +can then define the correlation matrix for the two vectors $\bm{x}$ +and $\bm{y}$ as + +!bt +\[ +\bm{K}[\bm{x},\bm{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\bm{x},\bm{y}] \\ + \mathrm{corr}[\bm{y},\bm{x}] & 1 \\ + \end{bmatrix}, +\] +!et + +In the above example this is the function we constructed using _pandas_. + + + +In our derivation of the various regression algorithms like _Ordinary Least Squares_ or _Ridge regression_ +we defined the design/feature matrix $\bm{X}$ as + +!bt +\[ +\bm{X}=\begin{bmatrix} +x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ +x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ +x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ +\dots & \dots & \dots & \dots \dots & \dots \\ +x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ +x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ +\end{bmatrix}, +\] +!et +with $\bm{X}\in {\mathbb{R}}^{n\times p}$, with the predictors/features $p$ refering to the column numbers and the +entries $n$ being the row elements. +We can rewrite the design/feature matrix in terms of its column vectors as +!bt +\[ +\bm{X}=\begin{bmatrix} \bm{x}_0 & \bm{x}_1 & \bm{x}_2 & \dots & \dots & \bm{x}_{p-1}\end{bmatrix}, +\] +!et +with a given vector +!bt +\[ +\bm{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}. +\] +!et + +With these definitions, we can now rewrite our $2\times 2$ +correaltion/covariance matrix in terms of a moe general design/feature +matrix $\bm{X}\in {\mathbb{R}}^{n\times p}$. This leads to a $p\times p$ +covariance matrix for the vectors $\bm{x}_i$ with $i=0,1,\dots,p-1$ + +!bt +\[ +\bm{C}[\bm{x}] = \begin{bmatrix} +\mathrm{var}[\bm{x}_0] & \mathrm{cov}[\bm{x}_0,\bm{x}_1] & \mathrm{cov}[\bm{x}_0,\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_0,\bm{x}_{p-1}]\\ +\mathrm{cov}[\bm{x}_1,\bm{x}_0] & \mathrm{var}[\bm{x}_1] & \mathrm{cov}[\bm{x}_1,\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_1,\bm{x}_{p-1}]\\ +\mathrm{cov}[\bm{x}_2,\bm{x}_0] & \mathrm{cov}[\bm{x}_2,\bm{x}_1] & \mathrm{var}[\bm{x}_2] & \dots & \dots & \mathrm{cov}[\bm{x}_2,\bm{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{cov}[\bm{x}_{p-1},\bm{x}_0] & \mathrm{cov}[\bm{x}_{p-1},\bm{x}_1] & \mathrm{cov}[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & \mathrm{var}[\bm{x}_{p-1}]\\ +\end{bmatrix}, +\] +!et +and the correlation matrix +!bt +\[ +\bm{K}[\bm{x}] = \begin{bmatrix} +1 & \mathrm{corr}[\bm{x}_0,\bm{x}_1] & \mathrm{corr}[\bm{x}_0,\bm{x}_2] & \dots & \dots & \mathrm{corr}[\bm{x}_0,\bm{x}_{p-1}]\\ +\mathrm{corr}[\bm{x}_1,\bm{x}_0] & 1 & \mathrm{corr}[\bm{x}_1,\bm{x}_2] & \dots & \dots & \mathrm{corr}[\bm{x}_1,\bm{x}_{p-1}]\\ +\mathrm{corr}[\bm{x}_2,\bm{x}_0] & \mathrm{corr}[\bm{x}_2,\bm{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\bm{x}_2,\bm{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{corr}[\bm{x}_{p-1},\bm{x}_0] & \mathrm{corr}[\bm{x}_{p-1},\bm{x}_1] & \mathrm{corr}[\bm{x}_{p-1},\bm{x}_{2}] & \dots & \dots & 1\\ +\end{bmatrix}, +\] +!et + + + +The Numpy function _np.cov_ calculates the covariance elements using +the factor $1/(n-1)$ instead of $1/n$ since it assumes we do not have +the exact mean values. The following simple function uses the +_np.vstack_ function which takes each vector of dimension $1\times n$ +and produces a $2\times n$ matrix $\bm{W}$ + + +!bt +\[ +\bm{W} = \begin{bmatrix} x_0 & y_0 \\ + x_1 & y_1 \\ + x_2 & y_2\\ + \dots & \dots \\ + x_{n-2} & y_{n-2}\\ + x_{n-1} & y_{n-1} & + \end{bmatrix}, +\] +!et + +which in turn is converted into into the $2\times 2$ covariance matrix +$\bm{C}$ via the Numpy function _np.cov()_. We note that we can also calculate +the mean value of each set of samples $\bm{x}$ etc using the Numpy +function _np.mean(x)_. We can also extract the eigenvalues of the +covariance matrix through the _np.linalg.eig()_ function. + +!bc pycod +# Importing various packages +import numpy as np +n = 100 +x = np.random.normal(size=n) +print(np.mean(x)) +y = 4+3*x+np.random.normal(size=n) +print(np.mean(y)) +W = np.vstack((x, y)) +C = np.cov(W) +print(C) +!ec + + +The previous example can be converted into the correlation matrix by +simply scaling the matrix elements with the variances. We should also +subtract the mean values for each column. This leads to the following +code which sets up the correlations matrix for the previous example in +a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the $2\times 2$ correlation matrix (since we have only two vectors). + +!bc pycod +import numpy as np +n = 100 +# define two vectors +x = np.random.random(size=n) +y = 4+3*x+np.random.normal(size=n) +#scaling the x and y vectors +x = x - np.mean(x) +y = y - np.mean(y) +variance_x = np.sum(x@x)/n +variance_y = np.sum(y@y)/n +print(variance_x) +print(variance_y) +cov_xy = np.sum(x@y)/n +cov_xx = np.sum(x@x)/n +cov_yy = np.sum(y@y)/n +C = np.zeros((2,2)) +C[0,0]= cov_xx/variance_x +C[1,1]= cov_yy/variance_y +C[0,1]= cov_xy/np.sqrt(variance_y*variance_x) +C[1,0]= C[0,1] +print(C) +!ec + +We see that the matrix elements along the diagonal are one as they +should be and that the matrix is symmetric. Furthermore, diagonalizing +this matrix we easily see that it is a positive definite matrix. + +The above procedure with _numpy_ can be made more compact if we use _pandas_. + + +We whow here how we can set up the correlation matrix using _pandas_, as done in this simple code +!bc pycod +import numpy as np +import pandas as pd +n = 10 +x = np.random.normal(size=n) +x = x - np.mean(x) +y = 4+3*x+np.random.normal(size=n) +y = y - np.mean(y) +X = (np.vstack((x, y))).T +print(X) +Xpd = pd.DataFrame(X) +print(Xpd) +correlation_matrix = Xpd.corr() +print(correlation_matrix) +!ec + + +We expand this model to the Franke function discussed above. + + + +!bc pycod +# Common imports +import numpy as np +import pandas as pd + + +def FrankeFunction(x,y): + term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2)) + term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1)) + term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2)) + term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2) + return term1 + term2 + term3 + term4 + + +def create_X(x, y, n ): + if len(x.shape) > 1: + x = np.ravel(x) + y = np.ravel(y) + + N = len(x) + l = int((n+1)*(n+2)/2) # Number of elements in beta + X = np.ones((N,l)) + + for i in range(1,n+1): + q = int((i)*(i+1)/2) + for k in range(i+1): + X[:,q+k] = (x**(i-k))*(y**k) + + return X + + +# Making meshgrid of datapoints and compute Franke's function +n = 4 +N = 100 +x = np.sort(np.random.uniform(0, 1, N)) +y = np.sort(np.random.uniform(0, 1, N)) +z = FrankeFunction(x, y) +X = create_X(x, y, n=n) + +Xpd = pd.DataFrame(X) +# subtract the mean values and set up the covariance matrix +Xpd = Xpd - Xpd.mean() +covariance_matrix = Xpd.cov() +print(covariance_matrix) +!ec + +We note here that the covariance is zero for the first rows and +columns since all matrix elements in the design matrix were set to one +(we are fitting the function in terms of a polynomial of degree $n$). + +This means that the variance for these elements will be zero and will +cause problems when we set up the correlation matrix. We can simply +drop these elements and construct a correlation +matrix without these elements. + + + + +We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix $\bm{X}$ as +!bt +\[ +\bm{C}[\bm{x}] = \frac{1}{n}\bm{X}^T\bm{X}= \mathbb{E}[\bm{X}^T\bm{X}]. +\] +!et + +To see this let us simply look at a design matrix $\bm{X}\in {\mathbb{R}}^{2\times 2}$ +!bt +\[ +\bm{X}=\begin{bmatrix} +x_{00} & x_{01}\\ +x_{10} & x_{11}\\ +\end{bmatrix}=\begin{bmatrix} +\bm{x}_{0} & \bm{x}_{1}\\ +\end{bmatrix}. +\] +!et + +If we then compute the expectation value +!bt +\[ +\mathbb{E}[\bm{X}^T\bm{X}] = \frac{1}{n}\bm{X}^T\bm{X}=\begin{bmatrix} +x_{00}^2+x_{01}^2 & x_{00}x_{10}+x_{01}x_{11}\\ +x_{10}x_{00}+x_{11}x_{01} & x_{10}^2+x_{11}^2\\ +\end{bmatrix}, +\] +!et +which is just +!bt +\[ +\bm{C}[\bm{x}_0,\bm{x}_1] = \bm{C}[\bm{x}]=\begin{bmatrix} \mathrm{var}[\bm{x}_0] & \mathrm{cov}[\bm{x}_0,\bm{x}_1] \\ + \mathrm{cov}[\bm{x}_1,\bm{x}_0] & \mathrm{var}[\bm{x}_1] \\ + \end{bmatrix}, +\] +!et +where we wrote $$\bm{C}[\bm{x}_0,\bm{x}_1] = \bm{C}[\bm{x}]$$ to indicate that this the covariance of the vectors $\bm{x}$ of the design/feature matrix $\bm{X}$. + +It is easy to generalize this to a matrix $\bm{X}\in {\mathbb{R}}^{n\times p}$. + + +===== Linking with SVD ===== + diff --git a/doc/BookChapters/chapter3.ipynb b/doc/BookChapters/chapter3.ipynb new file mode 100644 index 000000000..5ede9f687 --- /dev/null +++ b/doc/BookChapters/chapter3.ipynb @@ -0,0 +1,1378 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ridge and Lasso Regression\n", + "\n", + "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember10.mp4?vrtx=view-as-webpage)\n", + "\n", + "\n", + "## The singular value decomposition\n", + "\n", + "The examples we have looked at so far are cases where we normally can\n", + "invert the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$. Using a polynomial expansion as we\n", + "did both for the masses and the fitting of the equation of state,\n", + "leads to row vectors of the design matrix which are essentially\n", + "orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition. \n", + "\n", + "\n", + "\n", + "This may\n", + "however not the be case in general and a standard matrix inversion\n", + "algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.\n", + "\n", + "There is however a way to partially circumvent this problem and also gain some insights about the ordinary least squares approach, and later shrinkage methods like Ridge and Lasso regressions. \n", + "\n", + "This is given by the **Singular Value Decomposition** algorithm, perhaps\n", + "the most powerful linear algebra algorithm. Let us look at a\n", + "different example where we may have problems with the standard matrix\n", + "inversion algorithm. Thereafter we dive into the math of the SVD.\n", + "\n", + "\n", + "\n", + "One of the typical problems we encounter with linear regression, in particular \n", + "when the matrix $\\boldsymbol{X}$ (our so-called design matrix) is high-dimensional, \n", + "are problems with near singular or singular matrices. The column vectors of $\\boldsymbol{X}$ \n", + "may be linearly dependent, normally referred to as super-collinearity. \n", + "This means that the matrix may be rank deficient and it is basically impossible to \n", + "to model the data using linear regression. As an example, consider the matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "\\mathbf{X} & = \\left[\n", + "\\begin{array}{rrr}\n", + "1 & -1 & 2\n", + "\\\\\n", + "1 & 0 & 1\n", + "\\\\\n", + "1 & 2 & -1\n", + "\\\\\n", + "1 & 1 & 0\n", + "\\end{array} \\right]\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The columns of $\\boldsymbol{X}$ are linearly dependent. We see this easily since the \n", + "the first column is the row-wise sum of the other two columns. The rank (more correct,\n", + "the column rank) of a matrix is the dimension of the space spanned by the\n", + "column vectors. Hence, the rank of $\\mathbf{X}$ is equal to the number\n", + "of linearly independent columns. In this particular case the matrix has rank 2.\n", + "\n", + "Super-collinearity of an $(n \\times p)$-dimensional design matrix $\\mathbf{X}$ implies\n", + "that the inverse of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$ (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "\\boldsymbol{X} & = \\left[\n", + "\\begin{array}{rr}\n", + "1 & -1\n", + "\\\\\n", + "1 & -1\n", + "\\end{array} \\right].\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see easily that $\\mbox{det}(\\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \\times (-1) - 1 \\times (-1) = 0$. Hence, $\\mathbf{X}$ is singular and its inverse is undefined.\n", + "This is equivalent to saying that the matrix $\\boldsymbol{X}$ has at least an eigenvalue which is zero.\n", + "\n", + "\n", + "If our design matrix $\\boldsymbol{X}$ which enters the linear regression problem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "
\n", + "\n", + "$$\n", + "\\begin{equation}\n", + "\\boldsymbol{\\beta} = (\\boldsymbol{X}^{T} \\boldsymbol{X})^{-1} \\boldsymbol{X}^{T} \\boldsymbol{y},\n", + "\\label{_auto1} \\tag{1}\n", + "\\end{equation}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "has linearly dependent column vectors, we will not be able to compute the inverse\n", + "of $\\boldsymbol{X}^T\\boldsymbol{X}$ and we cannot find the parameters (estimators) $\\beta_i$. \n", + "The estimators are only well-defined if $(\\boldsymbol{X}^{T}\\boldsymbol{X})^{-1}$ exits. \n", + "This is more likely to happen when the matrix $\\boldsymbol{X}$ is high-dimensional. In this case it is likely to encounter a situation where \n", + "the regression parameters $\\beta_i$ cannot be estimated.\n", + "\n", + "A cheap *ad hoc* approach is simply to add a small diagonal component to the matrix to invert, that is we change" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^{T} \\boldsymbol{X} \\rightarrow \\boldsymbol{X}^{T} \\boldsymbol{X}+\\lambda \\boldsymbol{I},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\boldsymbol{I}$ is the identity matrix. When we discuss **Ridge** regression this is actually what we end up evaluating. The parameter $\\lambda$ is called a hyperparameter. More about this later. \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "From standard linear algebra we know that a square matrix $\\boldsymbol{X}$ can be diagonalized if and only it is \n", + "a so-called [normal matrix](https://en.wikipedia.org/wiki/Normal_matrix), that is if $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times n}$\n", + "we have $\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{X}^T\\boldsymbol{X}$ or if $\\boldsymbol{X}\\in {\\mathbb{C}}^{n\\times n}$ we have $\\boldsymbol{X}\\boldsymbol{X}^{\\dagger}=\\boldsymbol{X}^{\\dagger}\\boldsymbol{X}$.\n", + "The matrix has then a set of eigenpairs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "(\\lambda_1,\\boldsymbol{u}_1),\\dots, (\\lambda_n,\\boldsymbol{u}_n),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and the eigenvalues are given by the diagonal matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\Sigma}=\\mathrm{Diag}(\\lambda_1, \\dots,\\lambda_n).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The matrix $\\boldsymbol{X}$ can be written in terms of an orthogonal/unitary transformation $\\boldsymbol{U}$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X} = \\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $\\boldsymbol{U}\\boldsymbol{U}^T=\\boldsymbol{I}$ or $\\boldsymbol{U}\\boldsymbol{U}^{\\dagger}=\\boldsymbol{I}$.\n", + "\n", + "Not all square matrices are diagonalizable. A matrix like the one discussed above" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X} = \\begin{bmatrix} \n", + "1& -1 \\\\\n", + "1& -1\\\\\n", + "\\end{bmatrix}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "is not diagonalizable, it is a so-called [defective matrix](https://en.wikipedia.org/wiki/Defective_matrix). It is easy to see that the condition\n", + "$\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{X}^T\\boldsymbol{X}$ is not fulfilled. \n", + "\n", + "\n", + "\n", + "## The SVD, a Fantastic Algorithm\n", + "\n", + "\n", + "However, and this is the strength of the SVD algorithm, any general\n", + "matrix $\\boldsymbol{X}$ can be decomposed in terms of a diagonal matrix and\n", + "two orthogonal/unitary matrices. The [Singular Value Decompostion\n", + "(SVD) theorem](https://en.wikipedia.org/wiki/Singular_value_decomposition)\n", + "states that a general $m\\times n$ matrix $\\boldsymbol{X}$ can be written in\n", + "terms of a diagonal matrix $\\boldsymbol{\\Sigma}$ of dimensionality $m\\times n$\n", + "and two orthognal matrices $\\boldsymbol{U}$ and $\\boldsymbol{V}$, where the first has\n", + "dimensionality $m \\times m$ and the last dimensionality $n\\times n$.\n", + "We have then" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X} = \\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As an example, the above defective matrix can be decomposed as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X} = \\frac{1}{\\sqrt{2}}\\begin{bmatrix} 1& 1 \\\\ 1& -1\\\\ \\end{bmatrix} \\begin{bmatrix} 2& 0 \\\\ 0& 0\\\\ \\end{bmatrix} \\frac{1}{\\sqrt{2}}\\begin{bmatrix} 1& -1 \\\\ 1& 1\\\\ \\end{bmatrix}=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with eigenvalues $\\sigma_1=2$ and $\\sigma_2=0$. \n", + "The SVD exits always! \n", + "\n", + "The SVD\n", + "decomposition (singular values) gives eigenvalues \n", + "$\\sigma_i\\geq\\sigma_{i+1}$ for all $i$ and for dimensions larger than $i=p$, the\n", + "eigenvalues (singular values) are zero.\n", + "\n", + "In the general case, where our design matrix $\\boldsymbol{X}$ has dimension\n", + "$n\\times p$, the matrix is thus decomposed into an $n\\times n$\n", + "orthogonal matrix $\\boldsymbol{U}$, a $p\\times p$ orthogonal matrix $\\boldsymbol{V}$\n", + "and a diagonal matrix $\\boldsymbol{\\Sigma}$ with $r=\\mathrm{min}(n,p)$\n", + "singular values $\\sigma_i\\geq 0$ on the main diagonal and zeros filling\n", + "the rest of the matrix. There are at most $p$ singular values\n", + "assuming that $n > p$. In our regression examples for the nuclear\n", + "masses and the equation of state this is indeed the case, while for\n", + "the Ising model we have $p > n$. These are often cases that lead to\n", + "near singular or singular matrices.\n", + "\n", + "The columns of $\\boldsymbol{U}$ are called the left singular vectors while the columns of $\\boldsymbol{V}$ are the right singular vectors.\n", + "\n", + "## Economy-size SVD\n", + "\n", + "If we assume that $n > p$, then our matrix $\\boldsymbol{U}$ has dimension $n\n", + "\\times n$. The last $n-p$ columns of $\\boldsymbol{U}$ become however\n", + "irrelevant in our calculations since they are multiplied with the\n", + "zeros in $\\boldsymbol{\\Sigma}$.\n", + "\n", + "The economy-size decomposition removes extra rows or columns of zeros\n", + "from the diagonal matrix of singular values, $\\boldsymbol{\\Sigma}$, along with the columns\n", + "in either $\\boldsymbol{U}$ or $\\boldsymbol{V}$ that multiply those zeros in the expression. \n", + "Removing these zeros and columns can improve execution time\n", + "and reduce storage requirements without compromising the accuracy of\n", + "the decomposition.\n", + "\n", + "If $n > p$, we keep only the first $p$ columns of $\\boldsymbol{U}$ and $\\boldsymbol{\\Sigma}$ has dimension $p\\times p$. \n", + "If $p > n$, then only the first $n$ columns of $\\boldsymbol{V}$ are computed and $\\boldsymbol{\\Sigma}$ has dimension $n\\times n$.\n", + "The $n=p$ case is obvious, we retain the full SVD. \n", + "In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "# SVD inversion\n", + "def SVDinv(A):\n", + " ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).\n", + " SVD is numerically more stable than the inversion algorithms provided by\n", + " numpy and scipy.linalg at the cost of being slower.\n", + " '''\n", + " U, s, VT = np.linalg.svd(A)\n", + "# print('test U')\n", + "# print( (np.transpose(U) @ U - U @np.transpose(U)))\n", + "# print('test VT')\n", + "# print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))\n", + " print(U)\n", + " print(s)\n", + " print(VT)\n", + "\n", + " D = np.zeros((len(U),len(VT)))\n", + " for i in range(0,len(VT)):\n", + " D[i,i]=s[i]\n", + " UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)\n", + " return np.matmul(V,np.matmul(invD,UT))\n", + "\n", + "\n", + "X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])\n", + "print(X)\n", + "A = np.transpose(X) @ X\n", + "print(A)\n", + "# Brute force inversion of super-collinear matrix\n", + "#B = np.linalg.inv(A)\n", + "#print(B)\n", + "C = SVDinv(A)\n", + "print(C)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The matrix $\\boldsymbol{X}$ has columns that are linearly dependent. The first\n", + "column is the row-wise sum of the other two columns. The rank of a\n", + "matrix (the column rank) is the dimension of space spanned by the\n", + "column vectors. The rank of the matrix is the number of linearly\n", + "independent columns, in this case just $2$. We see this from the\n", + "singular values when running the above code. Running the standard\n", + "inversion algorithm for matrix inversion with $\\boldsymbol{X}^T\\boldsymbol{X}$ results\n", + "in the program terminating due to a singular matrix.\n", + "\n", + "\n", + "\n", + "\n", + "There are several interesting mathematical properties which will be\n", + "relevant when we are going to discuss the differences between say\n", + "ordinary least squares (OLS) and **Ridge** regression.\n", + "\n", + "We have from OLS that the parameters of the linear approximation are given by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta} = \\boldsymbol{X}\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The matrix to invert can be rewritten in terms of our SVD decomposition as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{X} = \\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the orthogonality properties of $\\boldsymbol{U}$ we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{X} = \\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T = \\boldsymbol{V}\\boldsymbol{D}\\boldsymbol{V}^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $\\boldsymbol{D}$ being a diagonal matrix with values along the diagonal given by the singular values squared. \n", + "\n", + "This means that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "(\\boldsymbol{X}^T\\boldsymbol{X})\\boldsymbol{V} = \\boldsymbol{V}\\boldsymbol{D},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "that is the eigenvectors of $(\\boldsymbol{X}^T\\boldsymbol{X})$ are given by the columns of the right singular matrix of $\\boldsymbol{X}$ and the eigenvalues are the squared singular values. It is easy to show (show this) that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "(\\boldsymbol{X}\\boldsymbol{X}^T)\\boldsymbol{U} = \\boldsymbol{U}\\boldsymbol{D},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "that is, the eigenvectors of $(\\boldsymbol{X}\\boldsymbol{X})^T$ are the columns of the left singular matrix and the eigenvalues are the same. \n", + "\n", + "Going back to our OLS equation we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}\\boldsymbol{\\beta} = \\boldsymbol{X}\\left(\\boldsymbol{V}\\boldsymbol{D}\\boldsymbol{V}^T \\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{D}\\boldsymbol{V}^T \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will come back to this expression when we discuss Ridge regression. \n", + "\n", + "\n", + "$$ \\tilde{y}^{OLS}=\\boldsymbol{X}\\hat{\\beta}^{OLS}=\\sum_{j=1}^p \\boldsymbol{u}_j\\boldsymbol{u}_j^T\\boldsymbol{y}$$ and for Ridge we have \n", + "\n", + "$$ \\tilde{y}^{Ridge}=\\boldsymbol{X}\\hat{\\beta}^{Ridge}=\\sum_{j=1}^p \\boldsymbol{u}_j\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{u}_j^T\\boldsymbol{y}$$ . \n", + "\n", + "It is indeed the economy-sized SVD, note the summation runs up tp $$p$$ only and not $$n$$. \n", + "\n", + "Here we have that $$\\boldsymbol{X} = \\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T$$, with $$\\Sigma$$ being an $$ n\\times p$$ matrix and $$\\boldsymbol{V}$$ being a $$ p\\times p$$ matrix. We also have assumed here that $$ n > p$$. \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "## Ridge and LASSO Regression\n", + "\n", + "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember11.mp4?vrtx=view-as-webpage)\n", + "\n", + "Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is \n", + "our optimization problem is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or we can state it as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we have used the definition of a norm-2 vector, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By minimizing the above equation with respect to the parameters\n", + "$\\boldsymbol{\\beta}$ we could then obtain an analytical expression for the\n", + "parameters $\\boldsymbol{\\beta}$. We can add a regularization parameter $\\lambda$ by\n", + "defining a new cost function to be optimized, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which leads to the Ridge regression minimization problem where we\n", + "require that $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\\le t$, where $t$ is\n", + "a finite number larger than zero. By defining" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have a new optimization equation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n", + "\n", + "Here we have defined the norm-1 as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the matrix-vector expression for Ridge regression," + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\boldsymbol{\\beta}^T\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "by taking the derivatives with respect to $\\boldsymbol{\\beta}$ we obtain then\n", + "a slightly modified matrix inversion problem which for finite values\n", + "of $\\lambda$ does not suffer from singularity problems. We obtain" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $\\boldsymbol{I}$ being a $p\\times p$ identity matrix with the constraint that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\sum_{i=0}^{p-1} \\beta_i^2 \\leq t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $t$ a finite positive number. \n", + "\n", + "We see that Ridge regression is nothing but the standard\n", + "OLS with a modified diagonal term added to $\\boldsymbol{X}^T\\boldsymbol{X}$. The\n", + "consequences, in particular for our discussion of the bias-variance tradeoff \n", + "are rather interesting.\n", + "\n", + "Furthermore, if we use the result above in terms of the SVD decomposition (our analysis was done for the OLS method), we had" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "(\\boldsymbol{X}\\boldsymbol{X}^T)\\boldsymbol{U} = \\boldsymbol{U}\\boldsymbol{D}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can analyse the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix $\\boldsymbol{U}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}\\boldsymbol{\\beta} = \\boldsymbol{X}\\left(\\boldsymbol{V}\\boldsymbol{D}\\boldsymbol{V}^T \\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{D}\\boldsymbol{V}^T \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For Ridge regression this becomes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{D}\\boldsymbol{V}^T+\\lambda\\boldsymbol{I} \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\sum_{j=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{y},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with the vectors $\\boldsymbol{u}_j$ being the columns of $\\boldsymbol{U}$. \n", + "\n", + "\n", + "Since $\\lambda \\geq 0$, it means that compared to OLS, we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda} \\leq 1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ridge regression finds the coordinates of $\\boldsymbol{y}$ with respect to the\n", + "orthonormal basis $\\boldsymbol{U}$, it then shrinks the coordinates by\n", + "$\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}$. Recall that the SVD has\n", + "eigenvalues ordered in a descending way, that is $\\sigma_i \\geq\n", + "\\sigma_{i+1}$.\n", + "\n", + "For small eigenvalues $\\sigma_i$ it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom.\n", + "Actually, calculating the variance of $\\boldsymbol{X}\\boldsymbol{v}_j$ shows that this quantity is equal to $\\sigma_j^2/n$.\n", + "With a parameter $\\lambda$ we can thus shrink the role of specific parameters. \n", + "\n", + "\n", + "\n", + "For the sake of simplicity, let us assume that the design matrix is orthonormal, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{X}=(\\boldsymbol{X}^T\\boldsymbol{X})^{-1} =\\boldsymbol{I}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this case the standard OLS results in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta}^{\\mathrm{OLS}} = \\boldsymbol{X}^T\\boldsymbol{y}=\\sum_{i=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\boldsymbol{y},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{I}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\left(1+\\lambda\\right)^{-1}\\boldsymbol{\\beta}^{\\mathrm{OLS}},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\\lambda$, and\n", + "the Ridge estimator converges to zero when the hyperparameter goes to\n", + "infinity.\n", + "\n", + "We will come back to more interpreations after we have gone through some of the statistical analysis part. \n", + "\n", + "For more discussions of Ridge and Lasso regression, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "Similarly, [Mehta et al's article](https://arxiv.org/abs/1803.08823) is also recommended.\n", + "\n", + "\n", + "\n", + "## A better understanding of regularization\n", + "\n", + "The parameter $\\lambda$ that we have introduced in the Ridge (and\n", + "Lasso as well) regression is often called a regularization parameter\n", + "or shrinkage parameter. It is common to call it a hyperparameter. What does it mean mathemtically?\n", + "\n", + "Here we will first look at how to analyze the difference between the\n", + "standard OLS equations and the Ridge expressions in terms of a linear\n", + "algebra analysis using the SVD algorithm. Thereafter, we will link\n", + "(see the material on the bias-variance tradeoff below) these\n", + "observation to the statisical analysis of the results. In particular\n", + "we consider how the variance of the parameters $\\boldsymbol{\\beta}$ is\n", + "affected by changing the parameter $\\lambda$.\n", + "\n", + "\n", + "We have our design matrix\n", + " $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$. With the SVD we decompose it as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X} = \\boldsymbol{U\\Sigma V^T},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $\\boldsymbol{U}\\in {\\mathbb{R}}^{n\\times n}$, $\\boldsymbol{\\Sigma}\\in {\\mathbb{R}}^{n\\times p}$\n", + "and $\\boldsymbol{V}\\in {\\mathbb{R}}^{p\\times p}$.\n", + "\n", + "The matrices $\\boldsymbol{U}$ and $\\boldsymbol{V}$ are unitary/orthonormal matrices, that is in case the matrices are real we have $\\boldsymbol{U}^T\\boldsymbol{U}=\\boldsymbol{U}\\boldsymbol{U}^T=\\boldsymbol{I}$ and $\\boldsymbol{V}^T\\boldsymbol{V}=\\boldsymbol{V}\\boldsymbol{V}^T=\\boldsymbol{I}$.\n", + "\n", + "\n", + "\n", + "## Introducing the Covariance and Correlation functions\n", + "\n", + "Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about\n", + "the definition of the covariance and the correlation function. These are quantities \n", + "\n", + "Suppose we have defined two vectors\n", + "$\\hat{x}$ and $\\hat{y}$ with $n$ elements each. The covariance matrix $\\boldsymbol{C}$ is defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", + " \\mathrm{cov}[\\boldsymbol{y},\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{y},\\boldsymbol{y}] \\\\\n", + " \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where for example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] =\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})(y_i- \\overline{y}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With this definition and recalling that the variance is defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathrm{var}[\\boldsymbol{x}]=\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we can rewrite the covariance matrix as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", + " \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] & \\mathrm{var}[\\boldsymbol{y}] \\\\\n", + " \\end{bmatrix}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The covariance takes values between zero and infinity and may thus\n", + "lead to problems with loss of numerical precision for particularly\n", + "large values. It is common to scale the covariance matrix by\n", + "introducing instead the correlation matrix defined via the so-called\n", + "correlation function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]=\\frac{\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}]}{\\sqrt{\\mathrm{var}[\\boldsymbol{x}] \\mathrm{var}[\\boldsymbol{y}]}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The correlation function is then given by values $\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]\n", + "\\in [-1,1]$. This avoids eventual problems with too large values. We\n", + "can then define the correlation matrix for the two vectors $\\boldsymbol{x}$\n", + "and $\\boldsymbol{y}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{K}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} 1 & \\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", + " \\mathrm{corr}[\\boldsymbol{y},\\boldsymbol{x}] & 1 \\\\\n", + " \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the above example this is the function we constructed using **pandas**.\n", + "\n", + "\n", + "\n", + "In our derivation of the various regression algorithms like **Ordinary Least Squares** or **Ridge regression**\n", + "we defined the design/feature matrix $\\boldsymbol{X}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\\begin{bmatrix}\n", + "x_{0,0} & x_{0,1} & x_{0,2}& \\dots & \\dots x_{0,p-1}\\\\\n", + "x_{1,0} & x_{1,1} & x_{1,2}& \\dots & \\dots x_{1,p-1}\\\\\n", + "x_{2,0} & x_{2,1} & x_{2,2}& \\dots & \\dots x_{2,p-1}\\\\\n", + "\\dots & \\dots & \\dots & \\dots \\dots & \\dots \\\\\n", + "x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \\dots & \\dots x_{n-2,p-1}\\\\\n", + "x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \\dots & \\dots x_{n-1,p-1}\\\\\n", + "\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$, with the predictors/features $p$ refering to the column numbers and the\n", + "entries $n$ being the row elements.\n", + "We can rewrite the design/feature matrix in terms of its column vectors as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\\begin{bmatrix} \\boldsymbol{x}_0 & \\boldsymbol{x}_1 & \\boldsymbol{x}_2 & \\dots & \\dots & \\boldsymbol{x}_{p-1}\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with a given vector" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{x}_i^T = \\begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \\dots & \\dots x_{n-1,i}\\end{bmatrix}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With these definitions, we can now rewrite our $2\\times 2$\n", + "correaltion/covariance matrix in terms of a moe general design/feature\n", + "matrix $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$. This leads to a $p\\times p$\n", + "covariance matrix for the vectors $\\boldsymbol{x}_i$ with $i=0,1,\\dots,p-1$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{C}[\\boldsymbol{x}] = \\begin{bmatrix}\n", + "\\mathrm{var}[\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{cov}[\\boldsymbol{x}_1,\\boldsymbol{x}_0] & \\mathrm{var}[\\boldsymbol{x}_1] & \\mathrm{cov}[\\boldsymbol{x}_1,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{cov}[\\boldsymbol{x}_1,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{cov}[\\boldsymbol{x}_2,\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_2,\\boldsymbol{x}_1] & \\mathrm{var}[\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{cov}[\\boldsymbol{x}_2,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", + "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", + "\\mathrm{cov}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_1] & \\mathrm{cov}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_{2}] & \\dots & \\dots & \\mathrm{var}[\\boldsymbol{x}_{p-1}]\\\\\n", + "\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and the correlation matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{K}[\\boldsymbol{x}] = \\begin{bmatrix}\n", + "1 & \\mathrm{corr}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] & \\mathrm{corr}[\\boldsymbol{x}_0,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{corr}[\\boldsymbol{x}_0,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{corr}[\\boldsymbol{x}_1,\\boldsymbol{x}_0] & 1 & \\mathrm{corr}[\\boldsymbol{x}_1,\\boldsymbol{x}_2] & \\dots & \\dots & \\mathrm{corr}[\\boldsymbol{x}_1,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\mathrm{corr}[\\boldsymbol{x}_2,\\boldsymbol{x}_0] & \\mathrm{corr}[\\boldsymbol{x}_2,\\boldsymbol{x}_1] & 1 & \\dots & \\dots & \\mathrm{corr}[\\boldsymbol{x}_2,\\boldsymbol{x}_{p-1}]\\\\\n", + "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", + "\\dots & \\dots & \\dots & \\dots & \\dots & \\dots \\\\\n", + "\\mathrm{corr}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_0] & \\mathrm{corr}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_1] & \\mathrm{corr}[\\boldsymbol{x}_{p-1},\\boldsymbol{x}_{2}] & \\dots & \\dots & 1\\\\\n", + "\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Numpy function **np.cov** calculates the covariance elements using\n", + "the factor $1/(n-1)$ instead of $1/n$ since it assumes we do not have\n", + "the exact mean values. The following simple function uses the\n", + "**np.vstack** function which takes each vector of dimension $1\\times n$\n", + "and produces a $2\\times n$ matrix $\\boldsymbol{W}$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{W} = \\begin{bmatrix} x_0 & y_0 \\\\\n", + " x_1 & y_1 \\\\\n", + " x_2 & y_2\\\\\n", + " \\dots & \\dots \\\\\n", + " x_{n-2} & y_{n-2}\\\\\n", + " x_{n-1} & y_{n-1} & \n", + " \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which in turn is converted into into the $2\\times 2$ covariance matrix\n", + "$\\boldsymbol{C}$ via the Numpy function **np.cov()**. We note that we can also calculate\n", + "the mean value of each set of samples $\\boldsymbol{x}$ etc using the Numpy\n", + "function **np.mean(x)**. We can also extract the eigenvalues of the\n", + "covariance matrix through the **np.linalg.eig()** function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Importing various packages\n", + "import numpy as np\n", + "n = 100\n", + "x = np.random.normal(size=n)\n", + "print(np.mean(x))\n", + "y = 4+3*x+np.random.normal(size=n)\n", + "print(np.mean(y))\n", + "W = np.vstack((x, y))\n", + "C = np.cov(W)\n", + "print(C)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The previous example can be converted into the correlation matrix by\n", + "simply scaling the matrix elements with the variances. We should also\n", + "subtract the mean values for each column. This leads to the following\n", + "code which sets up the correlations matrix for the previous example in\n", + "a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the $2\\times 2$ correlation matrix (since we have only two vectors)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "n = 100\n", + "# define two vectors \n", + "x = np.random.random(size=n)\n", + "y = 4+3*x+np.random.normal(size=n)\n", + "#scaling the x and y vectors \n", + "x = x - np.mean(x)\n", + "y = y - np.mean(y)\n", + "variance_x = np.sum(x@x)/n\n", + "variance_y = np.sum(y@y)/n\n", + "print(variance_x)\n", + "print(variance_y)\n", + "cov_xy = np.sum(x@y)/n\n", + "cov_xx = np.sum(x@x)/n\n", + "cov_yy = np.sum(y@y)/n\n", + "C = np.zeros((2,2))\n", + "C[0,0]= cov_xx/variance_x\n", + "C[1,1]= cov_yy/variance_y\n", + "C[0,1]= cov_xy/np.sqrt(variance_y*variance_x)\n", + "C[1,0]= C[0,1]\n", + "print(C)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see that the matrix elements along the diagonal are one as they\n", + "should be and that the matrix is symmetric. Furthermore, diagonalizing\n", + "this matrix we easily see that it is a positive definite matrix.\n", + "\n", + "The above procedure with **numpy** can be made more compact if we use **pandas**.\n", + "\n", + "\n", + "We whow here how we can set up the correlation matrix using **pandas**, as done in this simple code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "n = 10\n", + "x = np.random.normal(size=n)\n", + "x = x - np.mean(x)\n", + "y = 4+3*x+np.random.normal(size=n)\n", + "y = y - np.mean(y)\n", + "X = (np.vstack((x, y))).T\n", + "print(X)\n", + "Xpd = pd.DataFrame(X)\n", + "print(Xpd)\n", + "correlation_matrix = Xpd.corr()\n", + "print(correlation_matrix)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We expand this model to the Franke function discussed above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Common imports\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "\n", + "def FrankeFunction(x,y):\n", + "\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n", + "\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n", + "\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n", + "\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n", + "\treturn term1 + term2 + term3 + term4\n", + "\n", + "\n", + "def create_X(x, y, n ):\n", + "\tif len(x.shape) > 1:\n", + "\t\tx = np.ravel(x)\n", + "\t\ty = np.ravel(y)\n", + "\n", + "\tN = len(x)\n", + "\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n", + "\tX = np.ones((N,l))\n", + "\n", + "\tfor i in range(1,n+1):\n", + "\t\tq = int((i)*(i+1)/2)\n", + "\t\tfor k in range(i+1):\n", + "\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n", + "\n", + "\treturn X\n", + "\n", + "\n", + "# Making meshgrid of datapoints and compute Franke's function\n", + "n = 4\n", + "N = 100\n", + "x = np.sort(np.random.uniform(0, 1, N))\n", + "y = np.sort(np.random.uniform(0, 1, N))\n", + "z = FrankeFunction(x, y)\n", + "X = create_X(x, y, n=n) \n", + "\n", + "Xpd = pd.DataFrame(X)\n", + "# subtract the mean values and set up the covariance matrix\n", + "Xpd = Xpd - Xpd.mean()\n", + "covariance_matrix = Xpd.cov()\n", + "print(covariance_matrix)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We note here that the covariance is zero for the first rows and\n", + "columns since all matrix elements in the design matrix were set to one\n", + "(we are fitting the function in terms of a polynomial of degree $n$).\n", + "\n", + "This means that the variance for these elements will be zero and will\n", + "cause problems when we set up the correlation matrix. We can simply\n", + "drop these elements and construct a correlation\n", + "matrix without these elements. \n", + "\n", + "\n", + "\n", + "\n", + "We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix $\\boldsymbol{X}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{C}[\\boldsymbol{x}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}= \\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}].\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To see this let us simply look at a design matrix $\\boldsymbol{X}\\in {\\mathbb{R}}^{2\\times 2}$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\\begin{bmatrix}\n", + "x_{00} & x_{01}\\\\\n", + "x_{10} & x_{11}\\\\\n", + "\\end{bmatrix}=\\begin{bmatrix}\n", + "\\boldsymbol{x}_{0} & \\boldsymbol{x}_{1}\\\\\n", + "\\end{bmatrix}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we then compute the expectation value" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}=\\begin{bmatrix}\n", + "x_{00}^2+x_{01}^2 & x_{00}x_{10}+x_{01}x_{11}\\\\\n", + "x_{10}x_{00}+x_{11}x_{01} & x_{10}^2+x_{11}^2\\\\\n", + "\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which is just" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]=\\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] \\\\\n", + " \\mathrm{cov}[\\boldsymbol{x}_1,\\boldsymbol{x}_0] & \\mathrm{var}[\\boldsymbol{x}_1] \\\\\n", + " \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we wrote $$\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]$$ to indicate that this the covariance of the vectors $\\boldsymbol{x}$ of the design/feature matrix $\\boldsymbol{X}$.\n", + "\n", + "It is easy to generalize this to a matrix $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$.\n", + "\n", + "\n", + "## Linking with SVD" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/doc/BookChapters/chapter4.dlog b/doc/BookChapters/chapter4.dlog new file mode 100644 index 000000000..88a8be5bd --- /dev/null +++ b/doc/BookChapters/chapter4.dlog @@ -0,0 +1,7 @@ +*** error: file has a mako construction ${\bf \bm{J}' + but seemingly no definition in <%...%>' + (it is not a command-line given mako variable either). + However, if this is a variable in a Makefile or Bash script + run with --no_mako - and you cannot use mako and Makefile or Bash variables + in the same document! + diff --git a/doc/BookChapters/chapter4.do.txt b/doc/BookChapters/chapter4.do.txt new file mode 100644 index 000000000..bd2cbcebe --- /dev/null +++ b/doc/BookChapters/chapter4.do.txt @@ -0,0 +1,1888 @@ +======= Logistic Regression ======= + + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureSeptember18.mp4?vrtx=view-as-webpage" + + +===== Logistic Regression ===== + +In linear regression our main interest was centered on learning the +coefficients of a functional fit (say a polynomial) in order to be +able to predict the response of a continuous variable on some unseen +data. The fit to the continuous variable $y_i$ is based on some +independent variables $\hat{x}_i$. Linear regression resulted in +analytical expressions for standard ordinary Least Squares or Ridge +regression (in terms of matrices to invert) for several quantities, +ranging from the variance and thereby the confidence intervals of the +parameters $\hat{\beta}$ to the mean squared error. If we can invert +the product of the design matrices, linear regression gives then a +simple recipe for fitting our data. + + +Classification problems, however, are concerned with outcomes taking +the form of discrete variables (i.e. categories). We may for example, +on the basis of DNA sequencing for a number of patients, like to find +out which mutations are important for a certain disease; or based on +scans of various patients' brains, figure out if there is a tumor or +not; or given a specific physical system, we'd like to identify its +state, say whether it is an ordered or disordered system (typical +situation in solid state physics); or classify the status of a +patient, whether she/he has a stroke or not and many other similar +situations. + +The most common situation we encounter when we apply logistic +regression is that of two possible outcomes, normally denoted as a +binary outcome, true or false, positive or negative, success or +failure etc. + + +Logistic regression will also serve as our stepping stone towards +neural network algorithms and supervised deep learning. For logistic +learning, the minimization of the cost function leads to a non-linear +equation in the parameters $\hat{\beta}$. The optimization of the +problem calls therefore for minimization algorithms. This forms the +bottle neck of all machine learning algorithms, namely how to find +reliable minima of a multi-variable function. This leads us to the +family of gradient descent methods. The latter are the working horses +of basically all modern machine learning algorithms. + +We note also that many of the topics discussed here on logistic +regression are also commonly used in modern supervised Deep Learning +models, as we will see later. + + + +===== Basics ===== + +We consider the case where the dependent variables, also called the +responses or the outcomes, $y_i$ are discrete and only take values +from $k=0,\dots,K-1$ (i.e. $K$ classes). + +The goal is to predict the +output classes from the design matrix $\hat{X}\in\mathbb{R}^{n\times p}$ +made of $n$ samples, each of which carries $p$ features or predictors. The +primary goal is to identify the classes to which new unseen samples +belong. + +Let us specialize to the case of two classes only, with outputs +$y_i=0$ and $y_i=1$. Our outcomes could represent the status of a +credit card user that could default or not on her/his credit card +debt. That is + + +!bt +\[ +y_i = \begin{bmatrix} 0 & \mathrm{no}\\ 1 & \mathrm{yes} \end{bmatrix}. +\] +!et + + + + +Before moving to the logistic model, let us try to use our linear +regression model to classify these two outcomes. We could for example +fit a linear model to the default case if $y_i > 0.5$ and the no +default case $y_i \leq 0.5$. + +We would then have our +weighted linear combination, namely +!bt +\begin{equation} +\hat{y} = \hat{X}^T\hat{\beta} + \hat{\epsilon}, +\end{equation} +!et +where $\hat{y}$ is a vector representing the possible outcomes, $\hat{X}$ is our +$n\times p$ design matrix and $\hat{\beta}$ represents our estimators/predictors. + + +The main problem with our function is that it takes values on the +entire real axis. In the case of logistic regression, however, the +labels $y_i$ are discrete variables. A typical example is the credit +card data discussed below here, where we can set the state of +defaulting the debt to $y_i=1$ and not to $y_i=0$ for one the persons +in the data set (see the full example below). + +One simple way to get a discrete output is to have sign +functions that map the output of a linear regressor to values $\{0,1\}$, +$f(s_i)=sign(s_i)=1$ if $s_i\ge 0$ and 0 if otherwise. +We will encounter this model in our first demonstration of neural networks. Historically it is called the ``perceptron" model in the machine learning +literature. This model is extremely simple. However, in many cases it is more +favorable to use a ``soft" classifier that outputs +the probability of a given category. This leads us to the logistic function. + + +The following example on data for coronary heart disease (CHD) as function of age may serve as an illustration. In the code here we read and plot whether a person has had CHD (output = 1) or not (output = 0). This ouput is plotted the person's against age. Clearly, the figure shows that attempting to make a standard linear regression fit may not be very meaningful. + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.model_selection import train_test_split +from sklearn.utils import resample +from sklearn.metrics import mean_squared_error +from IPython.display import display +from pylab import plt, mpl +plt.style.use('seaborn') +mpl.rcParams['font.family'] = 'serif' + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("chddata.csv"),'r') + +# Read the chd data as csv file and organize the data into arrays with age group, age, and chd +chd = pd.read_csv(infile, names=('ID', 'Age', 'Agegroup', 'CHD')) +chd.columns = ['ID', 'Age', 'Agegroup', 'CHD'] +output = chd['CHD'] +age = chd['Age'] +agegroup = chd['Agegroup'] +numberID = chd['ID'] +display(chd) + +plt.scatter(age, output, marker='o') +plt.axis([18,70.0,-0.1, 1.2]) +plt.xlabel(r'Age') +plt.ylabel(r'CHD') +plt.title(r'Age distribution and Coronary heart disease') +plt.show() +!ec + + +What we could attempt however is to plot the mean value for each group. + +!bc pycod +agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800]) +group = np.array([1, 2, 3, 4, 5, 6, 7, 8]) +plt.plot(group, agegroupmean, "r-") +plt.axis([0,9,0, 1.0]) +plt.xlabel(r'Age group') +plt.ylabel(r'CHD mean values') +plt.title(r'Mean values for each age group') +plt.show() +!ec + +We are now trying to find a function $f(y\vert x)$, that is a function which gives us an expected value for the output $y$ with a given input $x$. +In standard linear regression with a linear dependence on $x$, we would write this in terms of our model +!bt +\[ +f(y_i\vert x_i)=\beta_0+\beta_1 x_i. +\] +!et + +This expression implies however that $f(y_i\vert x_i)$ could take any +value from minus infinity to plus infinity. If we however let +$f(y\vert y)$ be represented by the mean value, the above example +shows us that we can constrain the function to take values between +zero and one, that is we have $0 \le f(y_i\vert x_i) \le 1$. Looking +at our last curve we see also that it has an S-shaped form. This leads +us to a very popular model for the function $f$, namely the so-called +Sigmoid function or logistic model. We will consider this function as +representing the probability for finding a value of $y_i$ with a given +$x_i$. + + +===== The logistic function ===== + +Another widely studied model, is the so-called +perceptron model, which is an example of a ``hard classification'' model. We +will encounter this model when we discuss neural networks as +well. Each datapoint is deterministically assigned to a category (i.e +$y_i=0$ or $y_i=1$). In many cases, and the coronary heart disease data forms one of many such examples, it is favorable to have a ``soft'' +classifier that outputs the probability of a given category rather +than a single value. For example, given $x_i$, the classifier +outputs the probability of being in a category $k$. Logistic regression +is the most common example of a so-called soft classifier. In logistic +regression, the probability that a data point $x_i$ +belongs to a category $y_i=\{0,1\}$ is given by the so-called logit function (or Sigmoid) which is meant to represent the likelihood for a given event, +!bt +\[ +p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}. +\] +!et +Note that $1-p(t)= p(-t)$. + +===== Examples of likelihood functions used in logistic regression and nueral networks ===== + + +The following code plots the logistic function, the step function and other functions we will encounter from here and on. + + +!bc pycod +"""The sigmoid function (or the logistic curve) is a +function that takes any real number, z, and outputs a number (0,1). +It is useful in neural networks for assigning weights on a relative scale. +The value z is the weighted sum of parameters involved in the learning algorithm.""" + +import numpy +import matplotlib.pyplot as plt +import math as mt + +z = numpy.arange(-5, 5, .1) +sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z))) +sigma = sigma_fn(z) + +fig = plt.figure() +ax = fig.add_subplot(111) +ax.plot(z, sigma) +ax.set_ylim([-0.1, 1.1]) +ax.set_xlim([-5,5]) +ax.grid(True) +ax.set_xlabel('z') +ax.set_title('sigmoid function') + +plt.show() + +"""Step Function""" +z = numpy.arange(-5, 5, .02) +step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0) +step = step_fn(z) + +fig = plt.figure() +ax = fig.add_subplot(111) +ax.plot(z, step) +ax.set_ylim([-0.5, 1.5]) +ax.set_xlim([-5,5]) +ax.grid(True) +ax.set_xlabel('z') +ax.set_title('step function') + +plt.show() + +"""tanh Function""" +z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1) +t = numpy.tanh(z) + +fig = plt.figure() +ax = fig.add_subplot(111) +ax.plot(z, t) +ax.set_ylim([-1.0, 1.0]) +ax.set_xlim([-2*mt.pi,2*mt.pi]) +ax.grid(True) +ax.set_xlabel('z') +ax.set_title('tanh function') + +plt.show() +!ec + + + + + + + +We assume now that we have two classes with $y_i$ either $0$ or $1$. Furthermore we assume also that we have only two parameters $\beta$ in our fitting of the Sigmoid function, that is we define probabilities +!bt +\begin{align*} +p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\ +p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}), +\end{align*} +!et +where $\hat{\beta}$ are the weights we wish to extract from data, in our case $\beta_0$ and $\beta_1$. + +Note that we used +!bt +\[ +p(y_i=0\vert x_i, \hat{\beta}) = 1-p(y_i=1\vert x_i, \hat{\beta}). +\] +!et + + +In order to define the total likelihood for all possible outcomes from a +dataset $\mathcal{D}=\{(y_i,x_i)\}$, with the binary labels +$y_i\in\{0,1\}$ and where the data points are drawn independently, we use the so-called "Maximum Likelihood Estimation":"https://en.wikipedia.org/wiki/Maximum_likelihood_estimation" (MLE) principle. +We aim thus at maximizing +the probability of seeing the observed data. We can then approximate the +likelihood in terms of the product of the individual probabilities of a specific outcome $y_i$, that is +!bt +\begin{align*} +P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\ +\end{align*} +!et +from which we obtain the log-likelihood and our _cost/loss_ function +!bt +\[ +\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left( y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right]\right). +\] +!et + + +Reordering the logarithms, we can rewrite the _cost/loss_ function as +!bt +\[ +\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right). +\] +!et + +The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\beta$. +Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that +!bt +\[ +\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right). +\] +!et +This equation is known in statistics as the _cross entropy_. Finally, we note that just as in linear regression, +in practice we often supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression. + + +The cross entropy is a convex function of the weights $\hat{\beta}$ and, +therefore, any local minimizer is a global minimizer. + + +Minimizing this +cost function with respect to the two parameters $\beta_0$ and $\beta_1$ we obtain + +!bt +\[ +\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right), +\] +!et +and +!bt +\[ +\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right). +\] +!et + + +Let us now define a vector $\hat{y}$ with $n$ elements $y_i$, an +$n\times p$ matrix $\hat{X}$ which contains the $x_i$ values and a +vector $\hat{p}$ of fitted probabilities $p(y_i\vert x_i,\hat{\beta})$. We can rewrite in a more compact form the first +derivative of cost function as + +!bt +\[ +\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right). +\] +!et + +If we in addition define a diagonal matrix $\hat{W}$ with elements +$p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta})$, we can obtain a compact expression of the second derivative as + +!bt +\[ +\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}. +\] +!et + + +Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with $p$ predictors +!bt +\[ +\log{ \frac{p(\hat{\beta}\hat{x})}{1-p(\hat{\beta}\hat{x})}} = \beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p. +\] +!et +Here we defined $\hat{x}=[1,x_1,x_2,\dots,x_p]$ and $\hat{\beta}=[\beta_0, \beta_1, \dots, \beta_p]$ leading to +!bt +\[ +p(\hat{\beta}\hat{x})=\frac{ \exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}{1+\exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}. +\] +!et + + +Till now we have mainly focused on two classes, the so-called binary +system. Suppose we wish to extend to $K$ classes. Let us for the sake +of simplicity assume we have only two predictors. We have then following model + +!bt +\[ +\log{\frac{p(C=1\vert x)}{p(K\vert x)}} = \beta_{10}+\beta_{11}x_1, +\] +!et +and +!bt +\[ +\log{\frac{p(C=2\vert x)}{p(K\vert x)}} = \beta_{20}+\beta_{21}x_1, +\] +!et +and so on till the class $C=K-1$ class +!bt +\[ +\log{\frac{p(C=K-1\vert x)}{p(K\vert x)}} = \beta_{(K-1)0}+\beta_{(K-1)1}x_1, +\] +!et + +and the model is specified in term of $K-1$ so-called log-odds or +_logit_ transformations. + + + +In our discussion of neural networks we will encounter the above again +in terms of a slightly modified function, the so-called _Softmax_ function. + +The softmax function is used in various multiclass classification +methods, such as multinomial logistic regression (also known as +softmax regression), multiclass linear discriminant analysis, naive +Bayes classifiers, and artificial neural networks. Specifically, in +multinomial logistic regression and linear discriminant analysis, the +input to the function is the result of $K$ distinct linear functions, +and the predicted probability for the $k$-th class given a sample +vector $\hat{x}$ and a weighting vector $\hat{\beta}$ is (with two +predictors): + +!bt +\[ +p(C=k\vert \mathbf {x} )=\frac{\exp{(\beta_{k0}+\beta_{k1}x_1)}}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}}. +\] +!et +It is easy to extend to more predictors. The final class is +!bt +\[ +p(C=K\vert \mathbf {x} )=\frac{1}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}}, +\] +!et + +and they sum to one. Our earlier discussions were all specialized to +the case with two classes only. It is easy to see from the above that +what we derived earlier is compatible with these equations. + +To find the optimal parameters we would typically use a gradient +descent method. Newton's method and gradient descent methods are +discussed in the material on "optimization +methods":"https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html". + +===== Wisconsin Cancer Data ===== + +We show here how we can use a simple regression case on the breast +cancer data using Logistic regression as our algorithm for +classification. + + +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +# Logistic Regression +logreg = LogisticRegression(solver='lbfgs') +logreg.fit(X_train, y_train) +print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test))) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Logistic Regression +logreg.fit(X_train_scaled, y_train) +print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) +!ec + + + +In addition to the above scores, we could also study the covariance (and the correlation matrix). +We use _Pandas_ to compute the correlation matrix. +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression +cancer = load_breast_cancer() +import pandas as pd +# Making a data frame +cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names) + +fig, axes = plt.subplots(15,2,figsize=(10,20)) +malignant = cancer.data[cancer.target == 0] +benign = cancer.data[cancer.target == 1] +ax = axes.ravel() + +for i in range(30): + _, bins = np.histogram(cancer.data[:,i], bins =50) + ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5) + ax[i].hist(benign[:,i], bins = bins, alpha = 0.5) + ax[i].set_title(cancer.feature_names[i]) + ax[i].set_yticks(()) +ax[0].set_xlabel("Feature magnitude") +ax[0].set_ylabel("Frequency") +ax[0].legend(["Malignant", "Benign"], loc ="best") +fig.tight_layout() +plt.show() + +import seaborn as sns +correlation_matrix = cancerpd.corr().round(1) +# use the heatmap function from seaborn to plot the correlation matrix +# annot = True to print the values inside the square +plt.figure(figsize=(15,8)) +sns.heatmap(data=correlation_matrix, annot=True) +plt.show() + + +!ec + + +In the above example we note two things. In the first plot we display +the overlap of benign and malignant tumors as functions of the various +features in the Wisconsing breast cancer data set. We see that for +some of the features we can distinguish clearly the benign and +malignant cases while for other features we cannot. This can point to +us which features may be of greater interest when we wish to classify +a benign or not benign tumour. + +In the second figure we have computed the so-called correlation +matrix, which in our case with thirty features becomes a $30\times 30$ +matrix. + +We constructed this matrix using _pandas_ via the statements +!bc pycod +cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names) +!ec +and then +!bc pycod +correlation_matrix = cancerpd.corr().round(1) +!ec + +Diagonalizing this matrix we can in turn say something about which +features are of relevance and which are not. This leads us to +the classical Principal Component Analysis (PCA) theorem with +applications. This will be discussed later this semester ("week 43":"https://compphysics.github.io/MachineLearning/doc/pub/week43/html/week43-bs.html"). + + + + +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +# Logistic Regression +logreg = LogisticRegression(solver='lbfgs') +logreg.fit(X_train, y_train) +print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test))) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Logistic Regression +logreg.fit(X_train_scaled, y_train) +print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) + + +from sklearn.preprocessing import LabelEncoder +from sklearn.model_selection import cross_validate +#Cross validation +accuracy = cross_validate(logreg,X_test_scaled,y_test,cv=10)['test_score'] +print(accuracy) +print("Test set accuracy with Logistic Regression and scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) + + +import scikitplot as skplt +y_pred = logreg.predict(X_test_scaled) +skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True) +plt.show() +y_probas = logreg.predict_proba(X_test_scaled) +skplt.metrics.plot_roc(y_test, y_probas) +plt.show() +skplt.metrics.plot_cumulative_gain(y_test, y_probas) +plt.show() + +!ec + + + + +===== Optimization, the central part of any Machine Learning algortithm ===== + +Almost every problem in machine learning and data science starts with +a dataset $X$, a model $g(\beta)$, which is a function of the +parameters $\beta$ and a cost function $C(X, g(\beta))$ that allows +us to judge how well the model $g(\beta)$ explains the observations +$X$. The model is fit by finding the values of $\beta$ that minimize +the cost function. Ideally we would be able to solve for $\beta$ +analytically, however this is not possible in general and we must use +some approximative/numerical method to compute the minimum. + + + +===== Revisiting our Logistic Regression case ===== + +In our discussion on Logistic Regression we studied the +case of +two classes, with $y_i$ either +$0$ or $1$. Furthermore we assumed also that we have only two +parameters $\beta$ in our fitting, that is we +defined probabilities + +!bt +\begin{align*} +p(y_i=1|x_i,\bm{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\ +p(y_i=0|x_i,\bm{\beta}) &= 1 - p(y_i=1|x_i,\bm{\beta}), +\end{align*} +!et +where $\bm{\beta}$ are the weights we wish to extract from data, in our case $\beta_0$ and $\beta_1$. + + +===== The equations to solve ===== + +Our compact equations used a definition of a vector $\bm{y}$ with $n$ +elements $y_i$, an $n\times p$ matrix $\bm{X}$ which contains the +$x_i$ values and a vector $\bm{p}$ of fitted probabilities +$p(y_i\vert x_i,\bm{\beta})$. We rewrote in a more compact form +the first derivative of the cost function as + +!bt +\[ +\frac{\partial \mathcal{C}(\bm{\beta})}{\partial \bm{\beta}} = -\bm{X}^T\left(\bm{y}-\bm{p}\right). +\] +!et + +If we in addition define a diagonal matrix $\bm{W}$ with elements +$p(y_i\vert x_i,\bm{\beta})(1-p(y_i\vert x_i,\bm{\beta})$, we can obtain a compact expression of the second derivative as + +!bt +\[ +\frac{\partial^2 \mathcal{C}(\bm{\beta})}{\partial \bm{\beta}\partial \bm{\beta}^T} = \bm{X}^T\bm{W}\bm{X}. +\] +!et +This defines what is called the Hessian matrix. + + +===== Solving using Newton-Raphson's method ===== + +If we can set up these equations, Newton-Raphson's iterative method is normally the method of choice. It requires however that we can compute in an efficient way the matrices that define the first and second derivatives. + +Our iterative scheme is then given by + +!bt +\[ +\bm{\beta}^{\mathrm{new}} = \bm{\beta}^{\mathrm{old}}-\left(\frac{\partial^2 \mathcal{C}(\bm{\beta})}{\partial \bm{\beta}\partial \bm{\beta}^T}\right)^{-1}_{\bm{\beta}^{\mathrm{old}}}\times \left(\frac{\partial \mathcal{C}(\bm{\beta})}{\partial \bm{\beta}}\right)_{\bm{\beta}^{\mathrm{old}}}, +\] +!et +or in matrix form as + +!bt +\[ +\bm{\beta}^{\mathrm{new}} = \bm{\beta}^{\mathrm{old}}-\left(\bm{X}^T\bm{W}\bm{X} \right)^{-1}\times \left(-\bm{X}^T(\bm{y}-\bm{p}) \right)_{\bm{\beta}^{\mathrm{old}}}. +\] +!et +The right-hand side is computed with the old values of $\beta$. + +If we can compute these matrices, in particular the Hessian, the above is often the easiest method to implement. + + + +===== Brief reminder on Newton-Raphson's method ===== + +Let us quickly remind ourselves how we derive the above method. + +Perhaps the most celebrated of all one-dimensional root-finding +routines is Newton's method, also called the Newton-Raphson +method. This method requires the evaluation of both the +function $f$ and its derivative $f'$ at arbitrary points. +If you can only calculate the derivative +numerically and/or your function is not of the smooth type, we +normally discourage the use of this method. + + +===== The equations ===== + +The Newton-Raphson formula consists geometrically of extending the +tangent line at a current point until it crosses zero, then setting +the next guess to the abscissa of that zero-crossing. The mathematics +behind this method is rather simple. Employing a Taylor expansion for +$x$ sufficiently close to the solution $s$, we have + + +!bt +\[ + f(s)=0=f(x)+(s-x)f'(x)+\frac{(s-x)^2}{2}f''(x) +\dots. + \label{eq:taylornr} +\] +!et + +For small enough values of the function and for well-behaved +functions, the terms beyond linear are unimportant, hence we obtain + + +!bt +\[ + f(x)+(s-x)f'(x)\approx 0, +\] +!et +yielding +!bt +\[ + s\approx x-\frac{f(x)}{f'(x)}. +\] +!et + +Having in mind an iterative procedure, it is natural to start iterating with +!bt +\[ + x_{n+1}=x_n-\frac{f(x_n)}{f'(x_n)}. +\] +!et + + +===== Simple geometric interpretation ===== + +The above is Newton-Raphson's method. It has a simple geometric +interpretation, namely $x_{n+1}$ is the point where the tangent from +$(x_n,f(x_n))$ crosses the $x$-axis. Close to the solution, +Newton-Raphson converges fast to the desired result. However, if we +are far from a root, where the higher-order terms in the series are +important, the Newton-Raphson formula can give grossly inaccurate +results. For instance, the initial guess for the root might be so far +from the true root as to let the search interval include a local +maximum or minimum of the function. If an iteration places a trial +guess near such a local extremum, so that the first derivative nearly +vanishes, then Newton-Raphson may fail totally + + + +===== Extending to more than one variable ===== + +Newton's method can be generalized to systems of several non-linear equations +and variables. Consider the case with two equations +!bt +\[ + \begin{array}{cc} f_1(x_1,x_2) &=0\\ + f_2(x_1,x_2) &=0,\end{array} +\] +!et +which we Taylor expand to obtain + +!bt +\[ + \begin{array}{cc} 0=f_1(x_1+h_1,x_2+h_2)=&f_1(x_1,x_2)+h_1 + \partial f_1/\partial x_1+h_2 + \partial f_1/\partial x_2+\dots\\ + 0=f_2(x_1+h_1,x_2+h_2)=&f_2(x_1,x_2)+h_1 + \partial f_2/\partial x_1+h_2 + \partial f_2/\partial x_2+\dots + \end{array}. +\] +!et +Defining the Jacobian matrix ${\bf \bm{J}}$ we have +!bt +\[ + {\bf \bm{J}}=\left( \begin{array}{cc} + \partial f_1/\partial x_1 & \partial f_1/\partial x_2 \\ + \partial f_2/\partial x_1 &\partial f_2/\partial x_2 + \end{array} \right), +\] +!et +we can rephrase Newton's method as +!bt +\[ +\left(\begin{array}{c} x_1^{n+1} \\ x_2^{n+1} \end{array} \right)= +\left(\begin{array}{c} x_1^{n} \\ x_2^{n} \end{array} \right)+ +\left(\begin{array}{c} h_1^{n} \\ h_2^{n} \end{array} \right), +\] +!et +where we have defined +!bt +\[ + \left(\begin{array}{c} h_1^{n} \\ h_2^{n} \end{array} \right)= + -{\bf \bm{J}}^{-1} + \left(\begin{array}{c} f_1(x_1^{n},x_2^{n}) \\ f_2(x_1^{n},x_2^{n}) \end{array} \right). +\] +!et +We need thus to compute the inverse of the Jacobian matrix and it +is to understand that difficulties may +arise in case ${\bf \bm{J}}$ is nearly singular. + +It is rather straightforward to extend the above scheme to systems of +more than two non-linear equations. In our case, the Jacobian matrix is given by the Hessian that represents the second derivative of cost function. + + + + +===== Steepest descent ===== + +The basic idea of gradient descent is +that a function $F(\mathbf{x})$, +$\mathbf{x} \equiv (x_1,\cdots,x_n)$, decreases fastest if one goes from $\bf {x}$ in the +direction of the negative gradient $-\nabla F(\mathbf{x})$. + +It can be shown that if +!bt +\[ +\mathbf{x}_{k+1} = \mathbf{x}_k - \gamma_k \nabla F(\mathbf{x}_k), +\] +!et +with $\gamma_k > 0$. + +For $\gamma_k$ small enough, then $F(\mathbf{x}_{k+1}) \leq +F(\mathbf{x}_k)$. This means that for a sufficiently small $\gamma_k$ +we are always moving towards smaller function values, i.e a minimum. + + +===== More on Steepest descent ===== + +The previous observation is the basis of the method of steepest +descent, which is also referred to as just gradient descent (GD). One +starts with an initial guess $\mathbf{x}_0$ for a minimum of $F$ and +computes new approximations according to + +!bt +\[ +\mathbf{x}_{k+1} = \mathbf{x}_k - \gamma_k \nabla F(\mathbf{x}_k), \ \ k \geq 0. +\] +!et + +The parameter $\gamma_k$ is often referred to as the step length or +the learning rate within the context of Machine Learning. + + +===== The ideal ===== + +Ideally the sequence $\{\mathbf{x}_k \}_{k=0}$ converges to a global +minimum of the function $F$. In general we do not know if we are in a +global or local minimum. In the special case when $F$ is a convex +function, all local minima are also global minima, so in this case +gradient descent can converge to the global solution. The advantage of +this scheme is that it is conceptually simple and straightforward to +implement. However the method in this form has some severe +limitations: + +In machine learing we are often faced with non-convex high dimensional +cost functions with many local minima. Since GD is deterministic we +will get stuck in a local minimum, if the method converges, unless we +have a very good intial guess. This also implies that the scheme is +sensitive to the chosen initial condition. + +Note that the gradient is a function of $\mathbf{x} = +(x_1,\cdots,x_n)$ which makes it expensive to compute numerically. + + + +===== The sensitiveness of the gradient descent ===== + +The gradient descent method +is sensitive to the choice of learning rate $\gamma_k$. This is due +to the fact that we are only guaranteed that $F(\mathbf{x}_{k+1}) \leq +F(\mathbf{x}_k)$ for sufficiently small $\gamma_k$. The problem is to +determine an optimal learning rate. If the learning rate is chosen too +small the method will take a long time to converge and if it is too +large we can experience erratic behavior. + +Many of these shortcomings can be alleviated by introducing +randomness. One such method is that of Stochastic Gradient Descent +(SGD), see below. + + + +===== Convex functions ===== + +Ideally we want our cost/loss function to be convex(concave). + +First we give the definition of a convex set: A set $C$ in +$\mathbb{R}^n$ is said to be convex if, for all $x$ and $y$ in $C$ and +all $t \in (0,1)$ , the point $(1 − t)x + ty$ also belongs to +C. Geometrically this means that every point on the line segment +connecting $x$ and $y$ is in $C$ as discussed below. + +The convex subsets of $\mathbb{R}$ are the intervals of +$\mathbb{R}$. Examples of convex sets of $\mathbb{R}^2$ are the +regular polygons (triangles, rectangles, pentagons, etc...). + + +===== Convex function ===== + +_Convex function_: Let $X \subset \mathbb{R}^n$ be a convex set. Assume that the function $f: X \rightarrow \mathbb{R}$ is continuous, then $f$ is said to be convex if $$f(tx_1 + (1-t)x_2) \leq tf(x_1) + (1-t)f(x_2) $$ for all $x_1, x_2 \in X$ and for all $t \in [0,1]$. If $\leq$ is replaced with a strict inequaltiy in the definition, we demand $x_1 \neq x_2$ and $t\in(0,1)$ then $f$ is said to be strictly convex. For a single variable function, convexity means that if you draw a straight line connecting $f(x_1)$ and $f(x_2)$, the value of the function on the interval $[x_1,x_2]$ is always below the line as illustrated below. + + +===== Conditions on convex functions ===== + +In the following we state first and second-order conditions which +ensures convexity of a function $f$. We write $D_f$ to denote the +domain of $f$, i.e the subset of $R^n$ where $f$ is defined. For more +details and proofs we refer to: "S. Boyd and L. Vandenberghe. Convex Optimization. Cambridge University Press":"http://stanford.edu/boyd/cvxbook/, 2004". + +!bblock First order condition +Suppose $f$ is differentiable (i.e $\nabla f(x)$ is well defined for +all $x$ in the domain of $f$). Then $f$ is convex if and only if $D_f$ +is a convex set and $$f(y) \geq f(x) + \nabla f(x)^T (y-x) $$ holds +for all $x,y \in D_f$. This condition means that for a convex function +the first order Taylor expansion (right hand side above) at any point +a global under estimator of the function. To convince yourself you can +make a drawing of $f(x) = x^2+1$ and draw the tangent line to $f(x)$ and +note that it is always below the graph. +!eblock + +!bblock Second order condition +Assume that $f$ is twice +differentiable, i.e the Hessian matrix exists at each point in +$D_f$. Then $f$ is convex if and only if $D_f$ is a convex set and its +Hessian is positive semi-definite for all $x\in D_f$. For a +single-variable function this reduces to $f''(x) \geq 0$. Geometrically this means that $f$ has nonnegative curvature +everywhere. +!eblock + +This condition is particularly useful since it gives us an procedure for determining if the function under consideration is convex, apart from using the definition. + + +===== More on convex functions ===== + +The next result is of great importance to us and the reason why we are +going on about convex functions. In machine learning we frequently +have to minimize a loss/cost function in order to find the best +parameters for the model we are considering. + +Ideally we want the +global minimum (for high-dimensional models it is hard to know +if we have local or global minimum). However, if the cost/loss function +is convex the following result provides invaluable information: + +!bblock Any minimum is global for convex functions +Consider the problem of finding $x \in \mathbb{R}^n$ such that $f(x)$ +is minimal, where $f$ is convex and differentiable. Then, any point +$x^*$ that satisfies $\nabla f(x^*) = 0$ is a global minimum. +!eblock + +This result means that if we know that the cost/loss function is convex and we are able to find a minimum, we are guaranteed that it is a global minimum. + + +===== Some simple problems ===== + +o Show that $f(x)=x^2$ is convex for $x \in \mathbb{R}$ using the definition of convexity. Hint: If you re-write the definition, $f$ is convex if the following holds for all $x,y \in D_f$ and any $\lambda \in [0,1]$ $\lambda f(x)+(1-\lambda)f(y)-f(\lambda x + (1-\lambda) y ) \geq 0$. + +o Using the second order condition show that the following functions are convex on the specified domain. + * $f(x) = e^x$ is convex for $x \in \mathbb{R}$. + * $g(x) = -\ln(x)$ is convex for $x \in (0,\infty)$. +o Let $f(x) = x^2$ and $g(x) = e^x$. Show that $f(g(x))$ and $g(f(x))$ is convex for $x \in \mathbb{R}$. Also show that if $f(x)$ is any convex function than $h(x) = e^{f(x)}$ is convex. + +o A norm is any function that satisfy the following properties + * $f(\alpha x) = |\alpha| f(x)$ for all $\alpha \in \mathbb{R}$. + * $f(x+y) \leq f(x) + f(y)$ + * $f(x) \leq 0$ for all $x \in \mathbb{R}^n$ with equality if and only if $x = 0$ + +Using the definition of convexity, try to show that a function satisfying the properties above is convex (the third condition is not needed to show this). + + + +===== Friday September 25 ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember25.mp4?vrtx=view-as-webpage" and "link to handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember25.pdf". + + + +===== Standard steepest descent ===== + + +Before we proceed, we would like to discuss the approach called the +_standard Steepest descent_ (different from the above steepest descent discussion), which again leads to us having to be able +to compute a matrix. It belongs to the class of Conjugate Gradient methods (CG). + +"The success of the CG method":"https://www.cs.cmu.edu/~quake-papers/painless-conjugate-gradient.pdf" +for finding solutions of non-linear problems is based on the theory +of conjugate gradients for linear systems of equations. It belongs to +the class of iterative methods for solving problems from linear +algebra of the type +!bt +\begin{equation*} +\bm{A}\bm{x} = \bm{b}. +\end{equation*} +!et + +In the iterative process we end up with a problem like + +!bt +\begin{equation*} + \bm{r}= \bm{b}-\bm{A}\bm{x}, +\end{equation*} +!et +where $\bm{r}$ is the so-called residual or error in the iterative process. + +When we have found the exact solution, $\bm{r}=0$. + + +===== Gradient method ===== + +The residual is zero when we reach the minimum of the quadratic equation +!bt +\begin{equation*} + P(\bm{x})=\frac{1}{2}\bm{x}^T\bm{A}\bm{x} - \bm{x}^T\bm{b}, +\end{equation*} +!et + +with the constraint that the matrix $\bm{A}$ is positive definite and +symmetric. This defines also the Hessian and we want it to be positive definite. + + + +===== Steepest descent method ===== + +We denote the initial guess for $\bm{x}$ as $\bm{x}_0$. +We can assume without loss of generality that +!bt +\begin{equation*} +\bm{x}_0=0, +\end{equation*} +!et +or consider the system +!bt +\begin{equation*} +\bm{A}\bm{z} = \bm{b}-\bm{A}\bm{x}_0, +\end{equation*} +!et +instead. + + + +===== Steepest descent method ===== +!bblock +One can show that the solution $\bm{x}$ is also the unique minimizer of the quadratic form +!bt +\begin{equation*} + f(\bm{x}) = \frac{1}{2}\bm{x}^T\bm{A}\bm{x} - \bm{x}^T \bm{x} , \quad \bm{x}\in\mathbf{R}^n. +\end{equation*} +!et +This suggests taking the first basis vector $\bm{r}_1$ (see below for definition) +to be the gradient of $f$ at $\bm{x}=\bm{x}_0$, +which equals +!bt +\begin{equation*} +\bm{A}\bm{x}_0-\bm{b}, +\end{equation*} +!et +and +$\bm{x}_0=0$ it is equal $-\bm{b}$. + +!eblock + + +===== Final expressions ===== +!bblock +We can compute the residual iteratively as +!bt +\begin{equation*} +\bm{r}_{k+1}=\bm{b}-\bm{A}\bm{x}_{k+1}, + \end{equation*} +!et +which equals +!bt +\begin{equation*} +\bm{b}-\bm{A}(\bm{x}_k+\alpha_k\bm{r}_k), + \end{equation*} +!et +or +!bt +\begin{equation*} +(\bm{b}-\bm{A}\bm{x}_k)-\alpha_k\bm{A}\bm{r}_k, + \end{equation*} +!et +which gives + +!bt +\[ +\alpha_k = \frac{\bm{r}_k^T\bm{r}_k}{\bm{r}_k^T\bm{A}\bm{r}_k} +\] +!et +leading to the iterative scheme +!bt +\begin{equation*} +\bm{x}_{k+1}=\bm{x}_k-\alpha_k\bm{r}_{k}, + \end{equation*} +!et +!eblock + + + + +===== Steepest descent example ===== + +!bc pycod +import numpy as np +import numpy.linalg as la + +import scipy.optimize as sopt + +import matplotlib.pyplot as pt +from mpl_toolkits.mplot3d import axes3d + +def f(x): + return 0.5*x[0]**2 + 2.5*x[1]**2 + +def df(x): + return np.array([x[0], 5*x[1]]) + +fig = pt.figure() +ax = fig.gca(projection="3d") + +xmesh, ymesh = np.mgrid[-2:2:50j,-2:2:50j] +fmesh = f(np.array([xmesh, ymesh])) +ax.plot_surface(xmesh, ymesh, fmesh) +!ec +And then as countor plot +!bc pycod +pt.axis("equal") +pt.contour(xmesh, ymesh, fmesh) +guesses = [np.array([2, 2./5])] +!ec +Find guesses +!bc pycod +x = guesses[-1] +s = -df(x) +!ec +Run it! +!bc pycod +def f1d(alpha): + return f(x + alpha*s) + +alpha_opt = sopt.golden(f1d) +next_guess = x + alpha_opt * s +guesses.append(next_guess) +print(next_guess) +!ec +What happened? +!bc pycod +pt.axis("equal") +pt.contour(xmesh, ymesh, fmesh, 50) +it_array = np.array(guesses) +pt.plot(it_array.T[0], it_array.T[1], "x-") +!ec + + +===== Conjugate gradient method ===== +!bblock +In the CG method we define so-called conjugate directions and two vectors +$\bm{s}$ and $\bm{t}$ +are said to be +conjugate if +!bt +\begin{equation*} +\bm{s}^T\bm{A}\bm{t}= 0. +\end{equation*} +!et +The philosophy of the CG method is to perform searches in various conjugate directions +of our vectors $\bm{x}_i$ obeying the above criterion, namely +!bt +\begin{equation*} +\bm{x}_i^T\bm{A}\bm{x}_j= 0. +\end{equation*} +!et +Two vectors are conjugate if they are orthogonal with respect to +this inner product. Being conjugate is a symmetric relation: if $\bm{s}$ is conjugate to $\bm{t}$, then $\bm{t}$ is conjugate to $\bm{s}$. +!eblock + + +===== Conjugate gradient method ===== +!bblock +An example is given by the eigenvectors of the matrix +!bt +\begin{equation*} +\bm{v}_i^T\bm{A}\bm{v}_j= \lambda\bm{v}_i^T\bm{v}_j, +\end{equation*} +!et +which is zero unless $i=j$. +!eblock + + + +===== Conjugate gradient method ===== +!bblock +Assume now that we have a symmetric positive-definite matrix $\bm{A}$ of size +$n\times n$. At each iteration $i+1$ we obtain the conjugate direction of a vector +!bt +\begin{equation*} +\bm{x}_{i+1}=\bm{x}_{i}+\alpha_i\bm{p}_{i}. +\end{equation*} +!et +We assume that $\bm{p}_{i}$ is a sequence of $n$ mutually conjugate directions. +Then the $\bm{p}_{i}$ form a basis of $R^n$ and we can expand the solution +$ \bm{A}\bm{x} = \bm{b}$ in this basis, namely + +!bt +\begin{equation*} + \bm{x} = \sum^{n}_{i=1} \alpha_i \bm{p}_i. +\end{equation*} +!et +!eblock + + +===== Conjugate gradient method ===== +!bblock +The coefficients are given by +!bt +\begin{equation*} + \mathbf{A}\mathbf{x} = \sum^{n}_{i=1} \alpha_i \mathbf{A} \mathbf{p}_i = \mathbf{b}. +\end{equation*} +!et +Multiplying with $\bm{p}_k^T$ from the left gives + +!bt +\begin{equation*} + \bm{p}_k^T \bm{A}\bm{x} = \sum^{n}_{i=1} \alpha_i\bm{p}_k^T \bm{A}\bm{p}_i= \bm{p}_k^T \bm{b}, +\end{equation*} +!et +and we can define the coefficients $\alpha_k$ as + +!bt +\begin{equation*} + \alpha_k = \frac{\bm{p}_k^T \bm{b}}{\bm{p}_k^T \bm{A} \bm{p}_k} +\end{equation*} +!et +!eblock + + +===== Conjugate gradient method and iterations ===== +!bblock + +If we choose the conjugate vectors $\bm{p}_k$ carefully, +then we may not need all of them to obtain a good approximation to the solution +$\bm{x}$. +We want to regard the conjugate gradient method as an iterative method. +This will us to solve systems where $n$ is so large that the direct +method would take too much time. + +We denote the initial guess for $\bm{x}$ as $\bm{x}_0$. +We can assume without loss of generality that +!bt +\begin{equation*} +\bm{x}_0=0, +\end{equation*} +!et +or consider the system +!bt +\begin{equation*} +\bm{A}\bm{z} = \bm{b}-\bm{A}\bm{x}_0, +\end{equation*} +!et +instead. +!eblock + + + +===== Conjugate gradient method ===== +!bblock +One can show that the solution $\bm{x}$ is also the unique minimizer of the quadratic form +!bt +\begin{equation*} + f(\bm{x}) = \frac{1}{2}\bm{x}^T\bm{A}\bm{x} - \bm{x}^T \bm{x} , \quad \bm{x}\in\mathbf{R}^n. +\end{equation*} +!et +This suggests taking the first basis vector $\bm{p}_1$ +to be the gradient of $f$ at $\bm{x}=\bm{x}_0$, +which equals +!bt +\begin{equation*} +\bm{A}\bm{x}_0-\bm{b}, +\end{equation*} +!et +and +$\bm{x}_0=0$ it is equal $-\bm{b}$. +The other vectors in the basis will be conjugate to the gradient, +hence the name conjugate gradient method. +!eblock + + + +===== Conjugate gradient method ===== +!bblock +Let $\bm{r}_k$ be the residual at the $k$-th step: +!bt +\begin{equation*} +\bm{r}_k=\bm{b}-\bm{A}\bm{x}_k. +\end{equation*} +!et +Note that $\bm{r}_k$ is the negative gradient of $f$ at +$\bm{x}=\bm{x}_k$, +so the gradient descent method would be to move in the direction $\bm{r}_k$. +Here, we insist that the directions $\bm{p}_k$ are conjugate to each other, +so we take the direction closest to the gradient $\bm{r}_k$ +under the conjugacy constraint. +This gives the following expression +!bt +\begin{equation*} +\bm{p}_{k+1}=\bm{r}_k-\frac{\bm{p}_k^T \bm{A}\bm{r}_k}{\bm{p}_k^T\bm{A}\bm{p}_k} \bm{p}_k. +\end{equation*} +!et +!eblock + + +===== Conjugate gradient method ===== +!bblock +We can also compute the residual iteratively as +!bt +\begin{equation*} +\bm{r}_{k+1}=\bm{b}-\bm{A}\bm{x}_{k+1}, + \end{equation*} +!et +which equals +!bt +\begin{equation*} +\bm{b}-\bm{A}(\bm{x}_k+\alpha_k\bm{p}_k), + \end{equation*} +!et +or +!bt +\begin{equation*} +(\bm{b}-\bm{A}\bm{x}_k)-\alpha_k\bm{A}\bm{p}_k, + \end{equation*} +!et +which gives + +!bt +\begin{equation*} +\bm{r}_{k+1}=\bm{r}_k-\bm{A}\bm{p}_{k}, + \end{equation*} +!et +!eblock + + + + + + +===== Revisiting our first homework ===== + +We will use linear regression as a case study for the gradient descent +methods. Linear regression is a great test case for the gradient +descent methods discussed in the lectures since it has several +desirable properties such as: + +o An analytical solution (recall homework set 1). +o The gradient can be computed analytically. +o The cost function is convex which guarantees that gradient descent converges for small enough learning rates + +We revisit an example similar to what we had in the first homework set. We had a function of the type + +!bc pycod +x = 2*np.random.rand(m,1) +y = 4+3*x+np.random.randn(m,1) +!ec +with $x_i \in [0,1] $ is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution $\cal {N}(0,1)$. +The linear regression model is given by +!bt +\[ +h_\beta(x) = \bm{y} = \beta_0 + \beta_1 x, +\] +!et +such that +!bt +\[ +\bm{y}_i = \beta_0 + \beta_1 x_i. +\] +!et + + +===== Gradient descent example ===== + +Let $\mathbf{y} = (y_1,\cdots,y_n)^T$, $\mathbf{\bm{y}} = (\bm{y}_1,\cdots,\bm{y}_n)^T$ and $\beta = (\beta_0, \beta_1)^T$ + +It is convenient to write $\mathbf{\bm{y}} = X\beta$ where $X \in \mathbb{R}^{100 \times 2} $ is the design matrix given by (we keep the intercept here) +!bt +\[ +X \equiv \begin{bmatrix} +1 & x_1 \\ +\vdots & \vdots \\ +1 & x_{100} & \\ +\end{bmatrix}. +\] +!et +The cost/loss/risk function is given by ( +!bt +\[ +C(\beta) = \frac{1}{n}||X\beta-\mathbf{y}||_{2}^{2} = \frac{1}{n}\sum_{i=1}^{100}\left[ (\beta_0 + \beta_1 x_i)^2 - 2 y_i (\beta_0 + \beta_1 x_i) + y_i^2\right] +\] +!et +and we want to find $\beta$ such that $C(\beta)$ is minimized. + + +===== The derivative of the cost/loss function ===== + +Computing $\partial C(\beta) / \partial \beta_0$ and $\partial C(\beta) / \partial \beta_1$ we can show that the gradient can be written as +!bt +\[ +\nabla_{\beta} C(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ +\end{bmatrix} = \frac{2}{n}X^T(X\beta - \mathbf{y}), +\] +!et +where $X$ is the design matrix defined above. + + +===== The Hessian matrix ===== +The Hessian matrix of $C(\beta)$ is given by +!bt +\[ +\bm{H} \equiv \begin{bmatrix} +\frac{\partial^2 C(\beta)}{\partial \beta_0^2} & \frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} \\ +\frac{\partial^2 C(\beta)}{\partial \beta_0 \partial \beta_1} & \frac{\partial^2 C(\beta)}{\partial \beta_1^2} & \\ +\end{bmatrix} = \frac{2}{n}X^T X. +\] +!et +This result implies that $C(\beta)$ is a convex function since the matrix $X^T X$ always is positive semi-definite. + + + + + +===== Simple program ===== + +We can now write a program that minimizes $C(\beta)$ using the gradient descent method with a constant learning rate $\gamma$ according to +!bt +\[ +\beta_{k+1} = \beta_k - \gamma \nabla_\beta C(\beta_k), \ k=0,1,\cdots +\] +!et + +We can use the expression we computed for the gradient and let use a +$\beta_0$ be chosen randomly and let $\gamma = 0.001$. Stop iterating +when $||\nabla_\beta C(\beta_k) || \leq \epsilon = 10^{-8}$. _Note that the code below does not include the latter stop criterion_. + +And finally we can compare our solution for $\beta$ with the analytic result given by +$\beta= (X^TX)^{-1} X^T \mathbf{y}$. + + +===== Gradient Descent Example ===== + +Here our simple example +!bc pycod + +# Importing various packages +from random import random, seed +import numpy as np +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +from matplotlib.ticker import LinearLocator, FormatStrFormatter +import sys + +# the number of datapoints +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) + +X = np.c_[np.ones((n,1)), x] +# Hessian matrix +H = (2.0/n)* X.T @ X +# Get the eigenvalues +EigValues, EigVectors = np.linalg.eig(H) +print(EigValues) + +beta_linreg = np.linalg.inv(X.T @ X) @ X.T @ y +print(beta_linreg) +beta = np.random.randn(2,1) + +eta = 1.0/np.max(EigValues) +Niterations = 1000 + +for iter in range(Niterations): + gradient = (2.0/n)*X.T @ (X @ beta-y) + beta -= eta*gradient + +print(beta) +xnew = np.array([[0],[2]]) +xbnew = np.c_[np.ones((2,1)), xnew] +ypredict = xbnew.dot(beta) +ypredict2 = xbnew.dot(beta_linreg) +plt.plot(xnew, ypredict, "r-") +plt.plot(xnew, ypredict2, "b-") +plt.plot(x, y ,'ro') +plt.axis([0,2.0,0, 15.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Gradient descent example') +plt.show() + +!ec + + +===== And a corresponding example using _scikit-learn_ ===== + +!bc pycod +# Importing various packages +from random import random, seed +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import SGDRegressor + +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) + +X = np.c_[np.ones((n,1)), x] +beta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) +print(beta_linreg) +sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) +sgdreg.fit(x,y.ravel()) +print(sgdreg.intercept_, sgdreg.coef_) + +!ec + + + + +===== Gradient descent and Ridge ===== + +We have also discussed Ridge regression where the loss function contains a regularized term given by the $L_2$ norm of $\beta$, +!bt +\[ +C_{\text{ridge}}(\beta) = \frac{1}{n}||X\beta -\mathbf{y}||^2 + \lambda ||\beta||^2, \ \lambda \geq 0. +\] +!et + +In order to minimize $C_{\text{ridge}}(\beta)$ using GD we only have adjust the gradient as follows +!bt +\[ +\nabla_\beta C_{\text{ridge}}(\beta) = \frac{2}{n}\begin{bmatrix} \sum_{i=1}^{100} \left(\beta_0+\beta_1x_i-y_i\right) \\ +\sum_{i=1}^{100}\left( x_i (\beta_0+\beta_1x_i)-y_ix_i\right) \\ +\end{bmatrix} + 2\lambda\begin{bmatrix} \beta_0 \\ \beta_1\end{bmatrix} = 2 (X^T(X\beta - \mathbf{y})+\lambda \beta). +\] +!et + +We can easily extend our program to minimize $C_{\text{ridge}}(\beta)$ using gradient descent and compare with the analytical solution given by +!bt +\[ +\beta_{\text{ridge}} = \left(X^T X + \lambda I_{2 \times 2} \right)^{-1} X^T \mathbf{y}. +\] +!et + + + +===== Program example for gradient descent with Ridge Regression ===== +!bc pycod +from random import random, seed +import numpy as np +import matplotlib.pyplot as plt +from mpl_toolkits.mplot3d import Axes3D +from matplotlib import cm +from matplotlib.ticker import LinearLocator, FormatStrFormatter +import sys + +# the number of datapoints +n = 100 +x = 2*np.random.rand(n,1) +y = 4+3*x+np.random.randn(n,1) + +X = np.c_[np.ones((n,1)), x] +XT_X = X.T @ X + +#Ridge parameter lambda +lmbda = 0.001 +Id = lmbda* np.eye(XT_X.shape[0]) + +beta_linreg = np.linalg.inv(XT_X+Id) @ X.T @ y +print(beta_linreg) +# Start plain gradient descent +beta = np.random.randn(2,1) + +eta = 0.1 +Niterations = 100 + +for iter in range(Niterations): + gradients = 2.0/n*X.T @ (X @ (beta)-y)+2*lmbda*beta + beta -= eta*gradients + +print(beta) +ypredict = X @ beta +ypredict2 = X @ beta_linreg +plt.plot(x, ypredict, "r-") +plt.plot(x, ypredict2, "b-") +plt.plot(x, y ,'ro') +plt.axis([0,2.0,0, 15.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Gradient descent example for Ridge') +plt.show() + + +!ec + + +===== Using gradient descent methods, limitations ===== + +* _Gradient descent (GD) finds local minima of our function_. Since the GD algorithm is deterministic, if it converges, it will converge to a local minimum of our cost/loss/risk function. Because in ML we are often dealing with extremely rugged landscapes with many local minima, this can lead to poor performance. + +* _GD is sensitive to initial conditions_. One consequence of the local nature of GD is that initial conditions matter. Depending on where one starts, one will end up at a different local minima. Therefore, it is very important to think about how one initializes the training process. This is true for GD as well as more complicated variants of GD. + +* _Gradients are computationally expensive to calculate for large datasets_. In many cases in statistics and ML, the cost/loss/risk function is a sum of terms, with one term for each data point. For example, in linear regression, $E \propto \sum_{i=1}^n (y_i - \mathbf{w}^T\cdot\mathbf{x}_i)^2$; for logistic regression, the square error is replaced by the cross entropy. To calculate the gradient we have to sum over *all* $n$ data points. Doing this at every GD step becomes extremely computationally expensive. An ingenious solution to this, is to calculate the gradients using small subsets of the data called ``mini batches''. This has the added benefit of introducing stochasticity into our algorithm. + +* _GD is very sensitive to choices of learning rates_. GD is extremely sensitive to the choice of learning rates. If the learning rate is very small, the training process take an extremely long time. For larger learning rates, GD can diverge and give poor results. Furthermore, depending on what the local landscape looks like, we have to modify the learning rates to ensure convergence. Ideally, we would *adaptively* choose the learning rates to match the landscape. + +* _GD treats all directions in parameter space uniformly._ Another major drawback of GD is that unlike Newton's method, the learning rate for GD is the same in all directions in parameter space. For this reason, the maximum learning rate is set by the behavior of the steepest direction and this can significantly slow down training. Ideally, we would like to take large steps in flat directions and small steps in steep directions. Since we are exploring rugged landscapes where curvatures change, this requires us to keep track of not only the gradient but second derivatives. The ideal scenario would be to calculate the Hessian but this proves to be too computationally expensive. + +* GD can take exponential time to escape saddle points, even with random initialization. As we mentioned, GD is extremely sensitive to initial condition since it determines the particular local minimum GD would eventually reach. However, even with a good initialization scheme, through the introduction of randomness, GD can still take exponential time to escape saddle points. + + + + + +===== Stochastic Gradient Descent ===== + +Stochastic gradient descent (SGD) and variants thereof address some of +the shortcomings of the Gradient descent method discussed above. + +The underlying idea of SGD comes from the observation that the cost +function, which we want to minimize, can almost always be written as a +sum over $n$ data points $\{\mathbf{x}_i\}_{i=1}^n$, +!bt +\[ +C(\mathbf{\beta}) = \sum_{i=1}^n c_i(\mathbf{x}_i, +\mathbf{\beta}). +\] +!et + + +===== Computation of gradients ===== + +This in turn means that the gradient can be +computed as a sum over $i$-gradients +!bt +\[ +\nabla_\beta C(\mathbf{\beta}) = \sum_i^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}). +\] +!et + +Stochasticity/randomness is introduced by only taking the +gradient on a subset of the data called minibatches. If there are $n$ +data points and the size of each minibatch is $M$, there will be $n/M$ +minibatches. We denote these minibatches by $B_k$ where +$k=1,\cdots,n/M$. + + +===== SGD example ===== +As an example, suppose we have $10$ data points $(\mathbf{x}_1,\cdots, \mathbf{x}_{10})$ +and we choose to have $M=5$ minibathces, +then each minibatch contains two data points. In particular we have +$B_1 = (\mathbf{x}_1,\mathbf{x}_2), \cdots, B_5 = +(\mathbf{x}_9,\mathbf{x}_{10})$. Note that if you choose $M=1$ you +have only a single batch with all data points and on the other extreme, +you may choose $M=n$ resulting in a minibatch for each datapoint, i.e +$B_k = \mathbf{x}_k$. + +The idea is now to approximate the gradient by replacing the sum over +all data points with a sum over the data points in one the minibatches +picked at random in each gradient descent step +!bt +\[ +\nabla_{\beta} +C(\mathbf{\beta}) = \sum_{i=1}^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}) \rightarrow \sum_{i \in B_k}^n \nabla_\beta +c_i(\mathbf{x}_i, \mathbf{\beta}). +\] +!et + + +===== The gradient step ===== + +Thus a gradient descent step now looks like +!bt +\[ +\beta_{j+1} = \beta_j - \gamma_j \sum_{i \in B_k}^n \nabla_\beta c_i(\mathbf{x}_i, +\mathbf{\beta}) +\] +!et + +where $k$ is picked at random with equal +probability from $[1,n/M]$. An iteration over the number of +minibathces (n/M) is commonly referred to as an epoch. Thus it is +typical to choose a number of epochs and for each epoch iterate over +the number of minibatches, as exemplified in the code below. + + +===== Simple example code ===== + +!bc pycod +import numpy as np + +n = 100 #100 datapoints +M = 5 #size of each minibatch +m = int(n/M) #number of minibatches +n_epochs = 10 #number of epochs + +j = 0 +for epoch in range(1,n_epochs+1): + for i in range(m): + k = np.random.randint(m) #Pick the k-th minibatch at random + #Compute the gradient using the data in minibatch Bk + #Compute new suggestion for + j += 1 +!ec + +Taking the gradient only on a subset of the data has two important +benefits. First, it introduces randomness which decreases the chance +that our opmization scheme gets stuck in a local minima. Second, if +the size of the minibatches are small relative to the number of +datapoints ($M < n$), the computation of the gradient is much +cheaper since we sum over the datapoints in the $k-th$ minibatch and not +all $n$ datapoints. + + +===== When do we stop? ===== + +A natural question is when do we stop the search for a new minimum? +One possibility is to compute the full gradient after a given number +of epochs and check if the norm of the gradient is smaller than some +threshold and stop if true. However, the condition that the gradient +is zero is valid also for local minima, so this would only tell us +that we are close to a local/global minimum. However, we could also +evaluate the cost function at this point, store the result and +continue the search. If the test kicks in at a later stage we can +compare the values of the cost function and keep the $\beta$ that +gave the lowest value. + + +===== Slightly different approach ===== + +Another approach is to let the step length $\gamma_j$ depend on the +number of epochs in such a way that it becomes very small after a +reasonable time such that we do not move at all. + +As an example, let $e = 0,1,2,3,\cdots$ denote the current epoch and let $t_0, t_1 > 0$ be two fixed numbers. Furthermore, let $t = e \cdot m + i$ where $m$ is the number of minibatches and $i=0,\cdots,m-1$. Then the function $$\gamma_j(t; t_0, t_1) = \frac{t_0}{t+t_1} $$ goes to zero as the number of epochs gets large. I.e. we start with a step length $\gamma_j (0; t_0, t_1) = t_0/t_1$ which decays in *time* $t$. + +In this way we can fix the number of epochs, compute $\beta$ and +evaluate the cost function at the end. Repeating the computation will +give a different result since the scheme is random by design. Then we +pick the final $\beta$ that gives the lowest value of the cost +function. + +!bc pycod +import numpy as np + +def step_length(t,t0,t1): + return t0/(t+t1) + +n = 100 #100 datapoints +M = 5 #size of each minibatch +m = int(n/M) #number of minibatches +n_epochs = 500 #number of epochs +t0 = 1.0 +t1 = 10 + +gamma_j = t0/t1 +j = 0 +for epoch in range(1,n_epochs+1): + for i in range(m): + k = np.random.randint(m) #Pick the k-th minibatch at random + #Compute the gradient using the data in minibatch Bk + #Compute new suggestion for beta + t = epoch*m+i + gamma_j = step_length(t,t0,t1) + j += 1 + +print("gamma_j after %d epochs: %g" % (n_epochs,gamma_j)) +!ec + + + + + + +===== Program for stochastic gradient ===== + +!bc pycod +# Importing various packages +from math import exp, sqrt +from random import random, seed +import numpy as np +import matplotlib.pyplot as plt +from sklearn.linear_model import SGDRegressor + +m = 100 +x = 2*np.random.rand(m,1) +y = 4+3*x+np.random.randn(m,1) + +X = np.c_[np.ones((m,1)), x] +theta_linreg = np.linalg.inv(X.T @ X) @ (X.T @ y) +print("Own inversion") +print(theta_linreg) +sgdreg = SGDRegressor(max_iter = 50, penalty=None, eta0=0.1) +sgdreg.fit(x,y.ravel()) +print("sgdreg from scikit") +print(sgdreg.intercept_, sgdreg.coef_) + + +theta = np.random.randn(2,1) +eta = 0.1 +Niterations = 1000 + + +for iter in range(Niterations): + gradients = 2.0/m*X.T @ ((X @ theta)-y) + theta -= eta*gradients +print("theta from own gd") +print(theta) + +xnew = np.array([[0],[2]]) +Xnew = np.c_[np.ones((2,1)), xnew] +ypredict = Xnew.dot(theta) +ypredict2 = Xnew.dot(theta_linreg) + + +n_epochs = 50 +t0, t1 = 5, 50 +def learning_schedule(t): + return t0/(t+t1) + +theta = np.random.randn(2,1) + +for epoch in range(n_epochs): + for i in range(m): + random_index = np.random.randint(m) + xi = X[random_index:random_index+1] + yi = y[random_index:random_index+1] + gradients = 2 * xi.T @ ((xi @ theta)-yi) + eta = learning_schedule(epoch*m+i) + theta = theta - eta*gradients +print("theta from own sdg") +print(theta) + +plt.plot(xnew, ypredict, "r-") +plt.plot(xnew, ypredict2, "b-") +plt.plot(x, y ,'ro') +plt.axis([0,2.0,0, 15.0]) +plt.xlabel(r'$x$') +plt.ylabel(r'$y$') +plt.title(r'Random numbers ') +plt.show() + +!ec + +_Challenge_: try to write a similar code for a Logistic Regression case. + + + + + + + + + + + diff --git a/doc/BookChapters/chapter4.do.txt~ b/doc/BookChapters/chapter4.do.txt~ new file mode 100644 index 000000000..effaf97ec --- /dev/null +++ b/doc/BookChapters/chapter4.do.txt~ @@ -0,0 +1,1594 @@ +TITLE: Data Analysis and Machine Learning: Logistic Regression +AUTHOR: Morten Hjorth-Jensen {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo & Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University +DATE: today + + +!split +===== Plans for week 38 ===== + +* Thursday: Summary of regression methods and discussion of project 1. We revisit also cross-validation and bootstrap as resampling techniques with examples. Recommended reading: "Hastie et al":"https://www.springer.com/gp/book/9780387848570" chapters 3 and 7.1-7.6 and 7.10-7.12. +* Friday: Logistic Regression. Recommended reading: "Hastie et al":"https://www.springer.com/gp/book/9780387848570" chapters 4.1-4.4 and "Murphy":"https://mitpress.mit.edu/books/machine-learning-1" chapter 8.1-8.2 + + +!split +===== Thursday September 17 ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSeptember17.mp4?vrtx=view-as-webpage" and "link to handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember17.pdf". + +!split +===== Ridge and LASSO Regression, reminder ===== + +The expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is +our optimization problem is +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\bm{y}-\bm{X}\bm{\beta}\right)^T\left(\bm{y}-\bm{X}\bm{\beta}\right)\right\}. +\] +!et +or we can state it as +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2, +\] +!et +where we have used the definition of a norm-2 vector, that is +!bt +\[ +\vert\vert \bm{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}. +\] +!et + +By minimizing the above equation with respect to the parameters +$\bm{\beta}$ we could then obtain an analytical expression for the +parameters $\bm{\beta}$. We can add a regularization parameter $\lambda$ by +defining a new cost function to be optimized, that is + +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_2^2 +\] +!et + +which leads to the Ridge regression minimization problem where we +require that $\vert\vert \bm{\beta}\vert\vert_2^2\le t$, where $t$ is +a finite number larger than zero. By defining + +!bt +\[ +C(\bm{X},\bm{\beta})=\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_1, +\] +!et + +we have a new optimization equation +!bt +\[ +{\displaystyle \min_{\bm{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \bm{y}-\bm{X}\bm{\beta}\vert\vert_2^2+\lambda\vert\vert \bm{\beta}\vert\vert_1 +\] +!et +which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. + +Here we have defined the norm-1 as +!bt +\[ +\vert\vert \bm{x}\vert\vert_1 = \sum_i \vert x_i\vert. +\] +!et + + + +!split +===== Various steps in cross-validation ===== + +When the repetitive splitting of the data set is done randomly, +samples may accidently end up in a fast majority of the splits in +either training or test set. Such samples may have an unbalanced +influence on either model building or prediction evaluation. To avoid +this $k$-fold cross-validation structures the data splitting. The +samples are divided into $k$ more or less equally sized exhaustive and +mutually exclusive subsets. In turn (at each split) one of these +subsets plays the role of the test set while the union of the +remaining subsets constitutes the training set. Such a splitting +warrants a balanced representation of each sample in both training and +test set over the splits. Still the division into the $k$ subsets +involves a degree of randomness. This may be fully excluded when +choosing $k=n$. This particular case is referred to as leave-one-out +cross-validation (LOOCV). + +!split +===== How to set up the cross-validation for Ridge and/or Lasso ===== + +* Define a range of interest for the penalty parameter. + +* Divide the data set into training and test set comprising samples $\{1, \ldots, n\} \setminus i$ and $\{ i \}$, respectively. + +* Fit the linear regression model by means of ridge estimation for each $\lambda$ in the grid using the training set, and the corresponding estimate of the error variance $\bm{\sigma}_{-i}^2(\lambda)$, as +!bt +\begin{align*} +\bm{\beta}_{-i}(\lambda) & = ( \bm{X}_{-i, \ast}^{T} +\bm{X}_{-i, \ast} + \lambda \bm{I}_{pp})^{-1} +\bm{X}_{-i, \ast}^{T} \bm{y}_{-i} +\end{align*} +!et + +* Evaluate the prediction performance of these models on the test set by $\log\{L[y_i, \bm{X}_{i, \ast}; \bm{\beta}_{-i}(\lambda), \bm{\sigma}_{-i}^2(\lambda)]\}$. Or, by the prediction error $|y_i - \bm{X}_{i, \ast} \bm{\beta}_{-i}(\lambda)|$, the relative error, the error squared or the R2 score function. + +* Repeat the first three steps such that each sample plays the role of the test set once. + +* Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as +!bt +\begin{align*} +\frac{1}{n} \sum_{i = 1}^n \log\{L[y_i, \mathbf{X}_{i, \ast}; \bm{\beta}_{-i}(\lambda), \bm{\sigma}_{-i}^2(\lambda)]\}. +\end{align*} +!et + +!split +===== Cross-validation in brief ===== + +For the various values of $k$ + +o shuffle the dataset randomly. +o Split the dataset into $k$ groups. +o For each unique group: + o Decide which group to use as set for test data + o Take the remaining groups as a training data set + o Fit a model on the training set and evaluate it on the test set + o Retain the evaluation score and discard the model +o Summarize the model using the sample of model evaluation scores + + + +!split +===== Code Example for Cross-validation and $k$-fold Cross-validation ===== + +The code here uses Ridge regression with cross-validation (CV) resampling and $k$-fold CV in order to fit a specific polynomial. +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import KFold +from sklearn.linear_model import Ridge +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import PolynomialFeatures + +# A seed just to ensure that the random numbers are the same for every run. +# Useful for eventual debugging. +np.random.seed(3155) + +# Generate the data. +nsamples = 100 +x = np.random.randn(nsamples) +y = 3*x**2 + np.random.randn(nsamples) + +## Cross-validation on Ridge regression using KFold only + +# Decide degree on polynomial to fit +poly = PolynomialFeatures(degree = 6) + +# Decide which values of lambda to use +nlambdas = 500 +lambdas = np.logspace(-3, 5, nlambdas) + +# Initialize a KFold instance +k = 5 +kfold = KFold(n_splits = k) + +# Perform the cross-validation to estimate MSE +scores_KFold = np.zeros((nlambdas, k)) + +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + j = 0 + for train_inds, test_inds in kfold.split(x): + xtrain = x[train_inds] + ytrain = y[train_inds] + + xtest = x[test_inds] + ytest = y[test_inds] + + Xtrain = poly.fit_transform(xtrain[:, np.newaxis]) + ridge.fit(Xtrain, ytrain[:, np.newaxis]) + + Xtest = poly.fit_transform(xtest[:, np.newaxis]) + ypred = ridge.predict(Xtest) + + scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred) + + j += 1 + i += 1 + + +estimated_mse_KFold = np.mean(scores_KFold, axis = 1) + +## Cross-validation using cross_val_score from sklearn along with KFold + +# kfold is an instance initialized above as: +# kfold = KFold(n_splits = k) + +estimated_mse_sklearn = np.zeros(nlambdas) +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + + X = poly.fit_transform(x[:, np.newaxis]) + estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold) + + # cross_val_score return an array containing the estimated negative mse for every fold. + # we have to the the mean of every array in order to get an estimate of the mse of the model + estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds) + + i += 1 + +## Plot and compare the slightly different ways to perform cross-validation + +plt.figure() + +plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score') +plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold') + +plt.xlabel('log10(lambda)') +plt.ylabel('mse') + +plt.legend() + +plt.show() + +!ec + + + +!split +===== Bias-Variance tradeoff with Bootstrap ===== +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.preprocessing import PolynomialFeatures +from sklearn.model_selection import train_test_split +from sklearn.pipeline import make_pipeline +from sklearn.utils import resample + +np.random.seed(2018) + +n = 40 +n_boostraps = 100 +maxdegree = 14 + + +# Make data set. +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +error = np.zeros(maxdegree) +bias = np.zeros(maxdegree) +variance = np.zeros(maxdegree) +polydegree = np.zeros(maxdegree) +x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2) + +for degree in range(maxdegree): + model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False)) + y_pred = np.empty((y_test.shape[0], n_boostraps)) + for i in range(n_boostraps): + x_, y_ = resample(x_train, y_train) + y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel() + + polydegree[degree] = degree + error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) ) + bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 ) + variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) ) + print('Polynomial degree:', degree) + print('Error:', error[degree]) + print('Bias^2:', bias[degree]) + print('Var:', variance[degree]) + print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree])) + +plt.plot(polydegree, error, label='Error') +plt.plot(polydegree, bias, label='bias') +plt.plot(polydegree, variance, label='Variance') +plt.legend() +plt.show() + + + + +!ec + + +!split +===== Another Example from Scikit-Learn's Repository ===== +!bc pycod +""" +============================ +Underfitting vs. Overfitting +============================ + +This example demonstrates the problems of underfitting and overfitting and +how we can use linear regression with polynomial features to approximate +nonlinear functions. The plot shows the function that we want to approximate, +which is a part of the cosine function. In addition, the samples from the +real function and the approximations of different models are displayed. The +models have polynomial features of different degrees. We can see that a +linear function (polynomial with degree 1) is not sufficient to fit the +training samples. This is called **underfitting**. A polynomial of degree 4 +approximates the true function almost perfectly. However, for higher degrees +the model will **overfit** the training data, i.e. it learns the noise of the +training data. +We evaluate quantitatively **overfitting** / **underfitting** by using +cross-validation. We calculate the mean squared error (MSE) on the validation +set, the higher, the less likely the model generalizes correctly from the +training data. +""" + +print(__doc__) + +import numpy as np +import matplotlib.pyplot as plt +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import PolynomialFeatures +from sklearn.linear_model import LinearRegression +from sklearn.model_selection import cross_val_score + + +def true_fun(X): + return np.cos(1.5 * np.pi * X) + +np.random.seed(0) + +n_samples = 30 +degrees = [1, 4, 15] + +X = np.sort(np.random.rand(n_samples)) +y = true_fun(X) + np.random.randn(n_samples) * 0.1 + +plt.figure(figsize=(14, 5)) +for i in range(len(degrees)): + ax = plt.subplot(1, len(degrees), i + 1) + plt.setp(ax, xticks=(), yticks=()) + + polynomial_features = PolynomialFeatures(degree=degrees[i], + include_bias=False) + linear_regression = LinearRegression() + pipeline = Pipeline([("polynomial_features", polynomial_features), + ("linear_regression", linear_regression)]) + pipeline.fit(X[:, np.newaxis], y) + + # Evaluate the models using crossvalidation + scores = cross_val_score(pipeline, X[:, np.newaxis], y, + scoring="neg_mean_squared_error", cv=10) + + X_test = np.linspace(0, 1, 100) + plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model") + plt.plot(X_test, true_fun(X_test), label="True function") + plt.scatter(X, y, edgecolor='b', s=20, label="Samples") + plt.xlabel("x") + plt.ylabel("y") + plt.xlim((0, 1)) + plt.ylim((-2, 2)) + plt.legend(loc="best") + plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format( + degrees[i], -scores.mean(), scores.std())) +plt.show() +!ec + + + +!split +===== Cross-validation with Ridge ===== +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import KFold +from sklearn.linear_model import Ridge +from sklearn.model_selection import cross_val_score +from sklearn.preprocessing import PolynomialFeatures + +# A seed just to ensure that the random numbers are the same for every run. +np.random.seed(3155) +# Generate the data. +n = 100 +x = np.linspace(-3, 3, n).reshape(-1, 1) +y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape) +# Decide degree on polynomial to fit +poly = PolynomialFeatures(degree = 10) + +# Decide which values of lambda to use +nlambdas = 500 +lambdas = np.logspace(-3, 5, nlambdas) +# Initialize a KFold instance +k = 5 +kfold = KFold(n_splits = k) +estimated_mse_sklearn = np.zeros(nlambdas) +i = 0 +for lmb in lambdas: + ridge = Ridge(alpha = lmb) + estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold) + estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds) + i += 1 +plt.figure() +plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score') +plt.xlabel('log10(lambda)') +plt.ylabel('MSE') +plt.legend() +plt.show() + + +!ec + + + + + + + + + + +!split +===== The Ising model ===== + +The one-dimensional Ising model with nearest neighbor interaction, no +external field and a constant coupling constant $J$ is given by + +!bt +\begin{align} + H = -J \sum_{k}^L s_k s_{k + 1}, +\end{align} +!et + +where $s_i \in \{-1, 1\}$ and $s_{N + 1} = s_1$. The number of spins +in the system is determined by $L$. For the one-dimensional system +there is no phase transition. + +We will look at a system of $L = 40$ spins with a coupling constant of +$J = 1$. To get enough training data we will generate 10000 states +with their respective energies. + + +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1 import make_axes_locatable +import seaborn as sns +import scipy.linalg as scl +from sklearn.model_selection import train_test_split +import tqdm +sns.set(color_codes=True) +cmap_args=dict(vmin=-1., vmax=1., cmap='seismic') + +L = 40 +n = int(1e4) + +spins = np.random.choice([-1, 1], size=(n, L)) +J = 1.0 + +energies = np.zeros(n) + +for i in range(n): + energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1)) +!ec + +Here we use ordinary least squares +regression to predict the energy for the nearest neighbor +one-dimensional Ising model on a ring, i.e., the endpoints wrap +around. We will use linear regression to fit a value for +the coupling constant to achieve this. + +!split +===== Reformulating the problem to suit regression ===== + +A more general form for the one-dimensional Ising model is + +!bt +\begin{align} + H = - \sum_j^L \sum_k^L s_j s_k J_{jk}. +\end{align} +!et + +Here we allow for interactions beyond the nearest neighbors and a state dependent +coupling constant. This latter expression can be formulated as +a matrix-product +!bt +\begin{align} + \bm{H} = \bm{X} J, +\end{align} +!et + +where $X_{jk} = s_j s_k$ and $J$ is a matrix which consists of the +elements $-J_{jk}$. This form of writing the energy fits perfectly +with the form utilized in linear regression, that is + +!bt +\begin{align} + \bm{y} = \bm{X}\bm{\beta} + \bm{\epsilon}, +\end{align} +!et + +We split the data in training and test data as discussed in the previous example + +!bc pycod +X = np.zeros((n, L ** 2)) +for i in range(n): + X[i] = np.outer(spins[i], spins[i]).ravel() +y = energies +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) +!ec + +!split +===== Linear regression ===== + +In the ordinary least squares method we choose the cost function + +!bt +\begin{align} + C(\bm{X}, \bm{\beta})= \frac{1}{n}\left\{(\bm{X}\bm{\beta} - \bm{y})^T(\bm{X}\bm{\beta} - \bm{y})\right\}. +\end{align} +!et + +We then find the extremal point of $C$ by taking the derivative with respect to $\bm{\beta}$ as discussed above. +This yields the expression for $\bm{\beta}$ to be + +!bt +\[ + \bm{\beta} = \frac{\bm{X}^T \bm{y}}{\bm{X}^T \bm{X}}, +\] +!et + +which immediately imposes some requirements on $\bm{X}$ as there must exist +an inverse of $\bm{X}^T \bm{X}$. If the expression we are modeling contains an +intercept, i.e., a constant term, we must make sure that the +first column of $\bm{X}$ consists of $1$. We do this here + +!bc pycod +X_train_own = np.concatenate( + (np.ones(len(X_train))[:, np.newaxis], X_train), + axis=1 +) +X_test_own = np.concatenate( + (np.ones(len(X_test))[:, np.newaxis], X_test), + axis=1 +) +!ec + +!bc pycod +def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray: + return scl.inv(x.T @ x) @ (x.T @ y) +beta = ols_inv(X_train_own, y_train) +!ec + + +!split +===== Singular Value decomposition ===== + +Doing the inversion directly turns out to be a bad idea since the matrix +$\bm{X}^T\bm{X}$ is singular. An alternative approach is to use the _singular +value decomposition_. Using the definition of the Moore-Penrose +pseudoinverse we can write the equation for $\bm{\beta}$ as + +!bt +\[ + \bm{\beta} = \bm{X}^{+}\bm{y}, +\] +!et + +where the pseudoinverse of $\bm{X}$ is given by + +!bt +\[ + \bm{X}^{+} = \frac{\bm{X}^T}{\bm{X}^T\bm{X}}. +\] +!et + +Using singular value decomposition we can decompose the matrix $\bm{X} = \bm{U}\bm{\Sigma} \bm{V}^T$, +where $\bm{U}$ and $\bm{V}$ are orthogonal(unitary) matrices and $\bm{\Sigma}$ contains the singular values (more details below). +where $X^{+} = V\Sigma^{+} U^T$. This reduces the equation for +$\omega$ to +!bt +\begin{align} + \bm{\beta} = \bm{V}\bm{\Sigma}^{+} \bm{U}^T \bm{y}. +\end{align} +!et + +Note that solving this equation by actually doing the pseudoinverse +(which is what we will do) is not a good idea as this operation scales +as $\mathcal{O}(n^3)$, where $n$ is the number of elements in a +general matrix. Instead, doing $QR$-factorization and solving the +linear system as an equation would reduce this down to +$\mathcal{O}(n^2)$ operations. + + +!bc pycod +def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray: + u, s, v = scl.svd(x) + return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y +!ec + +!bc pycod +beta = ols_svd(X_train_own,y_train) +!ec + +When extracting the $J$-matrix we need to make sure that we remove the intercept, as is done here + +!bc pycod +J = beta[1:].reshape(L, L) +!ec + +A way of looking at the coefficients in $J$ is to plot the matrices as images. + + +!bc pycod +fig = plt.figure(figsize=(20, 14)) +im = plt.imshow(J, **cmap_args) +plt.title("OLS", fontsize=18) +plt.xticks(fontsize=18) +plt.yticks(fontsize=18) +cb = fig.colorbar(im) +cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18) +plt.show() +!ec +It is interesting to note that OLS +considers both $J_{j, j + 1} = -0.5$ and $J_{j, j - 1} = -0.5$ as +valid matrix elements for $J$. +In our discussion below on hyperparameters and Ridge and Lasso regression we will see that +this problem can be removed, partly and only with Lasso regression. + +In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD? + + + + + +!split +===== The one-dimensional Ising model ===== + +Let us bring back the Ising model again, but now with an additional +focus on Ridge and Lasso regression as well. We repeat some of the +basic parts of the Ising model and the setup of the training and test +data. The one-dimensional Ising model with nearest neighbor +interaction, no external field and a constant coupling constant $J$ is +given by + +!bt +\begin{align} + H = -J \sum_{k}^L s_k s_{k + 1}, +\end{align} +!et +where $s_i \in \{-1, 1\}$ and $s_{N + 1} = s_1$. The number of spins in the system is determined by $L$. For the one-dimensional system there is no phase transition. + +We will look at a system of $L = 40$ spins with a coupling constant of $J = 1$. To get enough training data we will generate 10000 states with their respective energies. + + +!bc pycod +import numpy as np +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1 import make_axes_locatable +import seaborn as sns +import scipy.linalg as scl +from sklearn.model_selection import train_test_split +import sklearn.linear_model as skl +import tqdm +sns.set(color_codes=True) +cmap_args=dict(vmin=-1., vmax=1., cmap='seismic') + +L = 40 +n = int(1e4) + +spins = np.random.choice([-1, 1], size=(n, L)) +J = 1.0 + +energies = np.zeros(n) + +for i in range(n): + energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1)) +!ec + +A more general form for the one-dimensional Ising model is + +!bt +\begin{align} + H = - \sum_j^L \sum_k^L s_j s_k J_{jk}. +\end{align} +!et + +Here we allow for interactions beyond the nearest neighbors and a more +adaptive coupling matrix. This latter expression can be formulated as +a matrix-product on the form +!bt +\begin{align} + H = X J, +\end{align} +!et + +where $X_{jk} = s_j s_k$ and $J$ is the matrix consisting of the +elements $-J_{jk}$. This form of writing the energy fits perfectly +with the form utilized in linear regression, viz. +!bt +\begin{align} + \bm{y} = \bm{X}\bm{\beta} + \bm{\epsilon}. +\end{align} +!et +We organize the data as we did above +!bc pycod +X = np.zeros((n, L ** 2)) +for i in range(n): + X[i] = np.outer(spins[i], spins[i]).ravel() +y = energies +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.96) + +X_train_own = np.concatenate( + (np.ones(len(X_train))[:, np.newaxis], X_train), + axis=1 +) + +X_test_own = np.concatenate( + (np.ones(len(X_test))[:, np.newaxis], X_test), + axis=1 +) +!ec + +We will do all fitting with _Scikit-Learn_, + +!bc pycod +clf = skl.LinearRegression().fit(X_train, y_train) +!ec +When extracting the $J$-matrix we make sure to remove the intercept +!bc pycod +J_sk = clf.coef_.reshape(L, L) +!ec +And then we plot the results +!bc pycod +fig = plt.figure(figsize=(20, 14)) +im = plt.imshow(J_sk, **cmap_args) +plt.title("LinearRegression from Scikit-learn", fontsize=18) +plt.xticks(fontsize=18) +plt.yticks(fontsize=18) +cb = fig.colorbar(im) +cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18) +plt.show() +!ec +The results perfectly with our previous discussion where we used our own code. + +!split +===== Ridge regression ===== + +Having explored the ordinary least squares we move on to ridge +regression. In ridge regression we include a _regularizer_. This +involves a new cost function which leads to a new estimate for the +weights $\bm{\beta}$. This results in a penalized regression problem. The +cost function is given by + +!bt +\begin{align} + C(\bm{X}, \bm{\beta}; \lambda) = (\bm{X}\bm{\beta} - \bm{y})^T(\bm{X}\bm{\beta} - \bm{y}) + \lambda \bm{\beta}^T\bm{\beta}. +\end{align} +!et +!bc pycod +_lambda = 0.1 +clf_ridge = skl.Ridge(alpha=_lambda).fit(X_train, y_train) +J_ridge_sk = clf_ridge.coef_.reshape(L, L) +fig = plt.figure(figsize=(20, 14)) +im = plt.imshow(J_ridge_sk, **cmap_args) +plt.title("Ridge from Scikit-learn", fontsize=18) +plt.xticks(fontsize=18) +plt.yticks(fontsize=18) +cb = fig.colorbar(im) +cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18) + +plt.show() +!ec + +!split +===== LASSO regression ===== + +In the _Least Absolute Shrinkage and Selection Operator_ (LASSO)-method we get a third cost function. + +!bt +\begin{align} + C(\bm{X}, \bm{\beta}; \lambda) = (\bm{X}\bm{\beta} - \bm{y})^T(\bm{X}\bm{\beta} - \bm{y}) + \lambda \sqrt{\bm{\beta}^T\bm{\beta}}. +\end{align} +!et + +Finding the extremal point of this cost function is not so straight-forward as in least squares and ridge. We will therefore rely solely on the function ``Lasso`` from _Scikit-Learn_. + +!bc pycod +clf_lasso = skl.Lasso(alpha=_lambda).fit(X_train, y_train) +J_lasso_sk = clf_lasso.coef_.reshape(L, L) +fig = plt.figure(figsize=(20, 14)) +im = plt.imshow(J_lasso_sk, **cmap_args) +plt.title("Lasso from Scikit-learn", fontsize=18) +plt.xticks(fontsize=18) +plt.yticks(fontsize=18) +cb = fig.colorbar(im) +cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18) + +plt.show() +!ec + +It is quite striking how LASSO breaks the symmetry of the coupling +constant as opposed to ridge and OLS. We get a sparse solution with +$J_{j, j + 1} = -1$. + + + +!split +===== Performance as function of the regularization parameter ===== + +We see how the different models perform for a different set of values for $\lambda$. + + +!bc pycod +lambdas = np.logspace(-4, 5, 10) + +train_errors = { + "ols_sk": np.zeros(lambdas.size), + "ridge_sk": np.zeros(lambdas.size), + "lasso_sk": np.zeros(lambdas.size) +} + +test_errors = { + "ols_sk": np.zeros(lambdas.size), + "ridge_sk": np.zeros(lambdas.size), + "lasso_sk": np.zeros(lambdas.size) +} + +plot_counter = 1 + +fig = plt.figure(figsize=(32, 54)) + +for i, _lambda in enumerate(tqdm.tqdm(lambdas)): + for key, method in zip( + ["ols_sk", "ridge_sk", "lasso_sk"], + [skl.LinearRegression(), skl.Ridge(alpha=_lambda), skl.Lasso(alpha=_lambda)] + ): + method = method.fit(X_train, y_train) + + train_errors[key][i] = method.score(X_train, y_train) + test_errors[key][i] = method.score(X_test, y_test) + + omega = method.coef_.reshape(L, L) + + plt.subplot(10, 5, plot_counter) + plt.imshow(omega, **cmap_args) + plt.title(r"%s, $\lambda = %.4f$" % (key, _lambda)) + plot_counter += 1 + +plt.show() +!ec + +We see that LASSO reaches a good solution for low +values of $\lambda$, but will "wither" when we increase $\lambda$ too +much. Ridge is more stable over a larger range of values for +$\lambda$, but eventually also fades away. + +!split +===== Finding the optimal value of $\lambda$ ===== + +To determine which value of $\lambda$ is best we plot the accuracy of +the models when predicting the training and the testing set. We expect +the accuracy of the training set to be quite good, but if the accuracy +of the testing set is much lower this tells us that we might be +subject to an overfit model. The ideal scenario is an accuracy on the +testing set that is close to the accuracy of the training set. + + +!bc pycod +fig = plt.figure(figsize=(20, 14)) + +colors = { + "ols_sk": "r", + "ridge_sk": "y", + "lasso_sk": "c" +} + +for key in train_errors: + plt.semilogx( + lambdas, + train_errors[key], + colors[key], + label="Train {0}".format(key), + linewidth=4.0 + ) + +for key in test_errors: + plt.semilogx( + lambdas, + test_errors[key], + colors[key] + "--", + label="Test {0}".format(key), + linewidth=4.0 + ) +plt.legend(loc="best", fontsize=18) +plt.xlabel(r"$\lambda$", fontsize=18) +plt.ylabel(r"$R^2$", fontsize=18) +plt.tick_params(labelsize=18) +plt.show() +!ec + +From the above figure we can see that LASSO with $\lambda = 10^{-2}$ +achieves a very good accuracy on the test set. This by far surpasses the +other models for all values of $\lambda$. + + + + + + + + + + +!split +===== Friday September 18: Intro to Logistic Regression ===== + +"Video of Lecture":"https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureSeptember18.mp4?vrtx=view-as-webpage" and "link to handwritten notes":"https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesSeptember18.pdf". + + + +!split +===== Logistic Regression ===== + +In linear regression our main interest was centered on learning the +coefficients of a functional fit (say a polynomial) in order to be +able to predict the response of a continuous variable on some unseen +data. The fit to the continuous variable $y_i$ is based on some +independent variables $\hat{x}_i$. Linear regression resulted in +analytical expressions for standard ordinary Least Squares or Ridge +regression (in terms of matrices to invert) for several quantities, +ranging from the variance and thereby the confidence intervals of the +parameters $\hat{\beta}$ to the mean squared error. If we can invert +the product of the design matrices, linear regression gives then a +simple recipe for fitting our data. + +!split +===== Classification problems ===== + + +Classification problems, however, are concerned with outcomes taking +the form of discrete variables (i.e. categories). We may for example, +on the basis of DNA sequencing for a number of patients, like to find +out which mutations are important for a certain disease; or based on +scans of various patients' brains, figure out if there is a tumor or +not; or given a specific physical system, we'd like to identify its +state, say whether it is an ordered or disordered system (typical +situation in solid state physics); or classify the status of a +patient, whether she/he has a stroke or not and many other similar +situations. + +The most common situation we encounter when we apply logistic +regression is that of two possible outcomes, normally denoted as a +binary outcome, true or false, positive or negative, success or +failure etc. + +!split +===== Optimization and Deep learning ===== + +Logistic regression will also serve as our stepping stone towards +neural network algorithms and supervised deep learning. For logistic +learning, the minimization of the cost function leads to a non-linear +equation in the parameters $\hat{\beta}$. The optimization of the +problem calls therefore for minimization algorithms. This forms the +bottle neck of all machine learning algorithms, namely how to find +reliable minima of a multi-variable function. This leads us to the +family of gradient descent methods. The latter are the working horses +of basically all modern machine learning algorithms. + +We note also that many of the topics discussed here on logistic +regression are also commonly used in modern supervised Deep Learning +models, as we will see later. + + +!split +===== Basics ===== + +We consider the case where the dependent variables, also called the +responses or the outcomes, $y_i$ are discrete and only take values +from $k=0,\dots,K-1$ (i.e. $K$ classes). + +The goal is to predict the +output classes from the design matrix $\hat{X}\in\mathbb{R}^{n\times p}$ +made of $n$ samples, each of which carries $p$ features or predictors. The +primary goal is to identify the classes to which new unseen samples +belong. + +Let us specialize to the case of two classes only, with outputs +$y_i=0$ and $y_i=1$. Our outcomes could represent the status of a +credit card user that could default or not on her/his credit card +debt. That is + + +!bt +\[ +y_i = \begin{bmatrix} 0 & \mathrm{no}\\ 1 & \mathrm{yes} \end{bmatrix}. +\] +!et + + + +!split +===== Linear classifier ===== + +Before moving to the logistic model, let us try to use our linear +regression model to classify these two outcomes. We could for example +fit a linear model to the default case if $y_i > 0.5$ and the no +default case $y_i \leq 0.5$. + +We would then have our +weighted linear combination, namely +!bt +\begin{equation} +\hat{y} = \hat{X}^T\hat{\beta} + \hat{\epsilon}, +\end{equation} +!et +where $\hat{y}$ is a vector representing the possible outcomes, $\hat{X}$ is our +$n\times p$ design matrix and $\hat{\beta}$ represents our estimators/predictors. + +!split +===== Some selected properties ===== + +The main problem with our function is that it takes values on the +entire real axis. In the case of logistic regression, however, the +labels $y_i$ are discrete variables. A typical example is the credit +card data discussed below here, where we can set the state of +defaulting the debt to $y_i=1$ and not to $y_i=0$ for one the persons +in the data set (see the full example below). + +One simple way to get a discrete output is to have sign +functions that map the output of a linear regressor to values $\{0,1\}$, +$f(s_i)=sign(s_i)=1$ if $s_i\ge 0$ and 0 if otherwise. +We will encounter this model in our first demonstration of neural networks. Historically it is called the ``perceptron" model in the machine learning +literature. This model is extremely simple. However, in many cases it is more +favorable to use a ``soft" classifier that outputs +the probability of a given category. This leads us to the logistic function. + +!split +===== Simple example ===== + +The following example on data for coronary heart disease (CHD) as function of age may serve as an illustration. In the code here we read and plot whether a person has had CHD (output = 1) or not (output = 0). This ouput is plotted the person's against age. Clearly, the figure shows that attempting to make a standard linear regression fit may not be very meaningful. + +!bc pycod +# Common imports +import os +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.linear_model import LinearRegression, Ridge, Lasso +from sklearn.model_selection import train_test_split +from sklearn.utils import resample +from sklearn.metrics import mean_squared_error +from IPython.display import display +from pylab import plt, mpl +plt.style.use('seaborn') +mpl.rcParams['font.family'] = 'serif' + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("chddata.csv"),'r') + +# Read the chd data as csv file and organize the data into arrays with age group, age, and chd +chd = pd.read_csv(infile, names=('ID', 'Age', 'Agegroup', 'CHD')) +chd.columns = ['ID', 'Age', 'Agegroup', 'CHD'] +output = chd['CHD'] +age = chd['Age'] +agegroup = chd['Agegroup'] +numberID = chd['ID'] +display(chd) + +plt.scatter(age, output, marker='o') +plt.axis([18,70.0,-0.1, 1.2]) +plt.xlabel(r'Age') +plt.ylabel(r'CHD') +plt.title(r'Age distribution and Coronary heart disease') +plt.show() +!ec + +!split +===== Plotting the mean value for each group ===== + +What we could attempt however is to plot the mean value for each group. + +!bc pycod +agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800]) +group = np.array([1, 2, 3, 4, 5, 6, 7, 8]) +plt.plot(group, agegroupmean, "r-") +plt.axis([0,9,0, 1.0]) +plt.xlabel(r'Age group') +plt.ylabel(r'CHD mean values') +plt.title(r'Mean values for each age group') +plt.show() +!ec + +We are now trying to find a function $f(y\vert x)$, that is a function which gives us an expected value for the output $y$ with a given input $x$. +In standard linear regression with a linear dependence on $x$, we would write this in terms of our model +!bt +\[ +f(y_i\vert x_i)=\beta_0+\beta_1 x_i. +\] +!et + +This expression implies however that $f(y_i\vert x_i)$ could take any +value from minus infinity to plus infinity. If we however let +$f(y\vert y)$ be represented by the mean value, the above example +shows us that we can constrain the function to take values between +zero and one, that is we have $0 \le f(y_i\vert x_i) \le 1$. Looking +at our last curve we see also that it has an S-shaped form. This leads +us to a very popular model for the function $f$, namely the so-called +Sigmoid function or logistic model. We will consider this function as +representing the probability for finding a value of $y_i$ with a given +$x_i$. + +!split +===== The logistic function ===== + +Another widely studied model, is the so-called +perceptron model, which is an example of a ``hard classification'' model. We +will encounter this model when we discuss neural networks as +well. Each datapoint is deterministically assigned to a category (i.e +$y_i=0$ or $y_i=1$). In many cases, and the coronary heart disease data forms one of many such examples, it is favorable to have a ``soft'' +classifier that outputs the probability of a given category rather +than a single value. For example, given $x_i$, the classifier +outputs the probability of being in a category $k$. Logistic regression +is the most common example of a so-called soft classifier. In logistic +regression, the probability that a data point $x_i$ +belongs to a category $y_i=\{0,1\}$ is given by the so-called logit function (or Sigmoid) which is meant to represent the likelihood for a given event, +!bt +\[ +p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}. +\] +!et +Note that $1-p(t)= p(-t)$. + +!split +===== Examples of likelihood functions used in logistic regression and nueral networks ===== + + +The following code plots the logistic function, the step function and other functions we will encounter from here and on. + + +!bc pycod +"""The sigmoid function (or the logistic curve) is a +function that takes any real number, z, and outputs a number (0,1). +It is useful in neural networks for assigning weights on a relative scale. +The value z is the weighted sum of parameters involved in the learning algorithm.""" + +import numpy +import matplotlib.pyplot as plt +import math as mt + +z = numpy.arange(-5, 5, .1) +sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z))) +sigma = sigma_fn(z) + +fig = plt.figure() +ax = fig.add_subplot(111) +ax.plot(z, sigma) +ax.set_ylim([-0.1, 1.1]) +ax.set_xlim([-5,5]) +ax.grid(True) +ax.set_xlabel('z') +ax.set_title('sigmoid function') + +plt.show() + +"""Step Function""" +z = numpy.arange(-5, 5, .02) +step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0) +step = step_fn(z) + +fig = plt.figure() +ax = fig.add_subplot(111) +ax.plot(z, step) +ax.set_ylim([-0.5, 1.5]) +ax.set_xlim([-5,5]) +ax.grid(True) +ax.set_xlabel('z') +ax.set_title('step function') + +plt.show() + +"""tanh Function""" +z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1) +t = numpy.tanh(z) + +fig = plt.figure() +ax = fig.add_subplot(111) +ax.plot(z, t) +ax.set_ylim([-1.0, 1.0]) +ax.set_xlim([-2*mt.pi,2*mt.pi]) +ax.grid(True) +ax.set_xlabel('z') +ax.set_title('tanh function') + +plt.show() +!ec + + + + + + + +!split +===== Two parameters ===== + +We assume now that we have two classes with $y_i$ either $0$ or $1$. Furthermore we assume also that we have only two parameters $\beta$ in our fitting of the Sigmoid function, that is we define probabilities +!bt +\begin{align*} +p(y_i=1|x_i,\hat{\beta}) &= \frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}},\nonumber\\ +p(y_i=0|x_i,\hat{\beta}) &= 1 - p(y_i=1|x_i,\hat{\beta}), +\end{align*} +!et +where $\hat{\beta}$ are the weights we wish to extract from data, in our case $\beta_0$ and $\beta_1$. + +Note that we used +!bt +\[ +p(y_i=0\vert x_i, \hat{\beta}) = 1-p(y_i=1\vert x_i, \hat{\beta}). +\] +!et + +!split +===== Maximum likelihood ===== + +In order to define the total likelihood for all possible outcomes from a +dataset $\mathcal{D}=\{(y_i,x_i)\}$, with the binary labels +$y_i\in\{0,1\}$ and where the data points are drawn independently, we use the so-called "Maximum Likelihood Estimation":"https://en.wikipedia.org/wiki/Maximum_likelihood_estimation" (MLE) principle. +We aim thus at maximizing +the probability of seeing the observed data. We can then approximate the +likelihood in terms of the product of the individual probabilities of a specific outcome $y_i$, that is +!bt +\begin{align*} +P(\mathcal{D}|\hat{\beta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\hat{\beta})\right]^{y_i}\left[1-p(y_i=1|x_i,\hat{\beta}))\right]^{1-y_i}\nonumber \\ +\end{align*} +!et +from which we obtain the log-likelihood and our _cost/loss_ function +!bt +\[ +\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left( y_i\log{p(y_i=1|x_i,\hat{\beta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\hat{\beta}))\right]\right). +\] +!et + +!split +===== The cost function rewritten ===== + +Reordering the logarithms, we can rewrite the _cost/loss_ function as +!bt +\[ +\mathcal{C}(\hat{\beta}) = \sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right). +\] +!et + +The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to $\beta$. +Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that +!bt +\[ +\mathcal{C}(\hat{\beta})=-\sum_{i=1}^n \left(y_i(\beta_0+\beta_1x_i) -\log{(1+\exp{(\beta_0+\beta_1x_i)})}\right). +\] +!et +This equation is known in statistics as the _cross entropy_. Finally, we note that just as in linear regression, +in practice we often supplement the cross-entropy with additional regularization terms, usually $L_1$ and $L_2$ regularization as we did for Ridge and Lasso regression. + +!split +===== Minimizing the cross entropy ===== + +The cross entropy is a convex function of the weights $\hat{\beta}$ and, +therefore, any local minimizer is a global minimizer. + + +Minimizing this +cost function with respect to the two parameters $\beta_0$ and $\beta_1$ we obtain + +!bt +\[ +\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right), +\] +!et +and +!bt +\[ +\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \beta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\beta_0+\beta_1x_i)}}{1+\exp{(\beta_0+\beta_1x_i)}}\right). +\] +!et + +!split +===== A more compact expression ===== + +Let us now define a vector $\hat{y}$ with $n$ elements $y_i$, an +$n\times p$ matrix $\hat{X}$ which contains the $x_i$ values and a +vector $\hat{p}$ of fitted probabilities $p(y_i\vert x_i,\hat{\beta})$. We can rewrite in a more compact form the first +derivative of cost function as + +!bt +\[ +\frac{\partial \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}} = -\hat{X}^T\left(\hat{y}-\hat{p}\right). +\] +!et + +If we in addition define a diagonal matrix $\hat{W}$ with elements +$p(y_i\vert x_i,\hat{\beta})(1-p(y_i\vert x_i,\hat{\beta})$, we can obtain a compact expression of the second derivative as + +!bt +\[ +\frac{\partial^2 \mathcal{C}(\hat{\beta})}{\partial \hat{\beta}\partial \hat{\beta}^T} = \hat{X}^T\hat{W}\hat{X}. +\] +!et + +!split +===== Extending to more predictors ===== + +Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with $p$ predictors +!bt +\[ +\log{ \frac{p(\hat{\beta}\hat{x})}{1-p(\hat{\beta}\hat{x})}} = \beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p. +\] +!et +Here we defined $\hat{x}=[1,x_1,x_2,\dots,x_p]$ and $\hat{\beta}=[\beta_0, \beta_1, \dots, \beta_p]$ leading to +!bt +\[ +p(\hat{\beta}\hat{x})=\frac{ \exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}{1+\exp{(\beta_0+\beta_1x_1+\beta_2x_2+\dots+\beta_px_p)}}. +\] +!et + +!split +===== Including more classes ===== + +Till now we have mainly focused on two classes, the so-called binary +system. Suppose we wish to extend to $K$ classes. Let us for the sake +of simplicity assume we have only two predictors. We have then following model + +!bt +\[ +\log{\frac{p(C=1\vert x)}{p(K\vert x)}} = \beta_{10}+\beta_{11}x_1, +\] +!et +and +!bt +\[ +\log{\frac{p(C=2\vert x)}{p(K\vert x)}} = \beta_{20}+\beta_{21}x_1, +\] +!et +and so on till the class $C=K-1$ class +!bt +\[ +\log{\frac{p(C=K-1\vert x)}{p(K\vert x)}} = \beta_{(K-1)0}+\beta_{(K-1)1}x_1, +\] +!et + +and the model is specified in term of $K-1$ so-called log-odds or +_logit_ transformations. + + +!split +===== More classes ===== + +In our discussion of neural networks we will encounter the above again +in terms of a slightly modified function, the so-called _Softmax_ function. + +The softmax function is used in various multiclass classification +methods, such as multinomial logistic regression (also known as +softmax regression), multiclass linear discriminant analysis, naive +Bayes classifiers, and artificial neural networks. Specifically, in +multinomial logistic regression and linear discriminant analysis, the +input to the function is the result of $K$ distinct linear functions, +and the predicted probability for the $k$-th class given a sample +vector $\hat{x}$ and a weighting vector $\hat{\beta}$ is (with two +predictors): + +!bt +\[ +p(C=k\vert \mathbf {x} )=\frac{\exp{(\beta_{k0}+\beta_{k1}x_1)}}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}}. +\] +!et +It is easy to extend to more predictors. The final class is +!bt +\[ +p(C=K\vert \mathbf {x} )=\frac{1}{1+\sum_{l=1}^{K-1}\exp{(\beta_{l0}+\beta_{l1}x_1)}}, +\] +!et + +and they sum to one. Our earlier discussions were all specialized to +the case with two classes only. It is easy to see from the above that +what we derived earlier is compatible with these equations. + +To find the optimal parameters we would typically use a gradient +descent method. Newton's method and gradient descent methods are +discussed in the material on "optimization +methods":"https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html". + +This will be discussed next week. Before we develop our own codes for logistic regression, we end this lecture by studying the functionality that _Scikit-learn_ offers. + + + + + +!split +===== Wisconsin Cancer Data ===== + +We show here how we can use a simple regression case on the breast +cancer data using Logistic regression as our algorithm for +classification. + + +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +# Logistic Regression +logreg = LogisticRegression(solver='lbfgs') +logreg.fit(X_train, y_train) +print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test))) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Logistic Regression +logreg.fit(X_train_scaled, y_train) +print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) +!ec + +!split +===== Using the correlation matrix ===== + +In addition to the above scores, we could also study the covariance (and the correlation matrix). +We use _Pandas_ to compute the correlation matrix. +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression +cancer = load_breast_cancer() +import pandas as pd +# Making a data frame +cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names) + +fig, axes = plt.subplots(15,2,figsize=(10,20)) +malignant = cancer.data[cancer.target == 0] +benign = cancer.data[cancer.target == 1] +ax = axes.ravel() + +for i in range(30): + _, bins = np.histogram(cancer.data[:,i], bins =50) + ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5) + ax[i].hist(benign[:,i], bins = bins, alpha = 0.5) + ax[i].set_title(cancer.feature_names[i]) + ax[i].set_yticks(()) +ax[0].set_xlabel("Feature magnitude") +ax[0].set_ylabel("Frequency") +ax[0].legend(["Malignant", "Benign"], loc ="best") +fig.tight_layout() +plt.show() + +import seaborn as sns +correlation_matrix = cancerpd.corr().round(1) +# use the heatmap function from seaborn to plot the correlation matrix +# annot = True to print the values inside the square +plt.figure(figsize=(15,8)) +sns.heatmap(data=correlation_matrix, annot=True) +plt.show() + + +!ec + +!split +===== Discussing the correlation data ===== + +In the above example we note two things. In the first plot we display +the overlap of benign and malignant tumors as functions of the various +features in the Wisconsing breast cancer data set. We see that for +some of the features we can distinguish clearly the benign and +malignant cases while for other features we cannot. This can point to +us which features may be of greater interest when we wish to classify +a benign or not benign tumour. + +In the second figure we have computed the so-called correlation +matrix, which in our case with thirty features becomes a $30\times 30$ +matrix. + +We constructed this matrix using _pandas_ via the statements +!bc pycod +cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names) +!ec +and then +!bc pycod +correlation_matrix = cancerpd.corr().round(1) +!ec + +Diagonalizing this matrix we can in turn say something about which +features are of relevance and which are not. This leads us to +the classical Principal Component Analysis (PCA) theorem with +applications. This will be discussed later this semester ("week 43":"https://compphysics.github.io/MachineLearning/doc/pub/week43/html/week43-bs.html"). + + + +!split +===== Other measures in classification studies: Cancer Data again ===== +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.linear_model import LogisticRegression + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +# Logistic Regression +logreg = LogisticRegression(solver='lbfgs') +logreg.fit(X_train, y_train) +print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test))) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Logistic Regression +logreg.fit(X_train_scaled, y_train) +print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) + + +from sklearn.preprocessing import LabelEncoder +from sklearn.model_selection import cross_validate +#Cross validation +accuracy = cross_validate(logreg,X_test_scaled,y_test,cv=10)['test_score'] +print(accuracy) +print("Test set accuracy with Logistic Regression and scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) + + +import scikitplot as skplt +y_pred = logreg.predict(X_test_scaled) +skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True) +plt.show() +y_probas = logreg.predict_proba(X_test_scaled) +skplt.metrics.plot_roc(y_test, y_probas) +plt.show() +skplt.metrics.plot_cumulative_gain(y_test, y_probas) +plt.show() + +!ec + + + + + + + + + + + diff --git a/doc/LectureNotes/.DS_Store b/doc/LectureNotes/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/doc/LectureNotes/.DS_Store differ diff --git a/doc/LectureNotes/chapter1.ipynb b/doc/LectureNotes/chapter1.ipynb new file mode 100644 index 000000000..7e2ad50e8 --- /dev/null +++ b/doc/LectureNotes/chapter1.ipynb @@ -0,0 +1,4066 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Linear Regression, basic Elements\n", + "\n", + "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureAug21.mp4?vrtx=view-as-webpage)\n", + "\n", + "\n", + "## Introduction\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Our emphasis throughout this series of lectures \n", + "is on understanding the mathematical aspects of\n", + "different algorithms used in the fields of data analysis and machine learning. \n", + "\n", + "However, where possible we will emphasize the\n", + "importance of using available software. We start thus with a hands-on\n", + "and top-down approach to machine learning. The aim is thus to start with\n", + "relevant data or data we have produced \n", + "and use these to introduce statistical data analysis\n", + "concepts and machine learning algorithms before we delve into the\n", + "algorithms themselves. The examples we will use in the beginning, start with simple\n", + "polynomials with random noise added. We will use the Python\n", + "software package [Scikit-Learn](http://scikit-learn.org/stable/) and\n", + "introduce various machine learning algorithms to make fits of\n", + "the data and predictions. We move thereafter to more interesting\n", + "cases such as data from say experiments (below we will look at experimental nuclear binding energies as an example).\n", + "These are examples where we can easily set up the data and\n", + "then use machine learning algorithms included in for example\n", + "**Scikit-Learn**. \n", + "\n", + "These examples will serve us the purpose of getting\n", + "started. Furthermore, they allow us to catch more than two birds with\n", + "a stone. They will allow us to bring in some programming specific\n", + "topics and tools as well as showing the power of various Python \n", + "libraries for machine learning and statistical data analysis. \n", + "\n", + "Here, we will mainly focus on two\n", + "specific Python packages for Machine Learning, Scikit-Learn and\n", + "Tensorflow (see below for links etc). Moreover, the examples we\n", + "introduce will serve as inputs to many of our discussions later, as\n", + "well as allowing you to set up models and produce your own data and\n", + "get started with programming.\n", + "\n", + "\n", + "\n", + "## What is Machine Learning?\n", + "\n", + "Statistics, data science and machine learning form important fields of\n", + "research in modern science. They describe how to learn and make\n", + "predictions from data, as well as allowing us to extract important\n", + "correlations about physical process and the underlying laws of motion\n", + "in large data sets. The latter, big data sets, appear frequently in\n", + "essentially all disciplines, from the traditional Science, Technology,\n", + "Mathematics and Engineering fields to Life Science, Law, education\n", + "research, the Humanities and the Social Sciences. \n", + "\n", + "It has become more\n", + "and more common to see research projects on big data in for example\n", + "the Social Sciences where extracting patterns from complicated survey\n", + "data is one of many research directions. Having a solid grasp of data\n", + "analysis and machine learning is thus becoming central to scientific\n", + "computing in many fields, and competences and skills within the fields\n", + "of machine learning and scientific computing are nowadays strongly\n", + "requested by many potential employers. The latter cannot be\n", + "overstated, familiarity with machine learning has almost become a\n", + "prerequisite for many of the most exciting employment opportunities,\n", + "whether they are in bioinformatics, life science, physics or finance,\n", + "in the private or the public sector. This author has had several\n", + "students or met students who have been hired recently based on their\n", + "skills and competences in scientific computing and data science, often\n", + "with marginal knowledge of machine learning.\n", + "\n", + "Machine learning is a subfield of computer science, and is closely\n", + "related to computational statistics. It evolved from the study of\n", + "pattern recognition in artificial intelligence (AI) research, and has\n", + "made contributions to AI tasks like computer vision, natural language\n", + "processing and speech recognition. Many of the methods we will study are also \n", + "strongly rooted in basic mathematics and physics research. \n", + "\n", + "Ideally, machine learning represents the science of giving computers\n", + "the ability to learn without being explicitly programmed. The idea is\n", + "that there exist generic algorithms which can be used to find patterns\n", + "in a broad class of data sets without having to write code\n", + "specifically for each problem. The algorithm will build its own logic\n", + "based on the data. You should however always keep in mind that\n", + "machines and algorithms are to a large extent developed by humans. The\n", + "insights and knowledge we have about a specific system, play a central\n", + "role when we develop a specific machine learning algorithm. \n", + "\n", + "Machine learning is an extremely rich field, in spite of its young\n", + "age. The increases we have seen during the last three decades in\n", + "computational capabilities have been followed by developments of\n", + "methods and techniques for analyzing and handling large date sets,\n", + "relying heavily on statistics, computer science and mathematics. The\n", + "field is rather new and developing rapidly. Popular software packages\n", + "written in Python for machine learning like\n", + "[Scikit-learn](http://scikit-learn.org/stable/),\n", + "[Tensorflow](https://www.tensorflow.org/),\n", + "[PyTorch](http://pytorch.org/) and [Keras](https://keras.io/), all\n", + "freely available at their respective GitHub sites, encompass\n", + "communities of developers in the thousands or more. And the number of\n", + "code developers and contributors keeps increasing. Not all the\n", + "algorithms and methods can be given a rigorous mathematical\n", + "justification, opening up thereby large rooms for experimenting and\n", + "trial and error and thereby exciting new developments. However, a\n", + "solid command of linear algebra, multivariate theory, probability\n", + "theory, statistical data analysis, understanding errors and Monte\n", + "Carlo methods are central elements in a proper understanding of many\n", + "of algorithms and methods we will discuss.\n", + "\n", + "\n", + "\n", + "The approaches to machine learning are many, but are often split into\n", + "two main categories. In *supervised learning* we know the answer to a\n", + "problem, and let the computer deduce the logic behind it. On the other\n", + "hand, *unsupervised learning* is a method for finding patterns and\n", + "relationship in data sets without any prior knowledge of the system.\n", + "Some authours also operate with a third category, namely\n", + "*reinforcement learning*. This is a paradigm of learning inspired by\n", + "behavioral psychology, where learning is achieved by trial-and-error,\n", + "solely from rewards and punishment.\n", + "\n", + "Another way to categorize machine learning tasks is to consider the\n", + "desired output of a system. Some of the most common tasks are:\n", + "\n", + " * Classification: Outputs are divided into two or more classes. The goal is to produce a model that assigns inputs into one of these classes. An example is to identify digits based on pictures of hand-written ones. Classification is typically supervised learning.\n", + "\n", + " * Regression: Finding a functional relationship between an input data set and a reference data set. The goal is to construct a function that maps input data to continuous output values.\n", + "\n", + " * Clustering: Data are divided into groups with certain common traits, without knowing the different groups beforehand. It is thus a form of unsupervised learning.\n", + "\n", + "The methods we cover have three main topics in common, irrespective of\n", + "whether we deal with supervised or unsupervised learning. The first\n", + "ingredient is normally our data set (which can be subdivided into\n", + "training and test data), the second item is a model which is normally a\n", + "function of some parameters. The model reflects our knowledge of the system (or lack thereof). As an example, if we know that our data show a behavior similar to what would be predicted by a polynomial, fitting our data to a polynomial of some degree would then determin our model. \n", + "\n", + "The last ingredient is a so-called **cost**\n", + "function which allows us to present an estimate on how good our model\n", + "is in reproducing the data it is supposed to train. \n", + "At the heart of basically all ML algorithms there are so-called minimization algorithms, often we end up with various variants of **gradient** methods.\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "## Software and needed installations\n", + "\n", + "We will make extensive use of Python as programming language and its\n", + "myriad of available libraries. You will find\n", + "Jupyter notebooks invaluable in your work. You can run **R**\n", + "codes in the Jupyter/IPython notebooks, with the immediate benefit of\n", + "visualizing your data. You can also use compiled languages like C++,\n", + "Rust, Julia, Fortran etc if you prefer. The focus in these lectures will be\n", + "on Python.\n", + "\n", + "\n", + "If you have Python installed (we strongly recommend Python3) and you feel\n", + "pretty familiar with installing different packages, we recommend that\n", + "you install the following Python packages via **pip** as \n", + "\n", + "1. pip install numpy scipy matplotlib ipython scikit-learn mglearn sympy pandas pillow \n", + "\n", + "For Python3, replace **pip** with **pip3**.\n", + "\n", + "For OSX users we recommend, after having installed Xcode, to\n", + "install **brew**. Brew allows for a seamless installation of additional\n", + "software via for example \n", + "\n", + "1. brew install python3\n", + "\n", + "For Linux users, with its variety of distributions like for example the widely popular Ubuntu distribution,\n", + "you can use **pip** as well and simply install Python as \n", + "\n", + "1. sudo apt-get install python3 (or python for pyhton2.7)\n", + "\n", + "etc etc. \n", + "\n", + "\n", + "\n", + "## Python installers\n", + "\n", + "If you don't want to perform these operations separately and venture\n", + "into the hassle of exploring how to set up dependencies and paths, we\n", + "recommend two widely used distrubutions which set up all relevant\n", + "dependencies for Python, namely \n", + "\n", + "* [Anaconda](https://docs.anaconda.com/), \n", + "\n", + "which is an open source\n", + "distribution of the Python and R programming languages for large-scale\n", + "data processing, predictive analytics, and scientific computing, that\n", + "aims to simplify package management and deployment. Package versions\n", + "are managed by the package management system **conda**. \n", + "\n", + "* [Enthought canopy](https://www.enthought.com/product/canopy/) \n", + "\n", + "is a Python\n", + "distribution for scientific and analytic computing distribution and\n", + "analysis environment, available for free and under a commercial\n", + "license.\n", + "\n", + "Furthermore, [Google's Colab](https://colab.research.google.com/notebooks/welcome.ipynb) is a free Jupyter notebook environment that requires \n", + "no setup and runs entirely in the cloud. Try it out!\n", + "\n", + "\n", + "## Useful Python libraries\n", + "Here we list several useful Python libraries we strongly recommend (if you use anaconda many of these are already there)\n", + "\n", + "* [NumPy](https://www.numpy.org/) is a highly popular library for large, multi-dimensional arrays and matrices, along with a large collection of high-level mathematical functions to operate on these arrays\n", + "\n", + "* [The pandas](https://pandas.pydata.org/) library provides high-performance, easy-to-use data structures and data analysis tools \n", + "\n", + "* [Xarray](http://xarray.pydata.org/en/stable/) is a Python package that makes working with labelled multi-dimensional arrays simple, efficient, and fun!\n", + "\n", + "* [Scipy](https://www.scipy.org/) (pronounced “Sigh Pie”) is a Python-based ecosystem of open-source software for mathematics, science, and engineering. \n", + "\n", + "* [Matplotlib](https://matplotlib.org/) is a Python 2D plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms.\n", + "\n", + "* [Autograd](https://github.com/HIPS/autograd) can automatically differentiate native Python and Numpy code. It can handle a large subset of Python's features, including loops, ifs, recursion and closures, and it can even take derivatives of derivatives of derivatives\n", + "\n", + "* [SymPy](https://www.sympy.org/en/index.html) is a Python library for symbolic mathematics. \n", + "\n", + "* [scikit-learn](https://scikit-learn.org/stable/) has simple and efficient tools for machine learning, data mining and data analysis\n", + "\n", + "* [TensorFlow](https://www.tensorflow.org/) is a Python library for fast numerical computing created and released by Google\n", + "\n", + "* [Keras](https://keras.io/) is a high-level neural networks API, written in Python and capable of running on top of TensorFlow, CNTK, or Theano\n", + "\n", + "* And many more such as [pytorch](https://pytorch.org/), [Theano](https://pypi.org/project/Theano/) etc \n", + "\n", + "## Installing R, C++, cython or Julia\n", + "\n", + "You will also find it convenient to utilize **R**. We will mainly\n", + "use Python during our lectures and in various projects and exercises.\n", + "Those of you\n", + "already familiar with **R** should feel free to continue using **R**, keeping\n", + "however an eye on the parallel Python set ups. Similarly, if you are a\n", + "Python afecionado, feel free to explore **R** as well. Jupyter/Ipython\n", + "notebook allows you to run **R** codes interactively in your\n", + "browser. The software library **R** is really tailored for statistical data analysis\n", + "and allows for an easy usage of the tools and algorithms we will discuss in these\n", + "lectures.\n", + "\n", + "To install **R** with Jupyter notebook \n", + "[follow the link here](https://mpacer.org/maths/r-kernel-for-ipython-notebook)\n", + "\n", + "\n", + "\n", + "\n", + "## Installing R, C++, cython, Numba etc\n", + "\n", + "\n", + "For the C++ aficionados, Jupyter/IPython notebook allows you also to\n", + "install C++ and run codes written in this language interactively in\n", + "the browser. Since we will emphasize writing many of the algorithms\n", + "yourself, you can thus opt for either Python or C++ (or Fortran or other compiled languages) as programming\n", + "languages.\n", + "\n", + "To add more entropy, **cython** can also be used when running your\n", + "notebooks. It means that Python with the jupyter notebook\n", + "setup allows you to integrate widely popular softwares and tools for\n", + "scientific computing. Similarly, the \n", + "[Numba Python package](https://numba.pydata.org/) delivers increased performance\n", + "capabilities with minimal rewrites of your codes. With its\n", + "versatility, including symbolic operations, Python offers a unique\n", + "computational environment. Your jupyter notebook can easily be\n", + "converted into a nicely rendered **PDF** file or a Latex file for\n", + "further processing. For example, convert to latex as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " pycod jupyter nbconvert filename.ipynb --to latex \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And to add more versatility, the Python package [SymPy](http://www.sympy.org/en/index.html) is a Python library for symbolic mathematics. It aims to become a full-featured computer algebra system (CAS) and is entirely written in Python. \n", + "\n", + "Finally, if you wish to use the light mark-up language \n", + "[doconce](https://github.com/hplgit/doconce) you can convert a standard ascii text file into various HTML \n", + "formats, ipython notebooks, latex files, pdf files etc with minimal edits. These lectures were generated using **doconce**.\n", + "\n", + "\n", + "\n", + "## Numpy examples and Important Matrix and vector handling packages\n", + "\n", + "There are several central software libraries for linear algebra and eigenvalue problems. Several of the more\n", + "popular ones have been wrapped into ofter software packages like those from the widely used text **Numerical Recipes**. The original source codes in many of the available packages are often taken from the widely used\n", + "software package LAPACK, which follows two other popular packages\n", + "developed in the 1970s, namely EISPACK and LINPACK. We describe them shortly here.\n", + "\n", + " * LINPACK: package for linear equations and least square problems.\n", + "\n", + " * LAPACK:package for solving symmetric, unsymmetric and generalized eigenvalue problems. From LAPACK's website it is possible to download for free all source codes from this library. Both C/C++ and Fortran versions are available.\n", + "\n", + " * BLAS (I, II and III): (Basic Linear Algebra Subprograms) are routines that provide standard building blocks for performing basic vector and matrix operations. Blas I is vector operations, II vector-matrix operations and III matrix-matrix operations. Highly parallelized and efficient codes, all available for download from .\n", + "\n", + "## Basic Matrix Features\n", + "\n", + "Matrix properties reminder" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbf{A} =\n", + " \\begin{bmatrix} a_{11} & a_{12} & a_{13} & a_{14} \\\\\n", + " a_{21} & a_{22} & a_{23} & a_{24} \\\\\n", + " a_{31} & a_{32} & a_{33} & a_{34} \\\\\n", + " a_{41} & a_{42} & a_{43} & a_{44}\n", + " \\end{bmatrix}\\qquad\n", + "\\mathbf{I} =\n", + " \\begin{bmatrix} 1 & 0 & 0 & 0 \\\\\n", + " 0 & 1 & 0 & 0 \\\\\n", + " 0 & 0 & 1 & 0 \\\\\n", + " 0 & 0 & 0 & 1\n", + " \\end{bmatrix}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The inverse of a matrix is defined by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbf{A}^{-1} \\cdot \\mathbf{A} = I\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Relations Name matrix elements
$A = A^{T}$ symmetric $a_{ij} = a_{ji}$
$A = \\left (A^{T} \\right )^{-1}$ real orthogonal $\\sum_k a_{ik} a_{jk} = \\sum_k a_{ki} a_{kj} = \\delta_{ij}$
$A = A^{ * }$ real matrix $a_{ij} = a_{ij}^{ * }$
$A = A^{\\dagger}$ hermitian $a_{ij} = a_{ji}^{ * }$
$A = \\left (A^{\\dagger} \\right )^{-1}$ unitary $\\sum_k a_{ik} a_{jk}^{ * } = \\sum_k a_{ki}^{ * } a_{kj} = \\delta_{ij}$
\n", + "\n", + "\n", + "### Some famous Matrices\n", + "\n", + " * Diagonal if $a_{ij}=0$ for $i\\ne j$\n", + "\n", + " * Upper triangular if $a_{ij}=0$ for $i > j$\n", + "\n", + " * Lower triangular if $a_{ij}=0$ for $i < j$\n", + "\n", + " * Upper Hessenberg if $a_{ij}=0$ for $i > j+1$\n", + "\n", + " * Lower Hessenberg if $a_{ij}=0$ for $i < j+1$\n", + "\n", + " * Tridiagonal if $a_{ij}=0$ for $|i -j| > 1$\n", + "\n", + " * Lower banded with bandwidth $p$: $a_{ij}=0$ for $i > j+p$\n", + "\n", + " * Upper banded with bandwidth $p$: $a_{ij}=0$ for $i < j+p$\n", + "\n", + " * Banded, block upper triangular, block lower triangular....\n", + "\n", + "### More Basic Matrix Features\n", + "\n", + "Some Equivalent Statements\n", + "For an $N\\times N$ matrix $\\mathbf{A}$ the following properties are all equivalent\n", + "\n", + " * If the inverse of $\\mathbf{A}$ exists, $\\mathbf{A}$ is nonsingular.\n", + "\n", + " * The equation $\\mathbf{Ax}=0$ implies $\\mathbf{x}=0$.\n", + "\n", + " * The rows of $\\mathbf{A}$ form a basis of $R^N$.\n", + "\n", + " * The columns of $\\mathbf{A}$ form a basis of $R^N$.\n", + "\n", + " * $\\mathbf{A}$ is a product of elementary matrices.\n", + "\n", + " * $0$ is not eigenvalue of $\\mathbf{A}$.\n", + "\n", + "## Numpy and arrays\n", + "[Numpy](http://www.numpy.org/) provides an easy way to handle arrays in Python. The standard way to import this library is as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here follows a simple example where we set up an array of ten elements, all determined by random numbers drawn according to the normal distribution," + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "n = 10\n", + "x = np.random.normal(size=n)\n", + "print(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We defined a vector $x$ with $n=10$ elements with its values given by the Normal distribution $N(0,1)$.\n", + "Another alternative is to declare a vector as follows" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "x = np.array([1, 2, 3])\n", + "print(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we have defined a vector with three elements, with $x_0=1$, $x_1=2$ and $x_2=3$. Note that both Python and C++\n", + "start numbering array elements from $0$ and on. This means that a vector with $n$ elements has a sequence of entities $x_0, x_1, x_2, \\dots, x_{n-1}$. We could also let (recommended) Numpy to compute the logarithms of a specific array as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "x = np.log(np.array([4, 7, 8]))\n", + "print(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the last example we used Numpy's unary function $np.log$. This function is\n", + "highly tuned to compute array elements since the code is vectorized\n", + "and does not require looping. We normaly recommend that you use the\n", + "Numpy intrinsic functions instead of the corresponding **log** function\n", + "from Python's **math** module. The looping is done explicitely by the\n", + "**np.log** function. The alternative, and slower way to compute the\n", + "logarithms of a vector would be to write" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from math import log\n", + "x = np.array([4, 7, 8])\n", + "for i in range(0, len(x)):\n", + " x[i] = log(x[i])\n", + "print(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We note that our code is much longer already and we need to import the **log** function from the **math** module. \n", + "The attentive reader will also notice that the output is $[1, 1, 2]$. Python interprets automagically our numbers as integers (like the **automatic** keyword in C++). To change this we could define our array elements to be double precision numbers as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "x = np.log(np.array([4, 7, 8], dtype = np.float64))\n", + "print(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or simply write them as double precision numbers (Python uses 64 bits as default for floating point type variables), that is" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "x = np.log(np.array([4.0, 7.0, 8.0])\n", + "print(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To check the number of bytes (remember that one byte contains eight bits for double precision variables), you can use simple use the **itemsize** functionality (the array $x$ is actually an object which inherits the functionalities defined in Numpy) as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "x = np.log(np.array([4.0, 7.0, 8.0])\n", + "print(x.itemsize)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Matrices in Python\n", + "\n", + "Having defined vectors, we are now ready to try out matrices. We can\n", + "define a $3 \\times 3 $ real matrix $\\hat{A}$ as (recall that we user\n", + "lowercase letters for vectors and uppercase letters for matrices)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ]))\n", + "print(A)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we use the **shape** function we would get $(3, 3)$ as output, that is verifying that our matrix is a $3\\times 3$ matrix. We can slice the matrix and print for example the first column (Python organized matrix elements in a row-major order, see below) as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ]))\n", + "# print the first column, row-major order and elements start with 0\n", + "print(A[:,0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can continue this was by printing out other columns or rows. The example here prints out the second column" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "A = np.log(np.array([ [4.0, 7.0, 8.0], [3.0, 10.0, 11.0], [4.0, 5.0, 7.0] ]))\n", + "# print the first column, row-major order and elements start with 0\n", + "print(A[1,:])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Numpy contains many other functionalities that allow us to slice, subdivide etc etc arrays. We strongly recommend that you look up the [Numpy website for more details](http://www.numpy.org/). Useful functions when defining a matrix are the **np.zeros** function which declares a matrix of a given dimension and sets all elements to zero" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "n = 10\n", + "# define a matrix of dimension 10 x 10 and set all elements to zero\n", + "A = np.zeros( (n, n) )\n", + "print(A)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or initializing all elements to" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "n = 10\n", + "# define a matrix of dimension 10 x 10 and set all elements to one\n", + "A = np.ones( (n, n) )\n", + "print(A)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or as unitarily distributed random numbers (see the material on random number generators in the statistics part)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "n = 10\n", + "# define a matrix of dimension 10 x 10 and set all elements to random numbers with x \\in [0, 1]\n", + "A = np.random.rand(n, n)\n", + "print(A)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we will see throughout these lectures, there are several extremely useful functionalities in Numpy.\n", + "As an example, consider the discussion of the covariance matrix. Suppose we have defined three vectors\n", + "$\\hat{x}, \\hat{y}, \\hat{z}$ with $n$ elements each. The covariance matrix is defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\Sigma} = \\begin{bmatrix} \\sigma_{xx} & \\sigma_{xy} & \\sigma_{xz} \\\\\n", + " \\sigma_{yx} & \\sigma_{yy} & \\sigma_{yz} \\\\\n", + " \\sigma_{zx} & \\sigma_{zy} & \\sigma_{zz} \n", + " \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where for example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\sigma_{xy} =\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})(y_i- \\overline{y}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Numpy function **np.cov** calculates the covariance elements using the factor $1/(n-1)$ instead of $1/n$ since it assumes we do not have the exact mean values. \n", + "The following simple function uses the **np.vstack** function which takes each vector of dimension $1\\times n$ and produces a $3\\times n$ matrix $\\hat{W}$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{W} = \\begin{bmatrix} x_0 & y_0 & z_0 \\\\\n", + " x_1 & y_1 & z_1 \\\\\n", + " x_2 & y_2 & z_2 \\\\\n", + " \\dots & \\dots & \\dots \\\\\n", + " x_{n-2} & y_{n-2} & z_{n-2} \\\\\n", + " x_{n-1} & y_{n-1} & z_{n-1}\n", + " \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which in turn is converted into into the $3\\times 3$ covariance matrix\n", + "$\\hat{\\Sigma}$ via the Numpy function **np.cov()**. We note that we can also calculate\n", + "the mean value of each set of samples $\\hat{x}$ etc using the Numpy\n", + "function **np.mean(x)**. We can also extract the eigenvalues of the\n", + "covariance matrix through the **np.linalg.eig()** function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Importing various packages\n", + "import numpy as np\n", + "\n", + "n = 100\n", + "x = np.random.normal(size=n)\n", + "print(np.mean(x))\n", + "y = 4+3*x+np.random.normal(size=n)\n", + "print(np.mean(y))\n", + "z = x**3+np.random.normal(size=n)\n", + "print(np.mean(z))\n", + "W = np.vstack((x, y, z))\n", + "Sigma = np.cov(W)\n", + "print(Sigma)\n", + "Eigvals, Eigvecs = np.linalg.eig(Sigma)\n", + "print(Eigvals)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from scipy import sparse\n", + "eye = np.eye(4)\n", + "print(eye)\n", + "sparse_mtx = sparse.csr_matrix(eye)\n", + "print(sparse_mtx)\n", + "x = np.linspace(-10,10,100)\n", + "y = np.sin(x)\n", + "plt.plot(x,y,marker='x')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Meet the Pandas\n", + "\n", + "\n", + "\n", + "\n", + "Another useful Python package is\n", + "[pandas](https://pandas.pydata.org/), which is an open source library\n", + "providing high-performance, easy-to-use data structures and data\n", + "analysis tools for Python. **pandas** stands for panel data, a term borrowed from econometrics and is an efficient library for data analysis with an emphasis on tabular data.\n", + "**pandas** has two major classes, the **DataFrame** class with two-dimensional data objects and tabular data organized in columns and the class **Series** with a focus on one-dimensional data objects. Both classes allow you to index data easily as we will see in the examples below. \n", + "**pandas** allows you also to perform mathematical operations on the data, spanning from simple reshapings of vectors and matrices to statistical operations. \n", + "\n", + "The following simple example shows how we can, in an easy way make tables of our data. Here we define a data set which includes names, place of birth and date of birth, and displays the data in an easy to read way. We will see repeated use of **pandas**, in particular in connection with classification of data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from IPython.display import display\n", + "data = {'First Name': [\"Frodo\", \"Bilbo\", \"Aragorn II\", \"Samwise\"],\n", + " 'Last Name': [\"Baggins\", \"Baggins\",\"Elessar\",\"Gamgee\"],\n", + " 'Place of birth': [\"Shire\", \"Shire\", \"Eriador\", \"Shire\"],\n", + " 'Date of Birth T.A.': [2968, 2890, 2931, 2980]\n", + " }\n", + "data_pandas = pd.DataFrame(data)\n", + "display(data_pandas)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the above we have imported **pandas** with the shorthand **pd**, the latter has become the standard way we import **pandas**. We make then a list of various variables\n", + "and reorganize the aboves lists into a **DataFrame** and then print out a neat table with specific column labels as *Name*, *place of birth* and *date of birth*.\n", + "Displaying these results, we see that the indices are given by the default numbers from zero to three.\n", + "**pandas** is extremely flexible and we can easily change the above indices by defining a new type of indexing as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "data_pandas = pd.DataFrame(data,index=['Frodo','Bilbo','Aragorn','Sam'])\n", + "display(data_pandas)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Thereafter we display the content of the row which begins with the index **Aragorn**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "display(data_pandas.loc['Aragorn'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can easily append data to this, for example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "new_hobbit = {'First Name': [\"Peregrin\"],\n", + " 'Last Name': [\"Took\"],\n", + " 'Place of birth': [\"Shire\"],\n", + " 'Date of Birth T.A.': [2990]\n", + " }\n", + "data_pandas=data_pandas.append(pd.DataFrame(new_hobbit, index=['Pippin']))\n", + "display(data_pandas)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here are other examples where we use the **DataFrame** functionality to handle arrays, now with more interesting features for us, namely numbers. We set up a matrix \n", + "of dimensionality $10\\times 5$ and compute the mean value and standard deviation of each column. Similarly, we can perform mathematial operations like squaring the matrix elements and many other operations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "from IPython.display import display\n", + "np.random.seed(100)\n", + "# setting up a 10 x 5 matrix\n", + "rows = 10\n", + "cols = 5\n", + "a = np.random.randn(rows,cols)\n", + "df = pd.DataFrame(a)\n", + "display(df)\n", + "print(df.mean())\n", + "print(df.std())\n", + "display(df**2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Thereafter we can select specific columns only and plot final results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "df.columns = ['First', 'Second', 'Third', 'Fourth', 'Fifth']\n", + "df.index = np.arange(10)\n", + "\n", + "display(df)\n", + "print(df['Second'].mean() )\n", + "print(df.info())\n", + "print(df.describe())\n", + "\n", + "from pylab import plt, mpl\n", + "plt.style.use('seaborn')\n", + "mpl.rcParams['font.family'] = 'serif'\n", + "\n", + "df.cumsum().plot(lw=2.0, figsize=(10,6))\n", + "plt.show()\n", + "\n", + "\n", + "df.plot.bar(figsize=(10,6), rot=15)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can produce a $4\\times 4$ matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "b = np.arange(16).reshape((4,4))\n", + "print(b)\n", + "df1 = pd.DataFrame(b)\n", + "print(df1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and many other operations. \n", + "\n", + "The **Series** class is another important class included in\n", + "**pandas**. You can view it as a specialization of **DataFrame** but where\n", + "we have just a single column of data. It shares many of the same features as _DataFrame. As with **DataFrame**,\n", + "most operations are vectorized, achieving thereby a high performance when dealing with computations of arrays, in particular labeled arrays.\n", + "As we will see below it leads also to a very concice code close to the mathematical operations we may be interested in.\n", + "For multidimensional arrays, we recommend strongly [xarray](http://xarray.pydata.org/en/stable/). **xarray** has much of the same flexibility as **pandas**, but allows for the extension to higher dimensions than two. We will see examples later of the usage of both **pandas** and **xarray**. \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "In order to study various Machine Learning algorithms, we need to\n", + "access data. Acccessing data is an essential step in all machine\n", + "learning algorithms. In particular, setting up the so-called **design\n", + "matrix** (to be defined below) is often the first element we need in\n", + "order to perform our calculations. To set up the design matrix means\n", + "reading (and later, when the calculations are done, writing) data\n", + "in various formats, The formats span from reading files from disk,\n", + "loading data from databases and interacting with online sources\n", + "like web application programming interfaces (APIs).\n", + "\n", + "In handling various input formats, as discussed above, we will mainly stay with **pandas**,\n", + "a Python package which allows us, in a seamless and painless way, to\n", + "deal with a multitude of formats, from standard **csv** (comma separated\n", + "values) files, via **excel**, **html** to **hdf5** formats. With **pandas**\n", + "and the **DataFrame** and **Series** functionalities we are able to convert text data\n", + "into the calculational formats we need for a specific algorithm. And our code is going to be \n", + "pretty close the basic mathematical expressions.\n", + "\n", + "Our first data set is going to be a classic from nuclear physics, namely all\n", + "available data on binding energies. Don't be intimidated if you are not familiar with nuclear physics. It serves simply as an example here of a data set. \n", + "\n", + "We will show some of the\n", + "strengths of packages like **Scikit-Learn** in fitting nuclear binding energies to\n", + "specific functions using linear regression first. Then, as a teaser, we will show you how \n", + "you can easily implement other algorithms like decision trees and random forests and neural networks.\n", + "\n", + "But before we really start with nuclear physics data, let's just look at some simpler polynomial fitting cases, such as,\n", + "(don't be offended) fitting straight lines!\n", + "\n", + "\n", + "\n", + "\n", + "## Simple linear regression model using **scikit-learn**\n", + "\n", + "We start with perhaps our simplest possible example, using **Scikit-Learn** to perform linear regression analysis on a data set produced by us. \n", + "\n", + "What follows is a simple Python code where we have defined a function\n", + "$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries. \n", + "The numbers in the vector $\\hat{x}$ are given\n", + "by random numbers generated with a uniform distribution with entries\n", + "$x_i \\in [0,1]$ (more about probability distribution functions\n", + "later). These values are then used to define a function $y(x)$\n", + "(tabulated again as a vector) with a linear dependence on $x$ plus a\n", + "random noise added via the normal distribution.\n", + "\n", + "\n", + "The Numpy functions are imported used the **import numpy as np**\n", + "statement and the random number generator for the uniform distribution\n", + "is called using the function **np.random.rand()**, where we specificy\n", + "that we want $100$ random variables. Using Numpy we define\n", + "automatically an array with the specified number of elements, $100$ in\n", + "our case. With the Numpy function **randn()** we can compute random\n", + "numbers with the normal distribution (mean value $\\mu$ equal to zero and\n", + "variance $\\sigma^2$ set to one) and produce the values of $y$ assuming a linear\n", + "dependence as function of $x$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y = 2x+N(0,1),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $N(0,1)$ represents random numbers generated by the normal\n", + "distribution. From **Scikit-Learn** we import then the\n", + "**LinearRegression** functionality and make a prediction $\\tilde{y} =\n", + "\\alpha + \\beta x$ using the function **fit(x,y)**. We call the set of\n", + "data $(\\hat{x},\\hat{y})$ for our training data. The Python package\n", + "**scikit-learn** has also a functionality which extracts the above\n", + "fitting parameters $\\alpha$ and $\\beta$ (see below). Later we will\n", + "distinguish between training data and test data.\n", + "\n", + "For plotting we use the Python package\n", + "[matplotlib](https://matplotlib.org/) which produces publication\n", + "quality figures. Feel free to explore the extensive\n", + "[gallery](https://matplotlib.org/gallery/index.html) of examples. In\n", + "this example we plot our original values of $x$ and $y$ as well as the\n", + "prediction **ypredict** ($\\tilde{y}$), which attempts at fitting our\n", + "data with a straight line.\n", + "\n", + "The Python code follows here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Importing various packages\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "x = np.random.rand(100,1)\n", + "y = 2*x+np.random.randn(100,1)\n", + "linreg = LinearRegression()\n", + "linreg.fit(x,y)\n", + "xnew = np.array([[0],[1]])\n", + "ypredict = linreg.predict(xnew)\n", + "\n", + "plt.plot(xnew, ypredict, \"r-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0,1.0,0, 5.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Simple Linear Regression')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example serves several aims. It allows us to demonstrate several\n", + "aspects of data analysis and later machine learning algorithms. The\n", + "immediate visualization shows that our linear fit is not\n", + "impressive. It goes through the data points, but there are many\n", + "outliers which are not reproduced by our linear regression. We could\n", + "now play around with this small program and change for example the\n", + "factor in front of $x$ and the normal distribution. Try to change the\n", + "function $y$ to" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y = 10x+0.01 \\times N(0,1),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $x$ is defined as before. Does the fit look better? Indeed, by\n", + "reducing the role of the noise given by the normal distribution we see immediately that\n", + "our linear prediction seemingly reproduces better the training\n", + "set. However, this testing 'by the eye' is obviouly not satisfactory in the\n", + "long run. Here we have only defined the training data and our model, and \n", + "have not discussed a more rigorous approach to the **cost** function.\n", + "\n", + "We need more rigorous criteria in defining whether we have succeeded or\n", + "not in modeling our training data. You will be surprised to see that\n", + "many scientists seldomly venture beyond this 'by the eye' approach. A\n", + "standard approach for the *cost* function is the so-called $\\chi^2$\n", + "function (a variant of the mean-squared error (MSE))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\chi^2 = \\frac{1}{n}\n", + "\\sum_{i=0}^{n-1}\\frac{(y_i-\\tilde{y}_i)^2}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\sigma_i^2$ is the variance (to be defined later) of the entry\n", + "$y_i$. We may not know the explicit value of $\\sigma_i^2$, it serves\n", + "however the aim of scaling the equations and make the cost function\n", + "dimensionless. \n", + "\n", + "Minimizing the cost function is a central aspect of\n", + "our discussions to come. Finding its minima as function of the model\n", + "parameters ($\\alpha$ and $\\beta$ in our case) will be a recurring\n", + "theme in these series of lectures. Essentially all machine learning\n", + "algorithms we will discuss center around the minimization of the\n", + "chosen cost function. This depends in turn on our specific\n", + "model for describing the data, a typical situation in supervised\n", + "learning. Automatizing the search for the minima of the cost function is a\n", + "central ingredient in all algorithms. Typical methods which are\n", + "employed are various variants of **gradient** methods. These will be\n", + "discussed in more detail later. Again, you'll be surprised to hear that\n", + "many practitioners minimize the above function ''by the eye', popularly dubbed as \n", + "'chi by the eye'. That is, change a parameter and see (visually and numerically) that \n", + "the $\\chi^2$ function becomes smaller. \n", + "\n", + "There are many ways to define the cost function. A simpler approach is to look at the relative difference between the training data and the predicted data, that is we define \n", + "the relative error (why would we prefer the MSE instead of the relative error?) as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\epsilon_{\\mathrm{relative}}= \\frac{\\vert \\hat{y} -\\hat{\\tilde{y}}\\vert}{\\vert \\hat{y}\\vert}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The squared cost function results in an arithmetic mean-unbiased\n", + "estimator, and the absolute-value cost function results in a\n", + "median-unbiased estimator (in the one-dimensional case, and a\n", + "geometric median-unbiased estimator for the multi-dimensional\n", + "case). The squared cost function has the disadvantage that it has the tendency\n", + "to be dominated by outliers.\n", + "\n", + "We can modify easily the above Python code and plot the relative error instead" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "x = np.random.rand(100,1)\n", + "y = 5*x+0.01*np.random.randn(100,1)\n", + "linreg = LinearRegression()\n", + "linreg.fit(x,y)\n", + "ypredict = linreg.predict(x)\n", + "\n", + "plt.plot(x, np.abs(ypredict-y)/abs(y), \"ro\")\n", + "plt.axis([0,1.0,0.0, 0.5])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$\\epsilon_{\\mathrm{relative}}$')\n", + "plt.title(r'Relative error')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Depending on the parameter in front of the normal distribution, we may\n", + "have a small or larger relative error. Try to play around with\n", + "different training data sets and study (graphically) the value of the\n", + "relative error.\n", + "\n", + "As mentioned above, **Scikit-Learn** has an impressive functionality.\n", + "We can for example extract the values of $\\alpha$ and $\\beta$ and\n", + "their error estimates, or the variance and standard deviation and many\n", + "other properties from the statistical data analysis. \n", + "\n", + "Here we show an\n", + "example of the functionality of **Scikit-Learn**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np \n", + "import matplotlib.pyplot as plt \n", + "from sklearn.linear_model import LinearRegression \n", + "from sklearn.metrics import mean_squared_error, r2_score, mean_squared_log_error, mean_absolute_error\n", + "\n", + "x = np.random.rand(100,1)\n", + "y = 2.0+ 5*x+0.5*np.random.randn(100,1)\n", + "linreg = LinearRegression()\n", + "linreg.fit(x,y)\n", + "ypredict = linreg.predict(x)\n", + "print('The intercept alpha: \\n', linreg.intercept_)\n", + "print('Coefficient beta : \\n', linreg.coef_)\n", + "# The mean squared error \n", + "print(\"Mean squared error: %.2f\" % mean_squared_error(y, ypredict))\n", + "# Explained variance score: 1 is perfect prediction \n", + "print('Variance score: %.2f' % r2_score(y, ypredict))\n", + "# Mean squared log error \n", + "print('Mean squared log error: %.2f' % mean_squared_log_error(y, ypredict) )\n", + "# Mean absolute error \n", + "print('Mean absolute error: %.2f' % mean_absolute_error(y, ypredict))\n", + "plt.plot(x, ypredict, \"r-\")\n", + "plt.plot(x, y ,'ro')\n", + "plt.axis([0.0,1.0,1.5, 7.0])\n", + "plt.xlabel(r'$x$')\n", + "plt.ylabel(r'$y$')\n", + "plt.title(r'Linear Regression fit ')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The function **coef** gives us the parameter $\\beta$ of our fit while **intercept** yields \n", + "$\\alpha$. Depending on the constant in front of the normal distribution, we get values near or far from $alpha =2$ and $\\beta =5$. Try to play around with different parameters in front of the normal distribution. The function **meansquarederror** gives us the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error or loss defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "MSE(\\hat{y},\\hat{\\tilde{y}}) = \\frac{1}{n}\n", + "\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The smaller the value, the better the fit. Ideally we would like to\n", + "have an MSE equal zero. The attentive reader has probably recognized\n", + "this function as being similar to the $\\chi^2$ function defined above.\n", + "\n", + "The **r2score** function computes $R^2$, the coefficient of\n", + "determination. It provides a measure of how well future samples are\n", + "likely to be predicted by the model. Best possible score is 1.0 and it\n", + "can be negative (because the model can be arbitrarily worse). A\n", + "constant model that always predicts the expected value of $\\hat{y}$,\n", + "disregarding the input features, would get a $R^2$ score of $0.0$.\n", + "\n", + "If $\\tilde{\\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "R^2(\\hat{y}, \\tilde{\\hat{y}}) = 1 - \\frac{\\sum_{i=0}^{n - 1} (y_i - \\tilde{y}_i)^2}{\\sum_{i=0}^{n - 1} (y_i - \\bar{y})^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we have defined the mean value of $\\hat{y}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\bar{y} = \\frac{1}{n} \\sum_{i=0}^{n - 1} y_i.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another quantity taht we will meet again in our discussions of regression analysis is \n", + " the mean absolute error (MAE), a risk metric corresponding to the expected value of the absolute error loss or what we call the $l1$-norm loss. In our discussion above we presented the relative error.\n", + "The MAE is defined as follows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\text{MAE}(\\hat{y}, \\hat{\\tilde{y}}) = \\frac{1}{n} \\sum_{i=0}^{n-1} \\left| y_i - \\tilde{y}_i \\right|.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We present the \n", + "squared logarithmic (quadratic) error" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\text{MSLE}(\\hat{y}, \\hat{\\tilde{y}}) = \\frac{1}{n} \\sum_{i=0}^{n - 1} (\\log_e (1 + y_i) - \\log_e (1 + \\tilde{y}_i) )^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\log_e (x)$ stands for the natural logarithm of $x$. This error\n", + "estimate is best to use when targets having exponential growth, such\n", + "as population counts, average sales of a commodity over a span of\n", + "years etc. \n", + "\n", + "\n", + "Finally, another cost function is the Huber cost function used in robust regression.\n", + "\n", + "The rationale behind this possible cost function is its reduced\n", + "sensitivity to outliers in the data set. In our discussions on\n", + "dimensionality reduction and normalization of data we will meet other\n", + "ways of dealing with outliers.\n", + "\n", + "The Huber cost function is defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "H_{\\delta}(a)={\\begin{cases}{\\frac {1}{2}}{a^{2}}&{\\text{for }}|a|\\leq \\delta ,\\\\\\delta (|a|-{\\frac {1}{2}}\\delta ),&{\\text{otherwise.}}\\end{cases}}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here $a=\\boldsymbol{y} - \\boldsymbol{\\tilde{y}}$.\n", + "We will discuss in more\n", + "detail these and other functions in the various lectures. We conclude this part with another example. Instead of \n", + "a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import random\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.linear_model import LinearRegression\n", + "\n", + "x=np.linspace(0.02,0.98,200)\n", + "noise = np.asarray(random.sample((range(200)),200))\n", + "y=x**3*noise\n", + "yn=x**3*100\n", + "poly3 = PolynomialFeatures(degree=3)\n", + "X = poly3.fit_transform(x[:,np.newaxis])\n", + "clf3 = LinearRegression()\n", + "clf3.fit(X,y)\n", + "\n", + "Xplot=poly3.fit_transform(x[:,np.newaxis])\n", + "poly3_plot=plt.plot(x, clf3.predict(Xplot), label='Cubic Fit')\n", + "plt.plot(x,yn, color='red', label=\"True Cubic\")\n", + "plt.scatter(x, y, label='Data', color='orange', s=15)\n", + "plt.legend()\n", + "plt.show()\n", + "\n", + "def error(a):\n", + " for i in y:\n", + " err=(y-yn)/yn\n", + " return abs(np.sum(err))/len(err)\n", + "\n", + "print (error(y))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let us now dive into nuclear physics and remind ourselves briefly about some basic features about binding\n", + "energies. A basic quantity which can be measured for the ground\n", + "states of nuclei is the atomic mass $M(N, Z)$ of the neutral atom with\n", + "atomic mass number $A$ and charge $Z$. The number of neutrons is $N$. There are indeed several sophisticated experiments worldwide which allow us to measure this quantity to high precision (parts per million even). \n", + "\n", + "Atomic masses are usually tabulated in terms of the mass excess defined by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\Delta M(N, Z) = M(N, Z) - uA,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $u$ is the Atomic Mass Unit" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "u = M(^{12}\\mathrm{C})/12 = 931.4940954(57) \\hspace{0.1cm} \\mathrm{MeV}/c^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The nucleon masses are" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "m_p = 1.00727646693(9)u,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "m_n = 939.56536(8)\\hspace{0.1cm} \\mathrm{MeV}/c^2 = 1.0086649156(6)u.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the [2016 mass evaluation of by W.J.Huang, G.Audi, M.Wang, F.G.Kondev, S.Naimi and X.Xu](http://nuclearmasses.org/resources_folder/Wang_2017_Chinese_Phys_C_41_030003.pdf)\n", + "there are data on masses and decays of 3437 nuclei.\n", + "\n", + "The nuclear binding energy is defined as the energy required to break\n", + "up a given nucleus into its constituent parts of $N$ neutrons and $Z$\n", + "protons. In terms of the atomic masses $M(N, Z)$ the binding energy is\n", + "defined by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "BE(N, Z) = ZM_H c^2 + Nm_n c^2 - M(N, Z)c^2 ,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $M_H$ is the mass of the hydrogen atom and $m_n$ is the mass of the neutron.\n", + "In terms of the mass excess the binding energy is given by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "BE(N, Z) = Z\\Delta_H c^2 + N\\Delta_n c^2 -\\Delta(N, Z)c^2 ,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\Delta_H c^2 = 7.2890$ MeV and $\\Delta_n c^2 = 8.0713$ MeV.\n", + "\n", + "\n", + "A popular and physically intuitive model which can be used to parametrize \n", + "the experimental binding energies as function of $A$, is the so-called \n", + "**liquid drop model**. The ansatz is based on the following expression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "BE(N,Z) = a_1A-a_2A^{2/3}-a_3\\frac{Z^2}{A^{1/3}}-a_4\\frac{(N-Z)^2}{A},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $A$ stands for the number of nucleons and the $a_i$s are parameters which are determined by a fit \n", + "to the experimental data. \n", + "\n", + "\n", + "\n", + "\n", + "To arrive at the above expression we have assumed that we can make the following assumptions:\n", + "\n", + " * There is a volume term $a_1A$ proportional with the number of nucleons (the energy is also an extensive quantity). When an assembly of nucleons of the same size is packed together into the smallest volume, each interior nucleon has a certain number of other nucleons in contact with it. This contribution is proportional to the volume.\n", + "\n", + " * There is a surface energy term $a_2A^{2/3}$. The assumption here is that a nucleon at the surface of a nucleus interacts with fewer other nucleons than one in the interior of the nucleus and hence its binding energy is less. This surface energy term takes that into account and is therefore negative and is proportional to the surface area.\n", + "\n", + " * There is a Coulomb energy term $a_3\\frac{Z^2}{A^{1/3}}$. The electric repulsion between each pair of protons in a nucleus yields less binding. \n", + "\n", + " * There is an asymmetry term $a_4\\frac{(N-Z)^2}{A}$. This term is associated with the Pauli exclusion principle and reflects the fact that the proton-neutron interaction is more attractive on the average than the neutron-neutron and proton-proton interactions.\n", + "\n", + "We could also add a so-called pairing term, which is a correction term that\n", + "arises from the tendency of proton pairs and neutron pairs to\n", + "occur. An even number of particles is more stable than an odd number. \n", + "\n", + "\n", + "### Organizing our data\n", + "\n", + "Let us start with reading and organizing our data. \n", + "We start with the compilation of masses and binding energies from 2016.\n", + "After having downloaded this file to our own computer, we are now ready to read the file and start structuring our data.\n", + "\n", + "\n", + "We start with preparing folders for storing our calculations and the data file over masses and binding energies. We import also various modules that we will find useful in order to present various Machine Learning methods. Here we focus mainly on the functionality of **scikit-learn**." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Common imports\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import sklearn.linear_model as skl\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n", + "import os\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"MassEval2016.dat\"),'r')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Before we proceed, we define also a function for making our plots. You can obviously avoid this and simply set up various **matplotlib** commands every time you need them. You may however find it convenient to collect all such commands in one function and simply call this function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from pylab import plt, mpl\n", + "plt.style.use('seaborn')\n", + "mpl.rcParams['font.family'] = 'serif'\n", + "\n", + "def MakePlot(x,y, styles, labels, axlabels):\n", + " plt.figure(figsize=(10,6))\n", + " for i in range(len(x)):\n", + " plt.plot(x[i], y[i], styles[i], label = labels[i])\n", + " plt.xlabel(axlabels[0])\n", + " plt.ylabel(axlabels[1])\n", + " plt.legend(loc=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our next step is to read the data on experimental binding energies and\n", + "reorganize them as functions of the mass number $A$, the number of\n", + "protons $Z$ and neutrons $N$ using **pandas**. Before we do this it is\n", + "always useful (unless you have a binary file or other types of compressed\n", + "data) to actually open the file and simply take a look at it!\n", + "\n", + "\n", + "In particular, the program that outputs the final nuclear masses is written in Fortran with a specific format. It means that we need to figure out the format and which columns contain the data we are interested in. Pandas comes with a function that reads formatted output. After having admired the file, we are now ready to start massaging it with **pandas**. The file begins with some basic format information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\" \n", + "This is taken from the data file of the mass 2016 evaluation. \n", + "All files are 3436 lines long with 124 character per line. \n", + " Headers are 39 lines long. \n", + " col 1 : Fortran character control: 1 = page feed 0 = line feed \n", + " format : a1,i3,i5,i5,i5,1x,a3,a4,1x,f13.5,f11.5,f11.3,f9.3,1x,a2,f11.3,f9.3,1x,i3,1x,f12.5,f11.5 \n", + " These formats are reflected in the pandas widths variable below, see the statement \n", + " widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1), \n", + " Pandas has also a variable header, with length 39 in this case. \n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The data we are interested in are in columns 2, 3, 4 and 11, giving us\n", + "the number of neutrons, protons, mass numbers and binding energies,\n", + "respectively. We add also for the sake of completeness the element name. The data are in fixed-width formatted lines and we will\n", + "covert them into the **pandas** DataFrame structure." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Read the experimental data with Pandas\n", + "Masses = pd.read_fwf(infile, usecols=(2,3,4,6,11),\n", + " names=('N', 'Z', 'A', 'Element', 'Ebinding'),\n", + " widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1),\n", + " header=39,\n", + " index_col=False)\n", + "\n", + "# Extrapolated values are indicated by '#' in place of the decimal place, so\n", + "# the Ebinding column won't be numeric. Coerce to float and drop these entries.\n", + "Masses['Ebinding'] = pd.to_numeric(Masses['Ebinding'], errors='coerce')\n", + "Masses = Masses.dropna()\n", + "# Convert from keV to MeV.\n", + "Masses['Ebinding'] /= 1000\n", + "\n", + "# Group the DataFrame by nucleon number, A.\n", + "Masses = Masses.groupby('A')\n", + "# Find the rows of the grouped DataFrame with the maximum binding energy.\n", + "Masses = Masses.apply(lambda t: t[t.Ebinding==t.Ebinding.max()])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have now read in the data, grouped them according to the variables we are interested in. \n", + "We see how easy it is to reorganize the data using **pandas**. If we\n", + "were to do these operations in C/C++ or Fortran, we would have had to\n", + "write various functions/subroutines which perform the above\n", + "reorganizations for us. Having reorganized the data, we can now start\n", + "to make some simple fits using both the functionalities in **numpy** and\n", + "**Scikit-Learn** afterwards. \n", + "\n", + "Now we define five variables which contain\n", + "the number of nucleons $A$, the number of protons $Z$ and the number of neutrons $N$, the element name and finally the energies themselves." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "A = Masses['A']\n", + "Z = Masses['Z']\n", + "N = Masses['N']\n", + "Element = Masses['Element']\n", + "Energies = Masses['Ebinding']\n", + "print(Masses)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The next step, and we will define this mathematically later, is to set up the so-called **design matrix**. We will throughout call this matrix $\\boldsymbol{X}$.\n", + "It has dimensionality $p\\times n$, where $n$ is the number of data points and $p$ are the so-called predictors. In our case here they are given by the number of polynomials in $A$ we wish to include in the fit." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Now we set up the design matrix X\n", + "X = np.zeros((len(A),5))\n", + "X[:,0] = 1\n", + "X[:,1] = A\n", + "X[:,2] = A**(2.0/3.0)\n", + "X[:,3] = A**(-1.0/3.0)\n", + "X[:,4] = A**(-1.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With **scikitlearn** we are now ready to use linear regression and fit our data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "clf = skl.LinearRegression().fit(X, Energies)\n", + "fity = clf.predict(X)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Pretty simple! \n", + "Now we can print measures of how our fit is doing, the coefficients from the fits and plot the final fit together with our data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# The mean squared error \n", + "print(\"Mean squared error: %.2f\" % mean_squared_error(Energies, fity))\n", + "# Explained variance score: 1 is perfect prediction \n", + "print('Variance score: %.2f' % r2_score(Energies, fity))\n", + "# Mean absolute error \n", + "print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity))\n", + "print(clf.coef_, clf.intercept_)\n", + "\n", + "Masses['Eapprox'] = fity\n", + "# Generate a plot comparing the experimental with the fitted values values.\n", + "fig, ax = plt.subplots()\n", + "ax.set_xlabel(r'$A = N + Z$')\n", + "ax.set_ylabel(r'$E_\\mathrm{bind}\\,/\\mathrm{MeV}$')\n", + "ax.plot(Masses['A'], Masses['Ebinding'], alpha=0.7, lw=2,\n", + " label='Ame2016')\n", + "ax.plot(Masses['A'], Masses['Eapprox'], alpha=0.7, lw=2, c='m',\n", + " label='Fit')\n", + "ax.legend()\n", + "save_fig(\"Masses2016\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As a teaser, let us now see how we can do this with decision trees using **scikit-learn**. Later we will switch to so-called **random forests**!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\n", + "#Decision Tree Regression\n", + "from sklearn.tree import DecisionTreeRegressor\n", + "regr_1=DecisionTreeRegressor(max_depth=5)\n", + "regr_2=DecisionTreeRegressor(max_depth=7)\n", + "regr_3=DecisionTreeRegressor(max_depth=9)\n", + "regr_1.fit(X, Energies)\n", + "regr_2.fit(X, Energies)\n", + "regr_3.fit(X, Energies)\n", + "\n", + "\n", + "y_1 = regr_1.predict(X)\n", + "y_2 = regr_2.predict(X)\n", + "y_3=regr_3.predict(X)\n", + "Masses['Eapprox'] = y_3\n", + "# Plot the results\n", + "plt.figure()\n", + "plt.plot(A, Energies, color=\"blue\", label=\"Data\", linewidth=2)\n", + "plt.plot(A, y_1, color=\"red\", label=\"max_depth=5\", linewidth=2)\n", + "plt.plot(A, y_2, color=\"green\", label=\"max_depth=7\", linewidth=2)\n", + "plt.plot(A, y_3, color=\"m\", label=\"max_depth=9\", linewidth=2)\n", + "\n", + "plt.xlabel(\"$A$\")\n", + "plt.ylabel(\"$E$[MeV]\")\n", + "plt.title(\"Decision Tree Regression\")\n", + "plt.legend()\n", + "save_fig(\"Masses2016Trees\")\n", + "plt.show()\n", + "print(Masses)\n", + "print(np.mean( (Energies-y_1)**2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The **seaborn** package allows us to visualize data in an efficient way. Note that we use **scikit-learn**'s multi-layer perceptron (or feed forward neural network) \n", + "functionality." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.neural_network import MLPRegressor\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sns\n", + "\n", + "X_train = X\n", + "Y_train = Energies\n", + "n_hidden_neurons = 100\n", + "epochs = 100\n", + "# store models for later use\n", + "eta_vals = np.logspace(-5, 1, 7)\n", + "lmbd_vals = np.logspace(-5, 1, 7)\n", + "# store the models for later use\n", + "DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", + "train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))\n", + "sns.set()\n", + "for i, eta in enumerate(eta_vals):\n", + " for j, lmbd in enumerate(lmbd_vals):\n", + " dnn = MLPRegressor(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',\n", + " alpha=lmbd, learning_rate_init=eta, max_iter=epochs)\n", + " dnn.fit(X_train, Y_train)\n", + " DNN_scikit[i][j] = dnn\n", + " train_accuracy[i][j] = dnn.score(X_train, Y_train)\n", + "\n", + "fig, ax = plt.subplots(figsize = (10, 10))\n", + "sns.heatmap(train_accuracy, annot=True, ax=ax, cmap=\"viridis\")\n", + "ax.set_title(\"Training Accuracy\")\n", + "ax.set_ylabel(\"$\\eta$\")\n", + "ax.set_xlabel(\"$\\lambda$\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Linear Regression, basic elements\n", + "\n", + "\n", + "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureAug27.mp4?vrtx=view-as-webpage).\n", + "\n", + "\n", + "Fitting a continuous function with linear parameterization in terms of the parameters $\\boldsymbol{\\beta}$.\n", + "* Method of choice for fitting a continuous function!\n", + "\n", + "* Gives an excellent introduction to central Machine Learning features with **understandable pedagogical** links to other methods like **Neural Networks**, **Support Vector Machines** etc\n", + "\n", + "* Analytical expression for the fitting parameters $\\boldsymbol{\\beta}$\n", + "\n", + "* Analytical expressions for statistical propertiers like mean values, variances, confidence intervals and more\n", + "\n", + "* Analytical relation with probabilistic interpretations \n", + "\n", + "* Easy to introduce basic concepts like bias-variance tradeoff, cross-validation, resampling and regularization techniques and many other ML topics\n", + "\n", + "* Easy to code! And links well with classification problems and logistic regression and neural networks\n", + "\n", + "* Allows for **easy** hands-on understanding of gradient descent methods\n", + "\n", + "* and many more features\n", + "\n", + "For more discussions of Ridge and Lasso regression, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "Similarly, [Mehta et al's article](https://arxiv.org/abs/1803.08823) is also recommended.\n", + "\n", + "\n", + "\n", + "Regression modeling deals with the description of the sampling distribution of a given random variable $y$ and how it varies as function of another variable or a set of such variables $\\boldsymbol{x} =[x_0, x_1,\\dots, x_{n-1}]^T$. \n", + "The first variable is called the **dependent**, the **outcome** or the **response** variable while the set of variables $\\boldsymbol{x}$ is called the independent variable, or the predictor variable or the explanatory variable. \n", + "\n", + "A regression model aims at finding a likelihood function $p(\\boldsymbol{y}\\vert \\boldsymbol{x})$, that is the conditional distribution for $\\boldsymbol{y}$ with a given $\\boldsymbol{x}$. The estimation of $p(\\boldsymbol{y}\\vert \\boldsymbol{x})$ is made using a data set with \n", + "* $n$ cases $i = 0, 1, 2, \\dots, n-1$ \n", + "\n", + "* Response (target, dependent or outcome) variable $y_i$ with $i = 0, 1, 2, \\dots, n-1$ \n", + "\n", + "* $p$ so-called explanatory (independent or predictor) variables $\\boldsymbol{x}_i=[x_{i0}, x_{i1}, \\dots, x_{ip-1}]$ with $i = 0, 1, 2, \\dots, n-1$ and explanatory variables running from $0$ to $p-1$. See below for more explicit examples. \n", + "\n", + " The goal of the regression analysis is to extract/exploit relationship between $\\boldsymbol{y}$ and $\\boldsymbol{x}$ in or to infer causal dependencies, approximations to the likelihood functions, functional relationships and to make predictions, making fits and many other things.\n", + "\n", + "\n", + "Consider an experiment in which $p$ characteristics of $n$ samples are\n", + "measured. The data from this experiment, for various explanatory variables $p$ are normally represented by a matrix \n", + "$\\mathbf{X}$.\n", + "\n", + "The matrix $\\mathbf{X}$ is called the *design\n", + "matrix*. Additional information of the samples is available in the\n", + "form of $\\boldsymbol{y}$ (also as above). The variable $\\boldsymbol{y}$ is\n", + "generally referred to as the *response variable*. The aim of\n", + "regression analysis is to explain $\\boldsymbol{y}$ in terms of\n", + "$\\boldsymbol{X}$ through a functional relationship like $y_i =\n", + "f(\\mathbf{X}_{i,\\ast})$. When no prior knowledge on the form of\n", + "$f(\\cdot)$ is available, it is common to assume a linear relationship\n", + "between $\\boldsymbol{X}$ and $\\boldsymbol{y}$. This assumption gives rise to\n", + "the *linear regression model* where $\\boldsymbol{\\beta} = [\\beta_0, \\ldots,\n", + "\\beta_{p-1}]^{T}$ are the *regression parameters*. \n", + "\n", + "Linear regression gives us a set of analytical equations for the parameters $\\beta_j$.\n", + "\n", + "\n", + "In order to understand the relation among the predictors $p$, the set of data $n$ and the target (outcome, output etc) $\\boldsymbol{y}$,\n", + "consider the model we discussed for describing nuclear binding energies. \n", + "\n", + "There we assumed that we could parametrize the data using a polynomial approximation based on the liquid drop model.\n", + "Assuming" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "BE(A) = a_0+a_1A+a_2A^{2/3}+a_3A^{-1/3}+a_4A^{-1},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have five predictors, that is the intercept, the $A$ dependent term, the $A^{2/3}$ term and the $A^{-1/3}$ and $A^{-1}$ terms.\n", + "This gives $p=0,1,2,3,4$. Furthermore we have $n$ entries for each predictor. It means that our design matrix is a \n", + "$p\\times n$ matrix $\\boldsymbol{X}$.\n", + "\n", + "Here the predictors are based on a model we have made. A popular data set which is widely encountered in ML applications is the\n", + "so-called [credit card default data from Taiwan](https://www.sciencedirect.com/science/article/pii/S0957417407006719?via%3Dihub). The data set contains data on $n=30000$ credit card holders with predictors like gender, marital status, age, profession, education, etc. In total there are $24$ such predictors or attributes leading to a design matrix of dimensionality $24 \\times 30000$. This is however a classification problem and we will come back to it when we discuss Logistic Regression. \n", + "\n", + "\n", + "Before we proceed let us study a case from linear algebra where we aim at fitting a set of data $\\boldsymbol{y}=[y_0,y_1,\\dots,y_{n-1}]$. We could think of these data as a result of an experiment or a complicated numerical experiment. These data are functions of a series of variables $\\boldsymbol{x}=[x_0,x_1,\\dots,x_{n-1}]$, that is $y_i = y(x_i)$ with $i=0,1,2,\\dots,n-1$. The variables $x_i$ could represent physical quantities like time, temperature, position etc. We assume that $y(x)$ is a smooth function. \n", + "\n", + "Since obtaining these data points may not be trivial, we want to use these data to fit a function which can allow us to make predictions for values of $y$ which are not in the present set. The perhaps simplest approach is to assume we can parametrize our function in terms of a polynomial of degree $n-1$ with $n$ points, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y=y(x) \\rightarrow y(x_i)=\\tilde{y}_i+\\epsilon_i=\\sum_{j=0}^{n-1} \\beta_j x_i^j+\\epsilon_i,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\epsilon_i$ is the error in our approximation. \n", + "\n", + "\n", + "For every set of values $y_i,x_i$ we have thus the corresponding set of equations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "y_0&=\\beta_0+\\beta_1x_0^1+\\beta_2x_0^2+\\dots+\\beta_{n-1}x_0^{n-1}+\\epsilon_0\\\\\n", + "y_1&=\\beta_0+\\beta_1x_1^1+\\beta_2x_1^2+\\dots+\\beta_{n-1}x_1^{n-1}+\\epsilon_1\\\\\n", + "y_2&=\\beta_0+\\beta_1x_2^1+\\beta_2x_2^2+\\dots+\\beta_{n-1}x_2^{n-1}+\\epsilon_2\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{n-1}&=\\beta_0+\\beta_1x_{n-1}^1+\\beta_2x_{n-1}^2+\\dots+\\beta_{n-1}x_{n-1}^{n-1}+\\epsilon_{n-1}.\\\\\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Defining the vectors" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = [y_0,y_1, y_2,\\dots, y_{n-1}]^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta} = [\\beta_0,\\beta_1, \\beta_2,\\dots, \\beta_{n-1}]^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\epsilon} = [\\epsilon_0,\\epsilon_1, \\epsilon_2,\\dots, \\epsilon_{n-1}]^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and the design matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\n", + "\\begin{bmatrix} \n", + "1& x_{0}^1 &x_{0}^2& \\dots & \\dots &x_{0}^{n-1}\\\\\n", + "1& x_{1}^1 &x_{1}^2& \\dots & \\dots &x_{1}^{n-1}\\\\\n", + "1& x_{2}^1 &x_{2}^2& \\dots & \\dots &x_{2}^{n-1}\\\\ \n", + "\\dots& \\dots &\\dots& \\dots & \\dots &\\dots\\\\\n", + "1& x_{n-1}^1 &x_{n-1}^2& \\dots & \\dots &x_{n-1}^{n-1}\\\\\n", + "\\end{bmatrix}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we can rewrite our equations as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = \\boldsymbol{X}\\boldsymbol{\\beta}+\\boldsymbol{\\epsilon}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The above design matrix is called a [Vandermonde matrix](https://en.wikipedia.org/wiki/Vandermonde_matrix).\n", + "\n", + "We are obviously not limited to the above polynomial expansions. We\n", + "could replace the various powers of $x$ with elements of Fourier\n", + "series or instead of $x_i^j$ we could have $\\cos{(j x_i)}$ or $\\sin{(j\n", + "x_i)}$, or time series or other orthogonal functions. For every set\n", + "of values $y_i,x_i$ we can then generalize the equations to" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "y_0&=\\beta_0x_{00}+\\beta_1x_{01}+\\beta_2x_{02}+\\dots+\\beta_{n-1}x_{0n-1}+\\epsilon_0\\\\\n", + "y_1&=\\beta_0x_{10}+\\beta_1x_{11}+\\beta_2x_{12}+\\dots+\\beta_{n-1}x_{1n-1}+\\epsilon_1\\\\\n", + "y_2&=\\beta_0x_{20}+\\beta_1x_{21}+\\beta_2x_{22}+\\dots+\\beta_{n-1}x_{2n-1}+\\epsilon_2\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{i}&=\\beta_0x_{i0}+\\beta_1x_{i1}+\\beta_2x_{i2}+\\dots+\\beta_{n-1}x_{in-1}+\\epsilon_i\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{n-1}&=\\beta_0x_{n-1,0}+\\beta_1x_{n-1,2}+\\beta_2x_{n-1,2}+\\dots+\\beta_{n-1}x_{n-1,n-1}+\\epsilon_{n-1}.\\\\\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note that we have $p=n$ here. The matrix is symmetric. This is generally not the case!**\n", + "\n", + "We redefine in turn the matrix $\\boldsymbol{X}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\n", + "\\begin{bmatrix} \n", + "x_{00}& x_{01} &x_{02}& \\dots & \\dots &x_{0,n-1}\\\\\n", + "x_{10}& x_{11} &x_{12}& \\dots & \\dots &x_{1,n-1}\\\\\n", + "x_{20}& x_{21} &x_{22}& \\dots & \\dots &x_{2,n-1}\\\\ \n", + "\\dots& \\dots &\\dots& \\dots & \\dots &\\dots\\\\\n", + "x_{n-1,0}& x_{n-1,1} &x_{n-1,2}& \\dots & \\dots &x_{n-1,n-1}\\\\\n", + "\\end{bmatrix}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and without loss of generality we rewrite again our equations as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = \\boldsymbol{X}\\boldsymbol{\\beta}+\\boldsymbol{\\epsilon}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The left-hand side of this equation is kwown. Our error vector $\\boldsymbol{\\epsilon}$ and the parameter vector $\\boldsymbol{\\beta}$ are our unknow quantities. How can we obtain the optimal set of $\\beta_i$ values? \n", + "\n", + "We have defined the matrix $\\boldsymbol{X}$ via the equations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "y_0&=\\beta_0x_{00}+\\beta_1x_{01}+\\beta_2x_{02}+\\dots+\\beta_{n-1}x_{0n-1}+\\epsilon_0\\\\\n", + "y_1&=\\beta_0x_{10}+\\beta_1x_{11}+\\beta_2x_{12}+\\dots+\\beta_{n-1}x_{1n-1}+\\epsilon_1\\\\\n", + "y_2&=\\beta_0x_{20}+\\beta_1x_{21}+\\beta_2x_{22}+\\dots+\\beta_{n-1}x_{2n-1}+\\epsilon_1\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{i}&=\\beta_0x_{i0}+\\beta_1x_{i1}+\\beta_2x_{i2}+\\dots+\\beta_{n-1}x_{in-1}+\\epsilon_1\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{n-1}&=\\beta_0x_{n-1,0}+\\beta_1x_{n-1,2}+\\beta_2x_{n-1,2}+\\dots+\\beta_{n-1}x_{n-1,n-1}+\\epsilon_{n-1}.\\\\\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we noted above, we stayed with a system with the design matrix \n", + " $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times n}$, that is we have $p=n$. For reasons to come later (algorithmic arguments) we will hereafter define \n", + "our matrix as $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$, with the predictors refering to the column numbers and the entries $n$ being the row elements.\n", + "\n", + "In our [introductory notes](https://compphysics.github.io/MachineLearning/doc/pub/How2ReadData/html/How2ReadData.html) we looked at the so-called [liquid drop model](https://en.wikipedia.org/wiki/Semi-empirical_mass_formula). Let us remind ourselves about what we did by looking at the code.\n", + "\n", + "We restate the parts of the code we are most interested in." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Common imports\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from IPython.display import display\n", + "import os\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"MassEval2016.dat\"),'r')\n", + "\n", + "\n", + "# Read the experimental data with Pandas\n", + "Masses = pd.read_fwf(infile, usecols=(2,3,4,6,11),\n", + " names=('N', 'Z', 'A', 'Element', 'Ebinding'),\n", + " widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1),\n", + " header=39,\n", + " index_col=False)\n", + "\n", + "# Extrapolated values are indicated by '#' in place of the decimal place, so\n", + "# the Ebinding column won't be numeric. Coerce to float and drop these entries.\n", + "Masses['Ebinding'] = pd.to_numeric(Masses['Ebinding'], errors='coerce')\n", + "Masses = Masses.dropna()\n", + "# Convert from keV to MeV.\n", + "Masses['Ebinding'] /= 1000\n", + "\n", + "# Group the DataFrame by nucleon number, A.\n", + "Masses = Masses.groupby('A')\n", + "# Find the rows of the grouped DataFrame with the maximum binding energy.\n", + "Masses = Masses.apply(lambda t: t[t.Ebinding==t.Ebinding.max()])\n", + "A = Masses['A']\n", + "Z = Masses['Z']\n", + "N = Masses['N']\n", + "Element = Masses['Element']\n", + "Energies = Masses['Ebinding']\n", + "\n", + "# Now we set up the design matrix X\n", + "X = np.zeros((len(A),5))\n", + "X[:,0] = 1\n", + "X[:,1] = A\n", + "X[:,2] = A**(2.0/3.0)\n", + "X[:,3] = A**(-1.0/3.0)\n", + "X[:,4] = A**(-1.0)\n", + "# Then nice printout using pandas\n", + "DesignMatrix = pd.DataFrame(X)\n", + "DesignMatrix.index = A\n", + "DesignMatrix.columns = ['1', 'A', 'A^(2/3)', 'A^(-1/3)', '1/A']\n", + "display(DesignMatrix)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With $\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p\\times 1}$, it means that we will hereafter write our equations for the approximation as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}}= \\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "throughout these lectures. \n", + "\n", + "With the above we use the design matrix to define the approximation $\\boldsymbol{\\tilde{y}}$ via the unknown quantity $\\boldsymbol{\\beta}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}}= \\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and in order to find the optimal parameters $\\beta_i$ instead of solving the above linear algebra problem, we define a function which gives a measure of the spread between the values $y_i$ (which represent hopefully the exact values) and the parameterized values $\\tilde{y}_i$, namely" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or using the matrix $\\boldsymbol{X}$ and in a more compact matrix-vector notation as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This function is one possible way to define the so-called cost function.\n", + "\n", + "\n", + "\n", + "It is also common to define\n", + "the function $C$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{2n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "since when taking the first derivative with respect to the unknown parameters $\\beta$, the factor of $2$ cancels out. \n", + "\n", + "The function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "can be linked to the variance of the quantity $y_i$ if we interpret the latter as the mean value. \n", + "When linking (see the discussion below) with the maximum likelihood approach below, we will indeed interpret $y_i$ as a mean value" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y_{i}=\\langle y_i \\rangle = \\beta_0x_{i,0}+\\beta_1x_{i,1}+\\beta_2x_{i,2}+\\dots+\\beta_{n-1}x_{i,n-1}+\\epsilon_i,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\langle y_i \\rangle$ is the mean value. Keep in mind also that\n", + "till now we have treated $y_i$ as the exact value. Normally, the\n", + "response (dependent or outcome) variable $y_i$ the outcome of a\n", + "numerical experiment or another type of experiment and is thus only an\n", + "approximation to the true value. It is then always accompanied by an\n", + "error estimate, often limited to a statistical error estimate given by\n", + "the standard deviation discussed earlier. In the discussion here we\n", + "will treat $y_i$ as our exact value for the response variable.\n", + "\n", + "In order to find the parameters $\\beta_i$ we will then minimize the spread of $C(\\boldsymbol{\\beta})$, that is we are going to solve the problem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In practical terms it means we will require" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_j} = \\frac{\\partial }{\\partial \\beta_j}\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}\\right)^2\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which results in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_j} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}x_{ij}\\left(y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}\\right)\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or in a matrix-vector form as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can rewrite" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{y} = \\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and if the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$ is invertible we have the solution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta} =\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We note also that since our design matrix is defined as $\\boldsymbol{X}\\in\n", + "{\\mathbb{R}}^{n\\times p}$, the product $\\boldsymbol{X}^T\\boldsymbol{X} \\in\n", + "{\\mathbb{R}}^{p\\times p}$. In the above case we have that $p \\ll n$,\n", + "in our case $p=5$ meaning that we end up with inverting a small\n", + "$5\\times 5$ matrix. This is a rather common situation, in many cases we end up with low-dimensional\n", + "matrices to invert. The methods discussed here and for many other\n", + "supervised learning algorithms like classification with logistic\n", + "regression or support vector machines, exhibit dimensionalities which\n", + "allow for the usage of direct linear algebra methods such as **LU** decomposition or **Singular Value Decomposition** (SVD) for finding the inverse of the matrix\n", + "$\\boldsymbol{X}^T\\boldsymbol{X}$. \n", + "\n", + "**Small question**: Do you think the example we have at hand here (the nuclear binding energies) can lead to problems in inverting the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$? What kind of problems can we expect? \n", + "\n", + "\n", + "The following matrix and vector relation will be useful here and for the rest of the course. Vectors are always written as boldfaced lower case letters and \n", + "matrices as upper case boldfaced letters." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "4\n", + "8\n", + " \n", + "<\n", + "<\n", + "<\n", + "!\n", + "!\n", + "M\n", + "A\n", + "T\n", + "H\n", + "_\n", + "B\n", + "L\n", + "O\n", + "C\n", + "K" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "4\n", + "9\n", + " \n", + "<\n", + "<\n", + "<\n", + "!\n", + "!\n", + "M\n", + "A\n", + "T\n", + "H\n", + "_\n", + "B\n", + "L\n", + "O\n", + "C\n", + "K" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "5\n", + "0\n", + " \n", + "<\n", + "<\n", + "<\n", + "!\n", + "!\n", + "M\n", + "A\n", + "T\n", + "H\n", + "_\n", + "B\n", + "L\n", + "O\n", + "C\n", + "K" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\log{\\vert\\boldsymbol{A}\\vert}}{\\partial \\boldsymbol{A}} = (\\boldsymbol{A}^{-1})^T.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The residuals $\\boldsymbol{\\epsilon}$ are in turn given by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\epsilon} = \\boldsymbol{y}-\\boldsymbol{\\tilde{y}} = \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and with" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)= 0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{\\epsilon}=\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)= 0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "meaning that the solution for $\\boldsymbol{\\beta}$ is the one which minimizes the residuals. Later we will link this with the maximum likelihood approach.\n", + "\n", + "\n", + "Let us now return to our nuclear binding energies and simply code the above equations. \n", + "\n", + "\n", + "It is rather straightforward to implement the matrix inversion and obtain the parameters $\\boldsymbol{\\beta}$. After having defined the matrix $\\boldsymbol{X}$ we simply need to \n", + "write" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# matrix inversion to find beta\n", + "beta = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Energies)\n", + "# and then make the prediction\n", + "ytilde = X @ beta" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, you can use the least squares functionality in **Numpy** as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "fit = np.linalg.lstsq(X, Energies, rcond =None)[0]\n", + "ytildenp = np.dot(fit,X.T)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And finally we plot our fit with and compare with data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "Masses['Eapprox'] = ytilde\n", + "# Generate a plot comparing the experimental with the fitted values values.\n", + "fig, ax = plt.subplots()\n", + "ax.set_xlabel(r'$A = N + Z$')\n", + "ax.set_ylabel(r'$E_\\mathrm{bind}\\,/\\mathrm{MeV}$')\n", + "ax.plot(Masses['A'], Masses['Ebinding'], alpha=0.7, lw=2,\n", + " label='Ame2016')\n", + "ax.plot(Masses['A'], Masses['Eapprox'], alpha=0.7, lw=2, c='m',\n", + " label='Fit')\n", + "ax.legend()\n", + "save_fig(\"Masses2016OLS\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can easily test our fit by computing the $R2$ score that we discussed in connection with the functionality of **Scikit-Learn** in the introductory slides.\n", + "Since we are not using **Scikit-Learn** here we can define our own $R2$ function as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "def R2(y_data, y_model):\n", + " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and we would be using it as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "print(R2(Energies,ytilde))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can easily add our **MSE** score as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "print(MSE(Energies,ytilde))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and finally the relative error as" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "def RelativeError(y_data,y_model):\n", + " return abs((y_data-y_model)/y_data)\n", + "print(RelativeError(Energies, ytilde))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The $\\chi^2$ function\n", + "\n", + "Normally, the response (dependent or outcome) variable $y_i$ is the\n", + "outcome of a numerical experiment or another type of experiment and is\n", + "thus only an approximation to the true value. It is then always\n", + "accompanied by an error estimate, often limited to a statistical error\n", + "estimate given by the standard deviation discussed earlier. In the\n", + "discussion here we will treat $y_i$ as our exact value for the\n", + "response variable.\n", + "\n", + "Introducing the standard deviation $\\sigma_i$ for each measurement\n", + "$y_i$, we define now the $\\chi^2$ function (omitting the $1/n$ term)\n", + "as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\chi^2(\\boldsymbol{\\beta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\frac{\\left(y_i-\\tilde{y}_i\\right)^2}{\\sigma_i^2}=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\frac{1}{\\boldsymbol{\\Sigma^2}}\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where the matrix $\\boldsymbol{\\Sigma}$ is a diagonal matrix with $\\sigma_i$ as matrix elements. \n", + "\n", + "\n", + "In order to find the parameters $\\beta_i$ we will then minimize the spread of $\\chi^2(\\boldsymbol{\\beta})$ by requiring" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_j} = \\frac{\\partial }{\\partial \\beta_j}\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(\\frac{y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}}{\\sigma_i}\\right)^2\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which results in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_j} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}\\frac{x_{ij}}{\\sigma_i}\\left(\\frac{y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}}{\\sigma_i}\\right)\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or in a matrix-vector form as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{A}^T\\left( \\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{\\beta}\\right).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we have defined the matrix $\\boldsymbol{A} =\\boldsymbol{X}/\\boldsymbol{\\Sigma}$ with matrix elements $a_{ij} = x_{ij}/\\sigma_i$ and the vector $\\boldsymbol{b}$ with elements $b_i = y_i/\\sigma_i$. \n", + "\n", + "We can rewrite" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{A}^T\\left( \\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{\\beta}\\right),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{A}^T\\boldsymbol{b} = \\boldsymbol{A}^T\\boldsymbol{A}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and if the matrix $\\boldsymbol{A}^T\\boldsymbol{A}$ is invertible we have the solution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta} =\\left(\\boldsymbol{A}^T\\boldsymbol{A}\\right)^{-1}\\boldsymbol{A}^T\\boldsymbol{b}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we then introduce the matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{H} = \\left(\\boldsymbol{A}^T\\boldsymbol{A}\\right)^{-1},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have then the following expression for the parameters $\\beta_j$ (the matrix elements of $\\boldsymbol{H}$ are $h_{ij}$)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_j = \\sum_{k=0}^{p-1}h_{jk}\\sum_{i=0}^{n-1}\\frac{y_i}{\\sigma_i}\\frac{x_{ik}}{\\sigma_i} = \\sum_{k=0}^{p-1}h_{jk}\\sum_{i=0}^{n-1}b_ia_{ik}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We state without proof the expression for the uncertainty in the parameters $\\beta_j$ as (we leave this as an exercise)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\sigma^2(\\beta_j) = \\sum_{i=0}^{n-1}\\sigma_i^2\\left( \\frac{\\partial \\beta_j}{\\partial y_i}\\right)^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "resulting in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\sigma^2(\\beta_j) = \\left(\\sum_{k=0}^{p-1}h_{jk}\\sum_{i=0}^{n-1}a_{ik}\\right)\\left(\\sum_{l=0}^{p-1}h_{jl}\\sum_{m=0}^{n-1}a_{ml}\\right) = h_{jj}!\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first step here is to approximate the function $y$ with a first-order polynomial, that is we write" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y=y(x) \\rightarrow y(x_i) \\approx \\beta_0+\\beta_1 x_i.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By computing the derivatives of $\\chi^2$ with respect to $\\beta_0$ and $\\beta_1$ show that these are given by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_0} = -2\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(\\frac{y_i-\\beta_0-\\beta_1x_{i}}{\\sigma_i^2}\\right)\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_1} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}x_i\\left(\\frac{y_i-\\beta_0-\\beta_1x_{i}}{\\sigma_i^2}\\right)\\right]=0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For a linear fit (a first-order polynomial) we don't need to invert a matrix!! \n", + "Defining" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma = \\sum_{i=0}^{n-1}\\frac{1}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_x = \\sum_{i=0}^{n-1}\\frac{x_{i}}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_y = \\sum_{i=0}^{n-1}\\left(\\frac{y_i}{\\sigma_i^2}\\right),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_{xx} = \\sum_{i=0}^{n-1}\\frac{x_ix_{i}}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_{xy} = \\sum_{i=0}^{n-1}\\frac{y_ix_{i}}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we obtain" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_0 = \\frac{\\gamma_{xx}\\gamma_y-\\gamma_x\\gamma_y}{\\gamma\\gamma_{xx}-\\gamma_x^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_1 = \\frac{\\gamma_{xy}\\gamma-\\gamma_x\\gamma_y}{\\gamma\\gamma_{xx}-\\gamma_x^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This approach (different linear and non-linear regression) suffers\n", + "often from both being underdetermined and overdetermined in the\n", + "unknown coefficients $\\beta_i$. A better approach is to use the\n", + "Singular Value Decomposition (SVD) method discussed below. Or using\n", + "Lasso and Ridge regression. See below.\n", + "\n", + "\n", + "### Fitting an Equation of State for Dense Nuclear Matter\n", + "\n", + "Before we continue, let us introduce yet another example. We are going to fit the\n", + "nuclear equation of state using results from many-body calculations.\n", + "The equation of state we have made available here, as function of\n", + "density, has been derived using modern nucleon-nucleon potentials with\n", + "[the addition of three-body\n", + "forces](https://www.sciencedirect.com/science/article/pii/S0370157399001106). This\n", + "time the file is presented as a standard **csv** file.\n", + "\n", + "The beginning of the Python code here is similar to what you have seen\n", + "before, with the same initializations and declarations. We use also\n", + "**pandas** again, rather extensively in order to organize our data.\n", + "\n", + "The difference now is that we use **Scikit-Learn's** regression tools\n", + "instead of our own matrix inversion implementation. Furthermore, we\n", + "sneak in **Ridge** regression (to be discussed below) which includes a\n", + "hyperparameter $\\lambda$, also to be explained below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.pyplot as plt\n", + "import sklearn.linear_model as skl\n", + "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organize the data into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "X = np.zeros((len(Density),4))\n", + "X[:,3] = Density**(4.0/3.0)\n", + "X[:,2] = Density\n", + "X[:,1] = Density**(2.0/3.0)\n", + "X[:,0] = 1\n", + "\n", + "# We use now Scikit-Learn's linear regressor and ridge regressor\n", + "# OLS part\n", + "clf = skl.LinearRegression().fit(X, Energies)\n", + "ytilde = clf.predict(X)\n", + "EoS['Eols'] = ytilde\n", + "# The mean squared error \n", + "print(\"Mean squared error: %.2f\" % mean_squared_error(Energies, ytilde))\n", + "# Explained variance score: 1 is perfect prediction \n", + "print('Variance score: %.2f' % r2_score(Energies, ytilde))\n", + "# Mean absolute error \n", + "print('Mean absolute error: %.2f' % mean_absolute_error(Energies, ytilde))\n", + "print(clf.coef_, clf.intercept_)\n", + "\n", + "# The Ridge regression with a hyperparameter lambda = 0.1\n", + "_lambda = 0.1\n", + "clf_ridge = skl.Ridge(alpha=_lambda).fit(X, Energies)\n", + "yridge = clf_ridge.predict(X)\n", + "EoS['Eridge'] = yridge\n", + "# The mean squared error \n", + "print(\"Mean squared error: %.2f\" % mean_squared_error(Energies, yridge))\n", + "# Explained variance score: 1 is perfect prediction \n", + "print('Variance score: %.2f' % r2_score(Energies, yridge))\n", + "# Mean absolute error \n", + "print('Mean absolute error: %.2f' % mean_absolute_error(Energies, yridge))\n", + "print(clf_ridge.coef_, clf_ridge.intercept_)\n", + "\n", + "fig, ax = plt.subplots()\n", + "ax.set_xlabel(r'$\\rho[\\mathrm{fm}^{-3}]$')\n", + "ax.set_ylabel(r'Energy per particle')\n", + "ax.plot(EoS['Density'], EoS['Energy'], alpha=0.7, lw=2,\n", + " label='Theoretical data')\n", + "ax.plot(EoS['Density'], EoS['Eols'], alpha=0.7, lw=2, c='m',\n", + " label='OLS')\n", + "ax.plot(EoS['Density'], EoS['Eridge'], alpha=0.7, lw=2, c='g',\n", + " label='Ridge $\\lambda = 0.1$')\n", + "ax.legend()\n", + "save_fig(\"EoSfitting\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The above simple polynomial in density $\\rho$ gives an excellent fit\n", + "to the data. \n", + "\n", + "We note also that there is a small deviation between the\n", + "standard OLS and the Ridge regression at higher densities. We discuss this in more detail\n", + "below.\n", + "\n", + "\n", + "## Splitting our Data in Training and Test data\n", + "\n", + "It is normal in essentially all Machine Learning studies to split the\n", + "data in a training set and a test set (sometimes also an additional\n", + "validation set). **Scikit-Learn** has an own function for this. There\n", + "is no explicit recipe for how much data should be included as training\n", + "data and say test data. An accepted rule of thumb is to use\n", + "approximately $2/3$ to $4/5$ of the data as training data. We will\n", + "postpone a discussion of this splitting to the end of these notes and\n", + "our discussion of the so-called **bias-variance** tradeoff. Here we\n", + "limit ourselves to repeat the above equation of state fitting example\n", + "but now splitting the data into a training set and a test set." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "def R2(y_data, y_model):\n", + " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organized into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "X = np.zeros((len(Density),5))\n", + "X[:,0] = 1\n", + "X[:,1] = Density**(2.0/3.0)\n", + "X[:,2] = Density\n", + "X[:,3] = Density**(4.0/3.0)\n", + "X[:,4] = Density**(5.0/3.0)\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)\n", + "# matrix inversion to find beta\n", + "beta = np.linalg.inv(X_train.T.dot(X_train)).dot(X_train.T).dot(y_train)\n", + "# and then make the prediction\n", + "ytilde = X_train @ beta\n", + "print(\"Training R2\")\n", + "print(R2(y_train,ytilde))\n", + "print(\"Training MSE\")\n", + "print(MSE(y_train,ytilde))\n", + "ypredict = X_test @ beta\n", + "print(\"Test R2\")\n", + "print(R2(y_test,ypredict))\n", + "print(\"Test MSE\")\n", + "print(MSE(y_test,ypredict))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The Boston housing data example\n", + "\n", + "The Boston housing \n", + "data set was originally a part of UCI Machine Learning Repository\n", + "and has been removed now. The data set is now included in **Scikit-Learn**'s \n", + "library. There are 506 samples and 13 feature (predictor) variables\n", + "in this data set. The objective is to predict the value of prices of\n", + "the house using the features (predictors) listed here.\n", + "\n", + "The features/predictors are\n", + "1. CRIM: Per capita crime rate by town\n", + "\n", + "2. ZN: Proportion of residential land zoned for lots over 25000 square feet\n", + "\n", + "3. INDUS: Proportion of non-retail business acres per town\n", + "\n", + "4. CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n", + "\n", + "5. NOX: Nitric oxide concentration (parts per 10 million)\n", + "\n", + "6. RM: Average number of rooms per dwelling\n", + "\n", + "7. AGE: Proportion of owner-occupied units built prior to 1940\n", + "\n", + "8. DIS: Weighted distances to five Boston employment centers\n", + "\n", + "9. RAD: Index of accessibility to radial highways\n", + "\n", + "10. TAX: Full-value property tax rate per USD10000\n", + "\n", + "11. B: $1000(Bk - 0.63)^2$, where $Bk$ is the proportion of [people of African American descent] by town\n", + "\n", + "12. LSTAT: Percentage of lower status of the population\n", + "\n", + "13. MEDV: Median value of owner-occupied homes in USD 1000s\n", + "\n", + "## Housing data, the code\n", + "We start by importing the libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt \n", + "\n", + "import pandas as pd \n", + "import seaborn as sns" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and load the Boston Housing DataSet from **Scikit-Learn**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.datasets import load_boston\n", + "\n", + "boston_dataset = load_boston()\n", + "\n", + "# boston_dataset is a dictionary\n", + "# let's check what it contains\n", + "boston_dataset.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we invoke Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)\n", + "boston.head()\n", + "boston['MEDV'] = boston_dataset.target" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and preprocess the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# check for missing values in all the columns\n", + "boston.isnull().sum()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then visualize the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# set the size of the figure\n", + "sns.set(rc={'figure.figsize':(11.7,8.27)})\n", + "\n", + "# plot a histogram showing the distribution of the target values\n", + "sns.distplot(boston['MEDV'], bins=30)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is now useful to look at the correlation matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# compute the pair wise correlation for all columns \n", + "correlation_matrix = boston.corr().round(2)\n", + "# use the heatmap function from seaborn to plot the correlation matrix\n", + "# annot = True to print the values inside the square\n", + "sns.heatmap(data=correlation_matrix, annot=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the above coorelation plot we can see that **MEDV** is strongly correlated to **LSTAT** and **RM**. We see also that **RAD** and **TAX** are stronly correlated, but we don't include this in our features together to avoid multi-colinearity" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(20, 5))\n", + "\n", + "features = ['LSTAT', 'RM']\n", + "target = boston['MEDV']\n", + "\n", + "for i, col in enumerate(features):\n", + " plt.subplot(1, len(features) , i+1)\n", + " x = boston[col]\n", + " y = target\n", + " plt.scatter(x, y, marker='o')\n", + " plt.title(col)\n", + " plt.xlabel(col)\n", + " plt.ylabel('MEDV')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we start training our model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "X = pd.DataFrame(np.c_[boston['LSTAT'], boston['RM']], columns = ['LSTAT','RM'])\n", + "Y = boston['MEDV']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We split the data into training and test sets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# splits the training and test data set in 80% : 20%\n", + "# assign random_state to any value.This ensures consistency.\n", + "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state=5)\n", + "print(X_train.shape)\n", + "print(X_test.shape)\n", + "print(Y_train.shape)\n", + "print(Y_test.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we use the linear regression functionality from **Scikit-Learn**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import mean_squared_error, r2_score\n", + "\n", + "lin_model = LinearRegression()\n", + "lin_model.fit(X_train, Y_train)\n", + "\n", + "# model evaluation for training set\n", + "\n", + "y_train_predict = lin_model.predict(X_train)\n", + "rmse = (np.sqrt(mean_squared_error(Y_train, y_train_predict)))\n", + "r2 = r2_score(Y_train, y_train_predict)\n", + "\n", + "print(\"The model performance for training set\")\n", + "print(\"--------------------------------------\")\n", + "print('RMSE is {}'.format(rmse))\n", + "print('R2 score is {}'.format(r2))\n", + "print(\"\\n\")\n", + "\n", + "# model evaluation for testing set\n", + "\n", + "y_test_predict = lin_model.predict(X_test)\n", + "# root mean square error of the model\n", + "rmse = (np.sqrt(mean_squared_error(Y_test, y_test_predict)))\n", + "\n", + "# r-squared score of the model\n", + "r2 = r2_score(Y_test, y_test_predict)\n", + "\n", + "print(\"The model performance for testing set\")\n", + "print(\"--------------------------------------\")\n", + "print('RMSE is {}'.format(rmse))\n", + "print('R2 score is {}'.format(r2))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# plotting the y_test vs y_pred\n", + "# ideally should have been a straight line\n", + "plt.scatter(Y_test, y_test_predict)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reducing the number of degrees of freedom, overarching view\n", + "\n", + "Many Machine Learning problems involve thousands or even millions of\n", + "features for each training instance. Not only does this make training\n", + "extremely slow, it can also make it much harder to find a good\n", + "solution, as we will see. This problem is often referred to as the\n", + "curse of dimensionality. Fortunately, in real-world problems, it is\n", + "often possible to reduce the number of features considerably, turning\n", + "an intractable problem into a tractable one.\n", + "\n", + "Later we will discuss some of the most popular dimensionality reduction\n", + "techniques: the principal component analysis (PCA), Kernel PCA, and\n", + "Locally Linear Embedding (LLE). \n", + "\n", + "\n", + "Principal component analysis and its various variants deal with the\n", + "problem of fitting a low-dimensional [affine\n", + "subspace](https://en.wikipedia.org/wiki/Affine_space) to a set of of\n", + "data points in a high-dimensional space. With its family of methods it\n", + "is one of the most used tools in data modeling, compression and\n", + "visualization.\n", + "\n", + "\n", + "Before we proceed however, we will discuss how to preprocess our\n", + "data. Till now and in connection with our previous examples we have\n", + "not met so many cases where we are too sensitive to the scaling of our\n", + "data. Normally the data may need a rescaling and/or may be sensitive\n", + "to extreme values. Scaling the data renders our inputs much more\n", + "suitable for the algorithms we want to employ.\n", + "\n", + "**Scikit-Learn** has several functions which allow us to rescale the\n", + "data, normally resulting in much better results in terms of various\n", + "accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n", + "ensures that for each feature/predictor we study the mean value is\n", + "zero and the variance is one (every column in the design/feature\n", + "matrix). This scaling has the drawback that it does not ensure that\n", + "we have a particular maximum or minimum in our data set. Another\n", + "function included in **Scikit-Learn** is the **MinMaxScaler** which\n", + "ensures that all features are exactly between $0$ and $1$. The\n", + "\n", + "\n", + "The **Normalizer** scales each data\n", + "point such that the feature vector has a euclidean length of one. In other words, it\n", + "projects a data point on the circle (or sphere in the case of higher dimensions) with a\n", + "radius of 1. This means every data point is scaled by a different number (by the\n", + "inverse of it’s length).\n", + "This normalization is often used when only the direction (or angle) of the data matters,\n", + "not the length of the feature vector.\n", + "\n", + "The **RobustScaler** works similarly to the StandardScaler in that it\n", + "ensures statistical properties for each feature that guarantee that\n", + "they are on the same scale. However, the RobustScaler uses the median\n", + "and quartiles, instead of mean and variance. This makes the\n", + "RobustScaler ignore data points that are very different from the rest\n", + "(like measurement errors). These odd data points are also called\n", + "outliers, and might often lead to trouble for other scaling\n", + "techniques.\n", + "\n", + "\n", + "### Simple preprocessing examples, Franke function and regression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import sklearn.linear_model as skl\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "\n", + "def FrankeFunction(x,y):\n", + "\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n", + "\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n", + "\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n", + "\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n", + "\treturn term1 + term2 + term3 + term4\n", + "\n", + "\n", + "def create_X(x, y, n ):\n", + "\tif len(x.shape) > 1:\n", + "\t\tx = np.ravel(x)\n", + "\t\ty = np.ravel(y)\n", + "\n", + "\tN = len(x)\n", + "\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n", + "\tX = np.ones((N,l))\n", + "\n", + "\tfor i in range(1,n+1):\n", + "\t\tq = int((i)*(i+1)/2)\n", + "\t\tfor k in range(i+1):\n", + "\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n", + "\n", + "\treturn X\n", + "\n", + "\n", + "# Making meshgrid of datapoints and compute Franke's function\n", + "n = 5\n", + "N = 1000\n", + "x = np.sort(np.random.uniform(0, 1, N))\n", + "y = np.sort(np.random.uniform(0, 1, N))\n", + "z = FrankeFunction(x, y)\n", + "X = create_X(x, y, n=n) \n", + "# split in training and test data\n", + "X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)\n", + "\n", + "\n", + "clf = skl.LinearRegression().fit(X_train, y_train)\n", + "\n", + "# The mean squared error and R2 score\n", + "print(\"MSE before scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test), y_test)))\n", + "print(\"R2 score before scaling {:.2f}\".format(clf.score(X_test,y_test)))\n", + "\n", + "scaler = StandardScaler()\n", + "scaler.fit(X_train)\n", + "X_train_scaled = scaler.transform(X_train)\n", + "X_test_scaled = scaler.transform(X_test)\n", + "\n", + "print(\"Feature min values before scaling:\\n {}\".format(X_train.min(axis=0)))\n", + "print(\"Feature max values before scaling:\\n {}\".format(X_train.max(axis=0)))\n", + "\n", + "print(\"Feature min values after scaling:\\n {}\".format(X_train_scaled.min(axis=0)))\n", + "print(\"Feature max values after scaling:\\n {}\".format(X_train_scaled.max(axis=0)))\n", + "\n", + "clf = skl.LinearRegression().fit(X_train_scaled, y_train)\n", + "\n", + "\n", + "print(\"MSE after scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))\n", + "print(\"R2 score for scaled data: {:.2f}\".format(clf.score(X_test_scaled,y_test)))" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/doc/LectureNotes/chapter2.ipynb b/doc/LectureNotes/chapter2.ipynb new file mode 100644 index 000000000..721b43cde --- /dev/null +++ b/doc/LectureNotes/chapter2.ipynb @@ -0,0 +1,1355 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Resampling Methods\n", + "\n", + "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureSept3.mp4?vrtx=view-as-webpage)\n", + "\n", + "\n", + "## Introduction\n", + "\n", + "Resampling methods are an indispensable tool in modern\n", + "statistics. They involve repeatedly drawing samples from a training\n", + "set and refitting a model of interest on each sample in order to\n", + "obtain additional information about the fitted model. For example, in\n", + "order to estimate the variability of a linear regression fit, we can\n", + "repeatedly draw different samples from the training data, fit a linear\n", + "regression to each new sample, and then examine the extent to which\n", + "the resulting fits differ. Such an approach may allow us to obtain\n", + "information that would not be available from fitting the model only\n", + "once using the original training sample.\n", + "\n", + "Two resampling methods are often used in Machine Learning analyses,\n", + "1. The **bootstrap method**\n", + "\n", + "2. and **Cross-Validation**\n", + "\n", + "In addition there are several other methods such as the Jackknife and the Blocking methods. We will discuss in particular\n", + "cross-validation and the bootstrap method. \n", + "\n", + "\n", + "Resampling approaches can be computationally expensive, because they\n", + "involve fitting the same statistical method multiple times using\n", + "different subsets of the training data. However, due to recent\n", + "advances in computing power, the computational requirements of\n", + "resampling methods generally are not prohibitive. In this chapter, we\n", + "discuss two of the most commonly used resampling methods,\n", + "cross-validation and the bootstrap. Both methods are important tools\n", + "in the practical application of many statistical learning\n", + "procedures. For example, cross-validation can be used to estimate the\n", + "test error associated with a given statistical learning method in\n", + "order to evaluate its performance, or to select the appropriate level\n", + "of flexibility. The process of evaluating a model’s performance is\n", + "known as model assessment, whereas the process of selecting the proper\n", + "level of flexibility for a model is known as model selection. The\n", + "bootstrap is widely used.\n", + "\n", + "\n", + "* Our simulations can be treated as *computer experiments*. This is particularly the case for Monte Carlo methods\n", + "\n", + "* The results can be analysed with the same statistical tools as we would use analysing experimental data.\n", + "\n", + "* As in all experiments, we are looking for expectation values and an estimate of how accurate they are, i.e., possible sources for errors.\n", + "\n", + "## Reminder on Statistics\n", + "\n", + "\n", + "* As in other experiments, many numerical experiments have two classes of errors:\n", + "\n", + " * Statistical errors\n", + "\n", + " * Systematical errors\n", + "\n", + "\n", + "* Statistical errors can be estimated using standard tools from statistics\n", + "\n", + "* Systematical errors are method specific and must be treated differently from case to case. \n", + "\n", + "The\n", + "advantage of doing linear regression is that we actually end up with\n", + "analytical expressions for several statistical quantities. \n", + "Standard least squares and Ridge regression allow us to\n", + "derive quantities like the variance and other expectation values in a\n", + "rather straightforward way.\n", + "\n", + "\n", + "It is assumed that $\\varepsilon_i\n", + "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", + "independent, i.e.:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*} \n", + "\\mbox{Cov}(\\varepsilon_{i_1},\n", + "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", + "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The randomness of $\\varepsilon_i$ implies that\n", + "$\\mathbf{y}_i$ is also a random variable. In particular,\n", + "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", + "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", + "non-random scalar. To specify the parameters of the distribution of\n", + "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "\n", + "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", + "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", + "row number $i$ and perform a sum over all values $p$.\n", + "\n", + "\n", + "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", + "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", + "which describe our data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We approximate this function with our model from the solution of the linear regression equations, that is our\n", + "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*} \n", + "\\mathbb{E}(y_i) & =\n", + "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", + "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "while\n", + "its variance is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", + "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", + "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", + "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", + "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", + "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", + "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", + "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", + "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", + "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", + "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", + "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD). \n", + "\n", + "\n", + "With the OLS expressions for the parameters $\\boldsymbol{\\beta}$ we can evaluate the expectation value" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}(\\boldsymbol{\\beta}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This means that the estimator of the regression parameters is unbiased.\n", + "\n", + "We can also calculate the variance\n", + "\n", + "The variance of $\\boldsymbol{\\beta}$ is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{eqnarray*}\n", + "\\mbox{Var}(\\boldsymbol{\\beta}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", + "\\\\\n", + "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", + "\\\\\n", + "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", + "% \\\\\n", + "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", + "\\\\\n", + "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", + "\\end{eqnarray*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", + "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", + "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", + "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", + "variance of the estimate of the $j$-th regression coefficient:\n", + "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 \\sqrt{\n", + "[(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} }$. This may be used to\n", + "construct a confidence interval for the estimates.\n", + "\n", + "\n", + "In a similar way, we can obtain analytical expressions for say the\n", + "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", + "when we employ Ridge regression, allowing us again to define a confidence interval. \n", + "\n", + "It is rather straightforward to show that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see clearly that \n", + "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", + "\n", + "We can also compute the variance as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", + "\n", + "With this, we can compute the difference" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The difference is non-negative definite since each component of the\n", + "matrix product is non-negative definite. \n", + "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below. \n", + "\n", + "\n", + "\n", + "## Resampling methods\n", + "\n", + "With all these analytical equations for both the OLS and Ridge\n", + "regression, we will now outline how to assess a given model. This will\n", + "lead us to a discussion of the so-called bias-variance tradeoff (see\n", + "below) and so-called resampling methods.\n", + "\n", + "One of the quantities we have discussed as a way to measure errors is\n", + "the mean-squared error (MSE), mainly used for fitting of continuous\n", + "functions. Another choice is the absolute error.\n", + "\n", + "In the discussions below we will focus on the MSE and in particular since we will split the data into test and training data,\n", + "we discuss the\n", + "1. prediction error or simply the **test error** $\\mathrm{Err_{Test}}$, where we have a fixed training set and the test error is the MSE arising from the data reserved for testing. We discuss also the \n", + "\n", + "2. training error $\\mathrm{Err_{Train}}$, which is the average loss over the training data.\n", + "\n", + "As our model becomes more and more complex, more of the training data tends to used. The training may thence adapt to more complicated structures in the data. This may lead to a decrease in the bias (see below for code example) and a slight increase of the variance for the test error.\n", + "For a certain level of complexity the test error will reach minimum, before starting to increase again. The\n", + "training error reaches a saturation.\n", + "\n", + "\n", + "\n", + "Two famous\n", + "resampling methods are the **independent bootstrap** and **the jackknife**. \n", + "\n", + "The jackknife is a special case of the independent bootstrap. Still, the jackknife was made\n", + "popular prior to the independent bootstrap. And as the popularity of\n", + "the independent bootstrap soared, new variants, such as **the dependent bootstrap**.\n", + "\n", + "The Jackknife and independent bootstrap work for\n", + "independent, identically distributed random variables.\n", + "If these conditions are not\n", + "satisfied, the methods will fail. Yet, it should be said that if the data are\n", + "independent, identically distributed, and we only want to estimate the\n", + "variance of $\\overline{X}$ (which often is the case), then there is no\n", + "need for bootstrapping. \n", + "\n", + "\n", + "The Jackknife works by making many replicas of the estimator $\\widehat{\\theta}$. \n", + "The jackknife is a resampling method where we systematically leave out one observation from the vector of observed values $\\boldsymbol{x} = (x_1,x_2,\\cdots,X_n)$. \n", + "Let $\\boldsymbol{x}_i$ denote the vector" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{x}_i = (x_1,x_2,\\cdots,x_{i-1},x_{i+1},\\cdots,x_n),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which equals the vector $\\boldsymbol{x}$ with the exception that observation\n", + "number $i$ is left out. Using this notation, define\n", + "$\\widehat{\\theta}_i$ to be the estimator\n", + "$\\widehat{\\theta}$ computed using $\\vec{X}_i$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from numpy import *\n", + "from numpy.random import randint, randn\n", + "from time import time\n", + "\n", + "def jackknife(data, stat):\n", + " n = len(data);t = zeros(n); inds = arange(n); t0 = time()\n", + " ## 'jackknifing' by leaving out an observation for each i \n", + " for i in range(n):\n", + " t[i] = stat(delete(data,i) )\n", + "\n", + " # analysis \n", + " print(\"Runtime: %g sec\" % (time()-t0)); print(\"Jackknife Statistics :\")\n", + " print(\"original bias std. error\")\n", + " print(\"%8g %14g %15g\" % (stat(data),(n-1)*mean(t)/n, (n*var(t))**.5))\n", + "\n", + " return t\n", + "\n", + "\n", + "# Returns mean of data samples \n", + "def stat(data):\n", + " return mean(data)\n", + "\n", + "\n", + "mu, sigma = 100, 15\n", + "datapoints = 10000\n", + "x = mu + sigma*random.randn(datapoints)\n", + "# jackknife returns the data sample \n", + "t = jackknife(x, stat)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Bootstrap\n", + "\n", + "Bootstrapping is a nonparametric approach to statistical inference\n", + "that substitutes computation for more traditional distributional\n", + "assumptions and asymptotic results. Bootstrapping offers a number of\n", + "advantages: \n", + "1. The bootstrap is quite general, although there are some cases in which it fails. \n", + "\n", + "2. Because it does not require distributional assumptions (such as normally distributed errors), the bootstrap can provide more accurate inferences when the data are not well behaved or when the sample size is small. \n", + "\n", + "3. It is possible to apply the bootstrap to statistics with sampling distributions that are difficult to derive, even asymptotically. \n", + "\n", + "4. It is relatively simple to apply the bootstrap to complex data-collection plans (such as stratified and clustered samples).\n", + "\n", + "Since $\\widehat{\\theta} = \\widehat{\\theta}(\\boldsymbol{X})$ is a function of random variables,\n", + "$\\widehat{\\theta}$ itself must be a random variable. Thus it has\n", + "a pdf, call this function $p(\\boldsymbol{t})$. The aim of the bootstrap is to\n", + "estimate $p(\\boldsymbol{t})$ by the relative frequency of\n", + "$\\widehat{\\theta}$. You can think of this as using a histogram\n", + "in the place of $p(\\boldsymbol{t})$. If the relative frequency closely\n", + "resembles $p(\\vec{t})$, then using numerics, it is straight forward to\n", + "estimate all the interesting parameters of $p(\\boldsymbol{t})$ using point\n", + "estimators. \n", + "\n", + "\n", + "\n", + "In the case that $\\widehat{\\theta}$ has\n", + "more than one component, and the components are independent, we use the\n", + "same estimator on each component separately. If the probability\n", + "density function of $X_i$, $p(x)$, had been known, then it would have\n", + "been straight forward to do this by: \n", + "1. Drawing lots of numbers from $p(x)$, suppose we call one such set of numbers $(X_1^*, X_2^*, \\cdots, X_n^*)$. \n", + "\n", + "2. Then using these numbers, we could compute a replica of $\\widehat{\\theta}$ called $\\widehat{\\theta}^*$. \n", + "\n", + "By repeated use of (1) and (2), many\n", + "estimates of $\\widehat{\\theta}$ could have been obtained. The\n", + "idea is to use the relative frequency of $\\widehat{\\theta}^*$\n", + "(think of a histogram) as an estimate of $p(\\boldsymbol{t})$.\n", + "\n", + "\n", + "But\n", + "unless there is enough information available about the process that\n", + "generated $X_1,X_2,\\cdots,X_n$, $p(x)$ is in general\n", + "unknown. Therefore, [Efron in 1979](https://projecteuclid.org/euclid.aos/1176344552) asked the\n", + "question: What if we replace $p(x)$ by the relative frequency\n", + "of the observation $X_i$; if we draw observations in accordance with\n", + "the relative frequency of the observations, will we obtain the same\n", + "result in some asymptotic sense? The answer is yes.\n", + "\n", + "\n", + "Instead of generating the histogram for the relative\n", + "frequency of the observation $X_i$, just draw the values\n", + "$(X_1^*,X_2^*,\\cdots,X_n^*)$ with replacement from the vector\n", + "$\\boldsymbol{X}$. \n", + "\n", + "\n", + "The independent bootstrap works like this: \n", + "\n", + "1. Draw with replacement $n$ numbers for the observed variables $\\boldsymbol{x} = (x_1,x_2,\\cdots,x_n)$. \n", + "\n", + "2. Define a vector $\\boldsymbol{x}^*$ containing the values which were drawn from $\\boldsymbol{x}$. \n", + "\n", + "3. Using the vector $\\boldsymbol{x}^*$ compute $\\widehat{\\theta}^*$ by evaluating $\\widehat \\theta$ under the observations $\\boldsymbol{x}^*$. \n", + "\n", + "4. Repeat this process $k$ times. \n", + "\n", + "When you are done, you can draw a histogram of the relative frequency\n", + "of $\\widehat \\theta^*$. This is your estimate of the probability\n", + "distribution $p(t)$. Using this probability distribution you can\n", + "estimate any statistics thereof. In principle you never draw the\n", + "histogram of the relative frequency of $\\widehat{\\theta}^*$. Instead\n", + "you use the estimators corresponding to the statistic of interest. For\n", + "example, if you are interested in estimating the variance of $\\widehat\n", + "\\theta$, apply the etsimator $\\widehat \\sigma^2$ to the values\n", + "$\\widehat \\theta ^*$.\n", + "\n", + "\n", + "\n", + "The following code starts with a Gaussian distribution with mean value\n", + "$\\mu =100$ and variance $\\sigma=15$. We use this to generate the data\n", + "used in the bootstrap analysis. The bootstrap analysis returns a data\n", + "set after a given number of bootstrap operations (as many as we have\n", + "data points). This data set consists of estimated mean values for each\n", + "bootstrap operation. The histogram generated by the bootstrap method\n", + "shows that the distribution for these mean values is also a Gaussian,\n", + "centered around the mean value $\\mu=100$ but with standard deviation\n", + "$\\sigma/\\sqrt{n}$, where $n$ is the number of bootstrap samples (in\n", + "this case the same as the number of original data points). The value\n", + "of the standard deviation is what we expect from the central limit\n", + "theorem." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "from numpy import *\n", + "from numpy.random import randint, randn\n", + "from time import time\n", + "import matplotlib.mlab as mlab\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Returns mean of bootstrap samples \n", + "def stat(data):\n", + " return mean(data)\n", + "\n", + "# Bootstrap algorithm\n", + "def bootstrap(data, statistic, R):\n", + " t = zeros(R); n = len(data); inds = arange(n); t0 = time()\n", + " # non-parametric bootstrap \n", + " for i in range(R):\n", + " t[i] = statistic(data[randint(0,n,n)])\n", + "\n", + " # analysis \n", + " print(\"Runtime: %g sec\" % (time()-t0)); print(\"Bootstrap Statistics :\")\n", + " print(\"original bias std. error\")\n", + " print(\"%8g %8g %14g %15g\" % (statistic(data), std(data),mean(t),std(t)))\n", + " return t\n", + "\n", + "\n", + "mu, sigma = 100, 15\n", + "datapoints = 10000\n", + "x = mu + sigma*random.randn(datapoints)\n", + "# bootstrap returns the data sample \n", + "t = bootstrap(x, stat, datapoints)\n", + "# the histogram of the bootstrapped data \n", + "n, binsboot, patches = plt.hist(t, 50, normed=1, facecolor='red', alpha=0.75)\n", + "\n", + "# add a 'best fit' line \n", + "y = mlab.normpdf( binsboot, mean(t), std(t))\n", + "lt = plt.plot(binsboot, y, 'r--', linewidth=1)\n", + "plt.xlabel('Smarts')\n", + "plt.ylabel('Probability')\n", + "plt.axis([99.5, 100.6, 0, 3.0])\n", + "plt.grid(True)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Various steps in cross-validation\n", + "\n", + "When the repetitive splitting of the data set is done randomly,\n", + "samples may accidently end up in a fast majority of the splits in\n", + "either training or test set. Such samples may have an unbalanced\n", + "influence on either model building or prediction evaluation. To avoid\n", + "this $k$-fold cross-validation structures the data splitting. The\n", + "samples are divided into $k$ more or less equally sized exhaustive and\n", + "mutually exclusive subsets. In turn (at each split) one of these\n", + "subsets plays the role of the test set while the union of the\n", + "remaining subsets constitutes the training set. Such a splitting\n", + "warrants a balanced representation of each sample in both training and\n", + "test set over the splits. Still the division into the $k$ subsets\n", + "involves a degree of randomness. This may be fully excluded when\n", + "choosing $k=n$. This particular case is referred to as leave-one-out\n", + "cross-validation (LOOCV). \n", + "\n", + "\n", + "* Define a range of interest for the penalty parameter.\n", + "\n", + "* Divide the data set into training and test set comprising samples $\\{1, \\ldots, n\\} \\setminus i$ and $\\{ i \\}$, respectively.\n", + "\n", + "* Fit the linear regression model by means of ridge estimation for each $\\lambda$ in the grid using the training set, and the corresponding estimate of the error variance $\\boldsymbol{\\sigma}_{-i}^2(\\lambda)$, as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "\\boldsymbol{\\beta}_{-i}(\\lambda) & = ( \\boldsymbol{X}_{-i, \\ast}^{T}\n", + "\\boldsymbol{X}_{-i, \\ast} + \\lambda \\boldsymbol{I}_{pp})^{-1}\n", + "\\boldsymbol{X}_{-i, \\ast}^{T} \\boldsymbol{y}_{-i}\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* Evaluate the prediction performance of these models on the test set by $\\log\\{L[y_i, \\boldsymbol{X}_{i, \\ast}; \\boldsymbol{\\beta}_{-i}(\\lambda), \\boldsymbol{\\sigma}_{-i}^2(\\lambda)]\\}$. Or, by the prediction error $|y_i - \\boldsymbol{X}_{i, \\ast} \\boldsymbol{\\beta}_{-i}(\\lambda)|$, the relative error, the error squared or the R2 score function.\n", + "\n", + "* Repeat the first three steps such that each sample plays the role of the test set once.\n", + "\n", + "* Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "\\frac{1}{n} \\sum_{i = 1}^n \\log\\{L[y_i, \\mathbf{X}_{i, \\ast}; \\boldsymbol{\\beta}_{-i}(\\lambda), \\boldsymbol{\\sigma}_{-i}^2(\\lambda)]\\}.\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For the various values of $k$\n", + "\n", + "1. shuffle the dataset randomly.\n", + "\n", + "2. Split the dataset into $k$ groups.\n", + "\n", + "3. For each unique group:\n", + "\n", + "a. Decide which group to use as set for test data\n", + "\n", + "b. Take the remaining groups as a training data set\n", + "\n", + "c. Fit a model on the training set and evaluate it on the test set\n", + "\n", + "d. Retain the evaluation score and discard the model\n", + "\n", + "\n", + "5. Summarize the model using the sample of model evaluation scores\n", + "\n", + "The code here uses Ridge regression with cross-validation (CV) resampling and $k$-fold CV in order to fit a specific polynomial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import KFold\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.model_selection import cross_val_score\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", + "\n", + "# Generate the data.\n", + "nsamples = 100\n", + "x = np.random.randn(nsamples)\n", + "y = 3*x**2 + np.random.randn(nsamples)\n", + "\n", + "## Cross-validation on Ridge regression using KFold only\n", + "\n", + "# Decide degree on polynomial to fit\n", + "poly = PolynomialFeatures(degree = 6)\n", + "\n", + "# Decide which values of lambda to use\n", + "nlambdas = 500\n", + "lambdas = np.logspace(-3, 5, nlambdas)\n", + "\n", + "# Initialize a KFold instance\n", + "k = 5\n", + "kfold = KFold(n_splits = k)\n", + "\n", + "# Perform the cross-validation to estimate MSE\n", + "scores_KFold = np.zeros((nlambdas, k))\n", + "\n", + "i = 0\n", + "for lmb in lambdas:\n", + " ridge = Ridge(alpha = lmb)\n", + " j = 0\n", + " for train_inds, test_inds in kfold.split(x):\n", + " xtrain = x[train_inds]\n", + " ytrain = y[train_inds]\n", + "\n", + " xtest = x[test_inds]\n", + " ytest = y[test_inds]\n", + "\n", + " Xtrain = poly.fit_transform(xtrain[:, np.newaxis])\n", + " ridge.fit(Xtrain, ytrain[:, np.newaxis])\n", + "\n", + " Xtest = poly.fit_transform(xtest[:, np.newaxis])\n", + " ypred = ridge.predict(Xtest)\n", + "\n", + " scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred)\n", + "\n", + " j += 1\n", + " i += 1\n", + "\n", + "\n", + "estimated_mse_KFold = np.mean(scores_KFold, axis = 1)\n", + "\n", + "## Cross-validation using cross_val_score from sklearn along with KFold\n", + "\n", + "# kfold is an instance initialized above as:\n", + "# kfold = KFold(n_splits = k)\n", + "\n", + "estimated_mse_sklearn = np.zeros(nlambdas)\n", + "i = 0\n", + "for lmb in lambdas:\n", + " ridge = Ridge(alpha = lmb)\n", + "\n", + " X = poly.fit_transform(x[:, np.newaxis])\n", + " estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold)\n", + "\n", + " # cross_val_score return an array containing the estimated negative mse for every fold.\n", + " # we have to the the mean of every array in order to get an estimate of the mse of the model\n", + " estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n", + "\n", + " i += 1\n", + "\n", + "## Plot and compare the slightly different ways to perform cross-validation\n", + "\n", + "plt.figure()\n", + "\n", + "plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n", + "plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold')\n", + "\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('mse')\n", + "\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The bias-variance tradeoff\n", + "\n", + "\n", + "We will discuss the bias-variance tradeoff in the context of\n", + "continuous predictions such as regression. However, many of the\n", + "intuitions and ideas discussed here also carry over to classification\n", + "tasks. Consider a dataset $\\mathcal{L}$ consisting of the data\n", + "$\\mathbf{X}_\\mathcal{L}=\\{(y_j, \\boldsymbol{x}_j), j=0\\ldots n-1\\}$. \n", + "\n", + "Let us assume that the true data is generated from a noisy model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y}=f(\\boldsymbol{x}) + \\boldsymbol{\\epsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\epsilon$ is normally distributed with mean zero and standard deviation $\\sigma^2$.\n", + "\n", + "In our derivation of the ordinary least squares method we defined then\n", + "an approximation to the function $f$ in terms of the parameters\n", + "$\\boldsymbol{\\beta}$ and the design matrix $\\boldsymbol{X}$ which embody our model,\n", + "that is $\\boldsymbol{\\tilde{y}}=\\boldsymbol{X}\\boldsymbol{\\beta}$. \n", + "\n", + "Thereafter we found the parameters $\\boldsymbol{\\beta}$ by optimizing the means squared error via the so-called cost function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta}) =\\frac{1}{n}\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2=\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right].\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can rewrite this as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\frac{1}{n}\\sum_i(f_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2+\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2+\\sigma^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The three terms represent the square of the bias of the learning\n", + "method, which can be thought of as the error caused by the simplifying\n", + "assumptions built into the method. The second term represents the\n", + "variance of the chosen model and finally the last terms is variance of\n", + "the error $\\boldsymbol{\\epsilon}$.\n", + "\n", + "To derive this equation, we need to recall that the variance of $\\boldsymbol{y}$ and $\\boldsymbol{\\epsilon}$ are both equal to $\\sigma^2$. The mean value of $\\boldsymbol{\\epsilon}$ is by definition equal to zero. Furthermore, the function $f$ is not a stochastics variable, idem for $\\boldsymbol{\\tilde{y}}$.\n", + "We use a more compact notation in terms of the expectation value" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{f}+\\boldsymbol{\\epsilon}-\\boldsymbol{\\tilde{y}})^2\\right],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and adding and subtracting $\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]$ we get" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{f}+\\boldsymbol{\\epsilon}-\\boldsymbol{\\tilde{y}}+\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2\\right],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which, using the abovementioned expectation values can be rewritten as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2\\right]+\\mathrm{Var}\\left[\\boldsymbol{\\tilde{y}}\\right]+\\sigma^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "that is the rewriting in terms of the so-called bias, the variance of the model $\\boldsymbol{\\tilde{y}}$ and the variance of $\\boldsymbol{\\epsilon}$." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.utils import resample\n", + "\n", + "np.random.seed(2018)\n", + "\n", + "n = 500\n", + "n_boostraps = 100\n", + "degree = 18 # A quite high value, just to show.\n", + "noise = 0.1\n", + "\n", + "# Make data set.\n", + "x = np.linspace(-1, 3, n).reshape(-1, 1)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1, x.shape)\n", + "\n", + "# Hold out some test data that is never used in training.\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "# Combine x transformation and model into one operation.\n", + "# Not neccesary, but convenient.\n", + "model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n", + "\n", + "# The following (m x n_bootstraps) matrix holds the column vectors y_pred\n", + "# for each bootstrap iteration.\n", + "y_pred = np.empty((y_test.shape[0], n_boostraps))\n", + "for i in range(n_boostraps):\n", + " x_, y_ = resample(x_train, y_train)\n", + "\n", + " # Evaluate the new model on the same test data each time.\n", + " y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()\n", + "\n", + "# Note: Expectations and variances taken w.r.t. different training\n", + "# data sets, hence the axis=1. Subsequent means are taken across the test data\n", + "# set in order to obtain a total value, but before this we have error/bias/variance\n", + "# calculated per data point in the test set.\n", + "# Note 2: The use of keepdims=True is important in the calculation of bias as this \n", + "# maintains the column vector form. Dropping this yields very unexpected results.\n", + "error = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n", + "bias = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n", + "variance = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n", + "print('Error:', error)\n", + "print('Bias^2:', bias)\n", + "print('Var:', variance)\n", + "print('{} >= {} + {} = {}'.format(error, bias, variance, bias+variance))\n", + "\n", + "plt.plot(x[::5, :], y[::5, :], label='f(x)')\n", + "plt.scatter(x_test, y_test, label='Data points')\n", + "plt.scatter(x_test, np.mean(y_pred, axis=1), label='Pred')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.utils import resample\n", + "\n", + "np.random.seed(2018)\n", + "\n", + "n = 40\n", + "n_boostraps = 100\n", + "maxdegree = 14\n", + "\n", + "\n", + "# Make data set.\n", + "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", + "error = np.zeros(maxdegree)\n", + "bias = np.zeros(maxdegree)\n", + "variance = np.zeros(maxdegree)\n", + "polydegree = np.zeros(maxdegree)\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "for degree in range(maxdegree):\n", + " model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n", + " y_pred = np.empty((y_test.shape[0], n_boostraps))\n", + " for i in range(n_boostraps):\n", + " x_, y_ = resample(x_train, y_train)\n", + " y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()\n", + "\n", + " polydegree[degree] = degree\n", + " error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n", + " bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n", + " variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n", + " print('Polynomial degree:', degree)\n", + " print('Error:', error[degree])\n", + " print('Bias^2:', bias[degree])\n", + " print('Var:', variance[degree])\n", + " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n", + "\n", + "plt.plot(polydegree, error, label='Error')\n", + "plt.plot(polydegree, bias, label='bias')\n", + "plt.plot(polydegree, variance, label='Variance')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The bias-variance tradeoff summarizes the fundamental tension in\n", + "machine learning, particularly supervised learning, between the\n", + "complexity of a model and the amount of training data needed to train\n", + "it. Since data is often limited, in practice it is often useful to\n", + "use a less-complex model with higher bias, that is a model whose asymptotic\n", + "performance is worse than another model because it is easier to\n", + "train and less sensitive to sampling noise arising from having a\n", + "finite-sized training dataset (smaller variance). \n", + "\n", + "\n", + "\n", + "The above equations tell us that in\n", + "order to minimize the expected test error, we need to select a\n", + "statistical learning method that simultaneously achieves low variance\n", + "and low bias. Note that variance is inherently a nonnegative quantity,\n", + "and squared bias is also nonnegative. Hence, we see that the expected\n", + "test MSE can never lie below $Var(\\epsilon)$, the irreducible error.\n", + "\n", + "\n", + "What do we mean by the variance and bias of a statistical learning\n", + "method? The variance refers to the amount by which our model would change if we\n", + "estimated it using a different training data set. Since the training\n", + "data are used to fit the statistical learning method, different\n", + "training data sets will result in a different estimate. But ideally the\n", + "estimate for our model should not vary too much between training\n", + "sets. However, if a method has high variance then small changes in\n", + "the training data can result in large changes in the model. In general, more\n", + "flexible statistical methods have higher variance.\n", + "\n", + "\n", + "You may also find this recent [article](https://www.pnas.org/content/116/32/15849) of interest." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "============================\n", + "Underfitting vs. Overfitting\n", + "============================\n", + "\n", + "This example demonstrates the problems of underfitting and overfitting and\n", + "how we can use linear regression with polynomial features to approximate\n", + "nonlinear functions. The plot shows the function that we want to approximate,\n", + "which is a part of the cosine function. In addition, the samples from the\n", + "real function and the approximations of different models are displayed. The\n", + "models have polynomial features of different degrees. We can see that a\n", + "linear function (polynomial with degree 1) is not sufficient to fit the\n", + "training samples. This is called **underfitting**. A polynomial of degree 4\n", + "approximates the true function almost perfectly. However, for higher degrees\n", + "the model will **overfit** the training data, i.e. it learns the noise of the\n", + "training data.\n", + "We evaluate quantitatively **overfitting** / **underfitting** by using\n", + "cross-validation. We calculate the mean squared error (MSE) on the validation\n", + "set, the higher, the less likely the model generalizes correctly from the\n", + "training data.\n", + "\"\"\"\n", + "\n", + "print(__doc__)\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "def true_fun(X):\n", + " return np.cos(1.5 * np.pi * X)\n", + "\n", + "np.random.seed(0)\n", + "\n", + "n_samples = 30\n", + "degrees = [1, 4, 15]\n", + "\n", + "X = np.sort(np.random.rand(n_samples))\n", + "y = true_fun(X) + np.random.randn(n_samples) * 0.1\n", + "\n", + "plt.figure(figsize=(14, 5))\n", + "for i in range(len(degrees)):\n", + " ax = plt.subplot(1, len(degrees), i + 1)\n", + " plt.setp(ax, xticks=(), yticks=())\n", + "\n", + " polynomial_features = PolynomialFeatures(degree=degrees[i],\n", + " include_bias=False)\n", + " linear_regression = LinearRegression()\n", + " pipeline = Pipeline([(\"polynomial_features\", polynomial_features),\n", + " (\"linear_regression\", linear_regression)])\n", + " pipeline.fit(X[:, np.newaxis], y)\n", + "\n", + " # Evaluate the models using crossvalidation\n", + " scores = cross_val_score(pipeline, X[:, np.newaxis], y,\n", + " scoring=\"neg_mean_squared_error\", cv=10)\n", + "\n", + " X_test = np.linspace(0, 1, 100)\n", + " plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label=\"Model\")\n", + " plt.plot(X_test, true_fun(X_test), label=\"True function\")\n", + " plt.scatter(X, y, edgecolor='b', s=20, label=\"Samples\")\n", + " plt.xlabel(\"x\")\n", + " plt.ylabel(\"y\")\n", + " plt.xlim((0, 1))\n", + " plt.ylim((-2, 2))\n", + " plt.legend(loc=\"best\")\n", + " plt.title(\"Degree {}\\nMSE = {:.2e}(+/- {:.2e})\".format(\n", + " degrees[i], -scores.mean(), scores.std()))\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.utils import resample\n", + "from sklearn.metrics import mean_squared_error\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organize the data into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "\n", + "Maxpolydegree = 30\n", + "X = np.zeros((len(Density),Maxpolydegree))\n", + "X[:,0] = 1.0\n", + "testerror = np.zeros(Maxpolydegree)\n", + "trainingerror = np.zeros(Maxpolydegree)\n", + "polynomial = np.zeros(Maxpolydegree)\n", + "\n", + "trials = 100\n", + "for polydegree in range(1, Maxpolydegree):\n", + " polynomial[polydegree] = polydegree\n", + " for degree in range(polydegree):\n", + " X[:,degree] = Density**(degree/3.0)\n", + "\n", + "# loop over trials in order to estimate the expectation value of the MSE\n", + " testerror[polydegree] = 0.0\n", + " trainingerror[polydegree] = 0.0\n", + " for samples in range(trials):\n", + " x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)\n", + " model = LinearRegression(fit_intercept=True).fit(x_train, y_train)\n", + " ypred = model.predict(x_train)\n", + " ytilde = model.predict(x_test)\n", + " testerror[polydegree] += mean_squared_error(y_test, ytilde)\n", + " trainingerror[polydegree] += mean_squared_error(y_train, ypred) \n", + "\n", + " testerror[polydegree] /= trials\n", + " trainingerror[polydegree] /= trials\n", + " print(\"Degree of polynomial: %3d\"% polynomial[polydegree])\n", + " print(\"Mean squared error on training data: %.8f\" % trainingerror[polydegree])\n", + " print(\"Mean squared error on test data: %.8f\" % testerror[polydegree])\n", + "\n", + "plt.plot(polynomial, np.log10(trainingerror), label='Training Error')\n", + "plt.plot(polynomial, np.log10(testerror), label='Test Error')\n", + "plt.xlabel('Polynomial degree')\n", + "plt.ylabel('log10[MSE]')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import KFold\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organize the data into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "\n", + "Maxpolydegree = 30\n", + "X = np.zeros((len(Density),Maxpolydegree))\n", + "X[:,0] = 1.0\n", + "estimated_mse_sklearn = np.zeros(Maxpolydegree)\n", + "polynomial = np.zeros(Maxpolydegree)\n", + "k =5\n", + "kfold = KFold(n_splits = k)\n", + "\n", + "for polydegree in range(1, Maxpolydegree):\n", + " polynomial[polydegree] = polydegree\n", + " for degree in range(polydegree):\n", + " X[:,degree] = Density**(degree/3.0)\n", + " OLS = LinearRegression()\n", + "# loop over trials in order to estimate the expectation value of the MSE\n", + " estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold)\n", + "#[:, np.newaxis]\n", + " estimated_mse_sklearn[polydegree] = np.mean(-estimated_mse_folds)\n", + "\n", + "plt.plot(polynomial, np.log10(estimated_mse_sklearn), label='Test Error')\n", + "plt.xlabel('Polynomial degree')\n", + "plt.ylabel('log10[MSE]')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import KFold\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.model_selection import cross_val_score\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "np.random.seed(3155)\n", + "# Generate the data.\n", + "n = 100\n", + "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", + "# Decide degree on polynomial to fit\n", + "poly = PolynomialFeatures(degree = 10)\n", + "\n", + "# Decide which values of lambda to use\n", + "nlambdas = 500\n", + "lambdas = np.logspace(-3, 5, nlambdas)\n", + "# Initialize a KFold instance\n", + "k = 5\n", + "kfold = KFold(n_splits = k)\n", + "estimated_mse_sklearn = np.zeros(nlambdas)\n", + "i = 0\n", + "for lmb in lambdas:\n", + " ridge = Ridge(alpha = lmb)\n", + " estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)\n", + " estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n", + " i += 1\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/doc/pub/week35/ipynb/.ipynb_checkpoints/week35-checkpoint.ipynb b/doc/pub/week35/ipynb/.ipynb_checkpoints/week35-checkpoint.ipynb new file mode 100644 index 000000000..2e85b9ad5 --- /dev/null +++ b/doc/pub/week35/ipynb/.ipynb_checkpoints/week35-checkpoint.ipynb @@ -0,0 +1,2314 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "# Week 35: Linear Regression and Review of Statistical Analysis and Probability Theory\n", + "\n", + " \n", + "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", + "\n", + "Date: **Sep 16, 2020**\n", + "\n", + "Copyright 1999-2020, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n", + "\n", + "\n", + "\n", + "\n", + "## Plans for week 35, August 24-28\n", + "\n", + "* Thursday: Introduction to ordinary Least Squares and derivation of basic equation\n", + "\n", + "* Friday: Linear regression and statistical analysis and probability theory\n", + "\n", + "## Thursday August 27\n", + "\n", + "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureAug27.mp4?vrtx=view-as-webpage).\n", + "\n", + "\n", + "## Why Linear Regression (aka Ordinary Least Squares and family)\n", + "\n", + "Fitting a continuous function with linear parameterization in terms of the parameters $\\boldsymbol{\\beta}$.\n", + "* Method of choice for fitting a continuous function!\n", + "\n", + "* Gives an excellent introduction to central Machine Learning features with **understandable pedagogical** links to other methods like **Neural Networks**, **Support Vector Machines** etc\n", + "\n", + "* Analytical expression for the fitting parameters $\\boldsymbol{\\beta}$\n", + "\n", + "* Analytical expressions for statistical propertiers like mean values, variances, confidence intervals and more\n", + "\n", + "* Analytical relation with probabilistic interpretations \n", + "\n", + "* Easy to introduce basic concepts like bias-variance tradeoff, cross-validation, resampling and regularization techniques and many other ML topics\n", + "\n", + "* Easy to code! And links well with classification problems and logistic regression and neural networks\n", + "\n", + "* Allows for **easy** hands-on understanding of gradient descent methods\n", + "\n", + "* and many more features\n", + "\n", + "For more discussions of Ridge and Lasso regression, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "Similarly, [Mehta et al's article](https://arxiv.org/abs/1803.08823) is also recommended.\n", + "\n", + "\n", + "## Regression analysis, overarching aims\n", + "\n", + "Regression modeling deals with the description of the sampling distribution of a given random variable $y$ and how it varies as function of another variable or a set of such variables $\\boldsymbol{x} =[x_0, x_1,\\dots, x_{n-1}]^T$. \n", + "The first variable is called the **dependent**, the **outcome** or the **response** variable while the set of variables $\\boldsymbol{x}$ is called the independent variable, or the predictor variable or the explanatory variable. \n", + "\n", + "A regression model aims at finding a likelihood function $p(\\boldsymbol{y}\\vert \\boldsymbol{x})$, that is the conditional distribution for $\\boldsymbol{y}$ with a given $\\boldsymbol{x}$. The estimation of $p(\\boldsymbol{y}\\vert \\boldsymbol{x})$ is made using a data set with \n", + "* $n$ cases $i = 0, 1, 2, \\dots, n-1$ \n", + "\n", + "* Response (target, dependent or outcome) variable $y_i$ with $i = 0, 1, 2, \\dots, n-1$ \n", + "\n", + "* $p$ so-called explanatory (independent or predictor) variables $\\boldsymbol{x}_i=[x_{i0}, x_{i1}, \\dots, x_{ip-1}]$ with $i = 0, 1, 2, \\dots, n-1$ and explanatory variables running from $0$ to $p-1$. See below for more explicit examples. \n", + "\n", + " The goal of the regression analysis is to extract/exploit relationship between $\\boldsymbol{y}$ and $\\boldsymbol{x}$ in or to infer causal dependencies, approximations to the likelihood functions, functional relationships and to make predictions, making fits and many other things.\n", + "\n", + "\n", + "\n", + "## Regression analysis, overarching aims II\n", + "\n", + "\n", + "Consider an experiment in which $p$ characteristics of $n$ samples are\n", + "measured. The data from this experiment, for various explanatory variables $p$ are normally represented by a matrix \n", + "$\\mathbf{X}$.\n", + "\n", + "The matrix $\\mathbf{X}$ is called the *design\n", + "matrix*. Additional information of the samples is available in the\n", + "form of $\\boldsymbol{y}$ (also as above). The variable $\\boldsymbol{y}$ is\n", + "generally referred to as the *response variable*. The aim of\n", + "regression analysis is to explain $\\boldsymbol{y}$ in terms of\n", + "$\\boldsymbol{X}$ through a functional relationship like $y_i =\n", + "f(\\mathbf{X}_{i,\\ast})$. When no prior knowledge on the form of\n", + "$f(\\cdot)$ is available, it is common to assume a linear relationship\n", + "between $\\boldsymbol{X}$ and $\\boldsymbol{y}$. This assumption gives rise to\n", + "the *linear regression model* where $\\boldsymbol{\\beta} = [\\beta_0, \\ldots,\n", + "\\beta_{p-1}]^{T}$ are the *regression parameters*. \n", + "\n", + "Linear regression gives us a set of analytical equations for the parameters $\\beta_j$.\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "## Examples\n", + "In order to understand the relation among the predictors $p$, the set of data $n$ and the target (outcome, output etc) $\\boldsymbol{y}$,\n", + "consider the model we discussed for describing nuclear binding energies. \n", + "\n", + "There we assumed that we could parametrize the data using a polynomial approximation based on the liquid drop model.\n", + "Assuming" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "BE(A) = a_0+a_1A+a_2A^{2/3}+a_3A^{-1/3}+a_4A^{-1},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have five predictors, that is the intercept, the $A$ dependent term, the $A^{2/3}$ term and the $A^{-1/3}$ and $A^{-1}$ terms.\n", + "This gives $p=0,1,2,3,4$. Furthermore we have $n$ entries for each predictor. It means that our design matrix is a \n", + "$p\\times n$ matrix $\\boldsymbol{X}$.\n", + "\n", + "Here the predictors are based on a model we have made. A popular data set which is widely encountered in ML applications is the\n", + "so-called [credit card default data from Taiwan](https://www.sciencedirect.com/science/article/pii/S0957417407006719?via%3Dihub). The data set contains data on $n=30000$ credit card holders with predictors like gender, marital status, age, profession, education, etc. In total there are $24$ such predictors or attributes leading to a design matrix of dimensionality $24 \\times 30000$. This is however a classification problem and we will come back to it when we discuss Logistic Regression.\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "## General linear models\n", + "Before we proceed let us study a case from linear algebra where we aim at fitting a set of data $\\boldsymbol{y}=[y_0,y_1,\\dots,y_{n-1}]$. We could think of these data as a result of an experiment or a complicated numerical experiment. These data are functions of a series of variables $\\boldsymbol{x}=[x_0,x_1,\\dots,x_{n-1}]$, that is $y_i = y(x_i)$ with $i=0,1,2,\\dots,n-1$. The variables $x_i$ could represent physical quantities like time, temperature, position etc. We assume that $y(x)$ is a smooth function. \n", + "\n", + "Since obtaining these data points may not be trivial, we want to use these data to fit a function which can allow us to make predictions for values of $y$ which are not in the present set. The perhaps simplest approach is to assume we can parametrize our function in terms of a polynomial of degree $n-1$ with $n$ points, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y=y(x) \\rightarrow y(x_i)=\\tilde{y}_i+\\epsilon_i=\\sum_{j=0}^{n-1} \\beta_j x_i^j+\\epsilon_i,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\epsilon_i$ is the error in our approximation.\n", + "\n", + "\n", + "\n", + "\n", + "## Rewriting the fitting procedure as a linear algebra problem\n", + "For every set of values $y_i,x_i$ we have thus the corresponding set of equations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "y_0&=\\beta_0+\\beta_1x_0^1+\\beta_2x_0^2+\\dots+\\beta_{n-1}x_0^{n-1}+\\epsilon_0\\\\\n", + "y_1&=\\beta_0+\\beta_1x_1^1+\\beta_2x_1^2+\\dots+\\beta_{n-1}x_1^{n-1}+\\epsilon_1\\\\\n", + "y_2&=\\beta_0+\\beta_1x_2^1+\\beta_2x_2^2+\\dots+\\beta_{n-1}x_2^{n-1}+\\epsilon_2\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{n-1}&=\\beta_0+\\beta_1x_{n-1}^1+\\beta_2x_{n-1}^2+\\dots+\\beta_{n-1}x_{n-1}^{n-1}+\\epsilon_{n-1}.\\\\\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Rewriting the fitting procedure as a linear algebra problem, more details\n", + "Defining the vectors" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = [y_0,y_1, y_2,\\dots, y_{n-1}]^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta} = [\\beta_0,\\beta_1, \\beta_2,\\dots, \\beta_{n-1}]^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\epsilon} = [\\epsilon_0,\\epsilon_1, \\epsilon_2,\\dots, \\epsilon_{n-1}]^T,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and the design matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\n", + "\\begin{bmatrix} \n", + "1& x_{0}^1 &x_{0}^2& \\dots & \\dots &x_{0}^{n-1}\\\\\n", + "1& x_{1}^1 &x_{1}^2& \\dots & \\dots &x_{1}^{n-1}\\\\\n", + "1& x_{2}^1 &x_{2}^2& \\dots & \\dots &x_{2}^{n-1}\\\\ \n", + "\\dots& \\dots &\\dots& \\dots & \\dots &\\dots\\\\\n", + "1& x_{n-1}^1 &x_{n-1}^2& \\dots & \\dots &x_{n-1}^{n-1}\\\\\n", + "\\end{bmatrix}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we can rewrite our equations as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = \\boldsymbol{X}\\boldsymbol{\\beta}+\\boldsymbol{\\epsilon}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The above design matrix is called a [Vandermonde matrix](https://en.wikipedia.org/wiki/Vandermonde_matrix).\n", + "\n", + "\n", + "\n", + "\n", + "## Generalizing the fitting procedure as a linear algebra problem\n", + "\n", + "We are obviously not limited to the above polynomial expansions. We\n", + "could replace the various powers of $x$ with elements of Fourier\n", + "series or instead of $x_i^j$ we could have $\\cos{(j x_i)}$ or $\\sin{(j\n", + "x_i)}$, or time series or other orthogonal functions. For every set\n", + "of values $y_i,x_i$ we can then generalize the equations to" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "y_0&=\\beta_0x_{00}+\\beta_1x_{01}+\\beta_2x_{02}+\\dots+\\beta_{n-1}x_{0n-1}+\\epsilon_0\\\\\n", + "y_1&=\\beta_0x_{10}+\\beta_1x_{11}+\\beta_2x_{12}+\\dots+\\beta_{n-1}x_{1n-1}+\\epsilon_1\\\\\n", + "y_2&=\\beta_0x_{20}+\\beta_1x_{21}+\\beta_2x_{22}+\\dots+\\beta_{n-1}x_{2n-1}+\\epsilon_2\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{i}&=\\beta_0x_{i0}+\\beta_1x_{i1}+\\beta_2x_{i2}+\\dots+\\beta_{n-1}x_{in-1}+\\epsilon_i\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{n-1}&=\\beta_0x_{n-1,0}+\\beta_1x_{n-1,2}+\\beta_2x_{n-1,2}+\\dots+\\beta_{n-1}x_{n-1,n-1}+\\epsilon_{n-1}.\\\\\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note that we have $p=n$ here. The matrix is symmetric. This is generally not the case!**\n", + "\n", + "\n", + "\n", + "\n", + "## Generalizing the fitting procedure as a linear algebra problem\n", + "We redefine in turn the matrix $\\boldsymbol{X}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\n", + "\\begin{bmatrix} \n", + "x_{00}& x_{01} &x_{02}& \\dots & \\dots &x_{0,n-1}\\\\\n", + "x_{10}& x_{11} &x_{12}& \\dots & \\dots &x_{1,n-1}\\\\\n", + "x_{20}& x_{21} &x_{22}& \\dots & \\dots &x_{2,n-1}\\\\ \n", + "\\dots& \\dots &\\dots& \\dots & \\dots &\\dots\\\\\n", + "x_{n-1,0}& x_{n-1,1} &x_{n-1,2}& \\dots & \\dots &x_{n-1,n-1}\\\\\n", + "\\end{bmatrix}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and without loss of generality we rewrite again our equations as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = \\boldsymbol{X}\\boldsymbol{\\beta}+\\boldsymbol{\\epsilon}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The left-hand side of this equation is kwown. Our error vector $\\boldsymbol{\\epsilon}$ and the parameter vector $\\boldsymbol{\\beta}$ are our unknow quantities. How can we obtain the optimal set of $\\beta_i$ values?\n", + "\n", + "\n", + "\n", + "\n", + "## Optimizing our parameters\n", + "We have defined the matrix $\\boldsymbol{X}$ via the equations" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "y_0&=\\beta_0x_{00}+\\beta_1x_{01}+\\beta_2x_{02}+\\dots+\\beta_{n-1}x_{0n-1}+\\epsilon_0\\\\\n", + "y_1&=\\beta_0x_{10}+\\beta_1x_{11}+\\beta_2x_{12}+\\dots+\\beta_{n-1}x_{1n-1}+\\epsilon_1\\\\\n", + "y_2&=\\beta_0x_{20}+\\beta_1x_{21}+\\beta_2x_{22}+\\dots+\\beta_{n-1}x_{2n-1}+\\epsilon_1\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{i}&=\\beta_0x_{i0}+\\beta_1x_{i1}+\\beta_2x_{i2}+\\dots+\\beta_{n-1}x_{in-1}+\\epsilon_1\\\\\n", + "\\dots & \\dots \\\\\n", + "y_{n-1}&=\\beta_0x_{n-1,0}+\\beta_1x_{n-1,2}+\\beta_2x_{n-1,2}+\\dots+\\beta_{n-1}x_{n-1,n-1}+\\epsilon_{n-1}.\\\\\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we noted above, we stayed with a system with the design matrix \n", + " $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times n}$, that is we have $p=n$. For reasons to come later (algorithmic arguments) we will hereafter define \n", + "our matrix as $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$, with the predictors refering to the column numbers and the entries $n$ being the row elements.\n", + "\n", + "\n", + "\n", + "\n", + "## Our model for the nuclear binding energies\n", + "\n", + "In our [introductory notes](https://compphysics.github.io/MachineLearning/doc/pub/How2ReadData/html/How2ReadData.html) we looked at the so-called [liquid drop model](https://en.wikipedia.org/wiki/Semi-empirical_mass_formula). Let us remind ourselves about what we did by looking at the code.\n", + "\n", + "We restate the parts of the code we are most interested in." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
1AA^(2/3)A^(-1/3)1/A
A
11.01.01.0000001.0000001.000000
21.02.01.5874010.7937010.500000
31.03.02.0800840.6933610.333333
41.04.02.5198420.6299610.250000
51.05.02.9240180.5848040.200000
..................
2641.0264.041.1531060.1558830.003788
2651.0265.041.2569620.1556870.003774
2661.0266.041.3606880.1554910.003759
2691.0269.041.6710890.1549110.003717
2701.0270.041.7743000.1547200.003704
\n", + "

267 rows × 5 columns

\n", + "
" + ], + "text/plain": [ + " 1 A A^(2/3) A^(-1/3) 1/A\n", + "A \n", + "1 1.0 1.0 1.000000 1.000000 1.000000\n", + "2 1.0 2.0 1.587401 0.793701 0.500000\n", + "3 1.0 3.0 2.080084 0.693361 0.333333\n", + "4 1.0 4.0 2.519842 0.629961 0.250000\n", + "5 1.0 5.0 2.924018 0.584804 0.200000\n", + ".. ... ... ... ... ...\n", + "264 1.0 264.0 41.153106 0.155883 0.003788\n", + "265 1.0 265.0 41.256962 0.155687 0.003774\n", + "266 1.0 266.0 41.360688 0.155491 0.003759\n", + "269 1.0 269.0 41.671089 0.154911 0.003717\n", + "270 1.0 270.0 41.774300 0.154720 0.003704\n", + "\n", + "[267 rows x 5 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "\n", + "# Common imports\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from IPython.display import display\n", + "import os\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"MassEval2016.dat\"),'r')\n", + "\n", + "\n", + "# Read the experimental data with Pandas\n", + "Masses = pd.read_fwf(infile, usecols=(2,3,4,6,11),\n", + " names=('N', 'Z', 'A', 'Element', 'Ebinding'),\n", + " widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1),\n", + " header=39,\n", + " index_col=False)\n", + "\n", + "# Extrapolated values are indicated by '#' in place of the decimal place, so\n", + "# the Ebinding column won't be numeric. Coerce to float and drop these entries.\n", + "Masses['Ebinding'] = pd.to_numeric(Masses['Ebinding'], errors='coerce')\n", + "Masses = Masses.dropna()\n", + "# Convert from keV to MeV.\n", + "Masses['Ebinding'] /= 1000\n", + "\n", + "# Group the DataFrame by nucleon number, A.\n", + "Masses = Masses.groupby('A')\n", + "# Find the rows of the grouped DataFrame with the maximum binding energy.\n", + "Masses = Masses.apply(lambda t: t[t.Ebinding==t.Ebinding.max()])\n", + "A = Masses['A']\n", + "Z = Masses['Z']\n", + "N = Masses['N']\n", + "Element = Masses['Element']\n", + "Energies = Masses['Ebinding']\n", + "\n", + "# Now we set up the design matrix X\n", + "X = np.zeros((len(A),5))\n", + "X[:,0] = 1\n", + "X[:,1] = A\n", + "X[:,2] = A**(2.0/3.0)\n", + "X[:,3] = A**(-1.0/3.0)\n", + "X[:,4] = A**(-1.0)\n", + "# Then nice printout using pandas\n", + "DesignMatrix = pd.DataFrame(X)\n", + "DesignMatrix.index = A\n", + "DesignMatrix.columns = ['1', 'A', 'A^(2/3)', 'A^(-1/3)', '1/A']\n", + "display(DesignMatrix)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With $\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p\\times 1}$, it means that we will hereafter write our equations for the approximation as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}}= \\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "throughout these lectures. \n", + "\n", + "\n", + "## Optimizing our parameters, more details\n", + "With the above we use the design matrix to define the approximation $\\boldsymbol{\\tilde{y}}$ via the unknown quantity $\\boldsymbol{\\beta}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}}= \\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and in order to find the optimal parameters $\\beta_i$ instead of solving the above linear algebra problem, we define a function which gives a measure of the spread between the values $y_i$ (which represent hopefully the exact values) and the parameterized values $\\tilde{y}_i$, namely" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or using the matrix $\\boldsymbol{X}$ and in a more compact matrix-vector notation as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This function is one possible way to define the so-called cost function.\n", + "\n", + "\n", + "\n", + "It is also common to define\n", + "the function $C$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{2n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "since when taking the first derivative with respect to the unknown parameters $\\beta$, the factor of $2$ cancels out.\n", + "\n", + "\n", + "\n", + "\n", + "## Interpretations and optimizing our parameters\n", + "\n", + "The function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "can be linked to the variance of the quantity $y_i$ if we interpret the latter as the mean value. \n", + "When linking (see the discussion below) with the maximum likelihood approach below, we will indeed interpret $y_i$ as a mean value" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y_{i}=\\langle y_i \\rangle = \\beta_0x_{i,0}+\\beta_1x_{i,1}+\\beta_2x_{i,2}+\\dots+\\beta_{n-1}x_{i,n-1}+\\epsilon_i,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\langle y_i \\rangle$ is the mean value. Keep in mind also that\n", + "till now we have treated $y_i$ as the exact value. Normally, the\n", + "response (dependent or outcome) variable $y_i$ the outcome of a\n", + "numerical experiment or another type of experiment and is thus only an\n", + "approximation to the true value. It is then always accompanied by an\n", + "error estimate, often limited to a statistical error estimate given by\n", + "the standard deviation discussed earlier. In the discussion here we\n", + "will treat $y_i$ as our exact value for the response variable.\n", + "\n", + "In order to find the parameters $\\beta_i$ we will then minimize the spread of $C(\\boldsymbol{\\beta})$, that is we are going to solve the problem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In practical terms it means we will require" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_j} = \\frac{\\partial }{\\partial \\beta_j}\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}\\right)^2\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which results in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_j} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}x_{ij}\\left(y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}\\right)\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or in a matrix-vector form as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpretations and optimizing our parameters\n", + "We can rewrite" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{y} = \\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and if the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$ is invertible we have the solution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta} =\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We note also that since our design matrix is defined as $\\boldsymbol{X}\\in\n", + "{\\mathbb{R}}^{n\\times p}$, the product $\\boldsymbol{X}^T\\boldsymbol{X} \\in\n", + "{\\mathbb{R}}^{p\\times p}$. In the above case we have that $p \\ll n$,\n", + "in our case $p=5$ meaning that we end up with inverting a small\n", + "$5\\times 5$ matrix. This is a rather common situation, in many cases we end up with low-dimensional\n", + "matrices to invert. The methods discussed here and for many other\n", + "supervised learning algorithms like classification with logistic\n", + "regression or support vector machines, exhibit dimensionalities which\n", + "allow for the usage of direct linear algebra methods such as **LU** decomposition or **Singular Value Decomposition** (SVD) for finding the inverse of the matrix\n", + "$\\boldsymbol{X}^T\\boldsymbol{X}$.\n", + "\n", + "\n", + "\n", + "**Small question**: Do you think the example we have at hand here (the nuclear binding energies) can lead to problems in inverting the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$? What kind of problems can we expect?\n", + "\n", + "\n", + "\n", + "## Some useful matrix and vector expressions\n", + "\n", + "The following matrix and vector relation will be useful here and for the rest of the course. Vectors are always written as boldfaced lower case letters and \n", + "matrices as upper case boldfaced letters." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "2\n", + "6\n", + " \n", + "<\n", + "<\n", + "<\n", + "!\n", + "!\n", + "M\n", + "A\n", + "T\n", + "H\n", + "_\n", + "B\n", + "L\n", + "O\n", + "C\n", + "K" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "2\n", + "7\n", + " \n", + "<\n", + "<\n", + "<\n", + "!\n", + "!\n", + "M\n", + "A\n", + "T\n", + "H\n", + "_\n", + "B\n", + "L\n", + "O\n", + "C\n", + "K" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "2\n", + "8\n", + " \n", + "<\n", + "<\n", + "<\n", + "!\n", + "!\n", + "M\n", + "A\n", + "T\n", + "H\n", + "_\n", + "B\n", + "L\n", + "O\n", + "C\n", + "K" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\log{\\vert\\boldsymbol{A}\\vert}}{\\partial \\boldsymbol{A}} = (\\boldsymbol{A}^{-1})^T.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interpretations and optimizing our parameters\n", + "The residuals $\\boldsymbol{\\epsilon}$ are in turn given by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\epsilon} = \\boldsymbol{y}-\\boldsymbol{\\tilde{y}} = \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and with" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)= 0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{\\epsilon}=\\boldsymbol{X}^T\\left( \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)= 0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "meaning that the solution for $\\boldsymbol{\\beta}$ is the one which minimizes the residuals. Later we will link this with the maximum likelihood approach.\n", + "\n", + "\n", + "\n", + "\n", + "Let us now return to our nuclear binding energies and simply code the above equations. \n", + "\n", + "## Own code for Ordinary Least Squares\n", + "\n", + "It is rather straightforward to implement the matrix inversion and obtain the parameters $\\boldsymbol{\\beta}$. After having defined the matrix $\\boldsymbol{X}$ we simply need to \n", + "write" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# matrix inversion to find beta\n", + "beta = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Energies)\n", + "# and then make the prediction\n", + "ytilde = X @ beta" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, you can use the least squares functionality in **Numpy** as" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "fit = np.linalg.lstsq(X, Energies, rcond =None)[0]\n", + "ytildenp = np.dot(fit,X.T)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And finally we plot our fit with and compare with data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEICAYAAABcVE8dAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAArwElEQVR4nO3de5Rc5Xnn++9T975f1bq1pJaQhBEgZBC+gG8xibHBxjGeeGHHsckahvFyJmOTkzXHyYxjJ+fMLJKVTILjJB7OeMY5tiNmfM/4GIfYBo/xBRBYCJBAgGhJ3a1Lq9WtVnfXde/n/FHVTeve3errrt9nrV5dXbuq9vuq4FdvPfvd7zZ3R0REoi+20A0QEZH5ocAXEakSCnwRkSqhwBcRqRIKfBGRKpFY6AacT3t7u3d1dS10M0RElpQnn3zyuLsvO9e2RRv4XV1d7Ny5c6GbISKypJjZgfNtU0lHRKRKKPBFRKqEAl9EpEoo8EVEqoQCX0SkSijwRUSqhAJfRKRKLNp5+HJhw7kih06M0VaX5sDAKA2ZJM/2naQulSBbDCiWAja11jEyUqI2neDx/QNc1dlEY22SxkyS9cvqIAaxVAwzW+juiMg8UOAvAWEp5OiBUfJHCvzg0V5yx4vkhoowEpLMOsmck8xBrAhjJccCiJfg+Umv0Qz0cGzi718YJGJGTSpBnpC2ljTpmgTpujhFc9J1CWobknjGaGxLE6+PE6+LQ8ZINibKf9fHiTfESTSV/w5CJxEvf2kMQ2fvkWFKgXPFykZSCX2ZFFloCvxFJCyFFHoL5Lpz5LpzZF/Jkj2QY+DQGAeOjwGQqfwApJMxSoFTm4pTDJ3aZBxPOJY0DMh5SDwVI1cKaUwnOJUtEosZuWJAKXDC0ClliwAczo6dt12pRIwgdNKJGGOFgFQixviXgrgZxcCJJ42T8ZBkS4LRlJPNOLkMFGuNr9UZqZYkazc0UNeRZsumZi7rqCeTjM/dP6aInEWBv4DCUsjYnjFGnx1l9LlRxl4Yw/PO8ZE8w9kiyXiMwbECIZBvMIpNMWpXpblySwvp5gSrO+tJNCaIN8ZJNCSI1cSwpBFLl3+fr1Tj7hQD5+CJUfoGs6RCY+eLJ4iVnIHBHHWxBEcHsoRjITUlIxwNSOYhkXcS+XjlNyRyTirnJLNOIgdpgNEiDUADkEzEiMcgVygBRWCMPLAzcZCf1RvJtiSZZSlirQlWr6+ntiPFZZuayHSkSLYksbhKTSKzyRbrJQ63b9/uUVpLZyRfIhk34jln+PFhnn3wKMd+McSKdJrQnZgZA6MFRhuco40ho20xRtqNkXaj2BzjHVtXcvu1q+et3l4ohQShYwaHT+aoS8fpG8qxrrWWwydzpBLlduSKIU01SY4P5uhKZ+g9NEJTMU46BwwHBCcDCkNFBvtyDPZmKQwUGR7KUyiFnOs/vVjMAKcmlYDmOOnlKZZ11ZHuSNGxvg7a4rSsqSW5LEksqTKRyJnM7El3337ObQr8udc3OMbf/t1zrHg2YNUBZ2y0NLFtpN0YXBPj5KoYQ6tjFGvLQXrjxnZO5Uq8+5qVbGivi9SB1WAsIHcsz4v7TpLrLzDYm2OwL0u2v0Chv0B6xElmwc7zn2Y6GcOAdHuKVEeK9PIUtSvSxNoTdG1uJL4sSdOaGmIpfSBI9blQ4KukMwd+eXCQf3ruKL+2cRk8OsoLX+nlNQPlkB81GFoTY2hTgqY3NPL02AjLmzKYO1e31PKmje0k4zG2rGpc4F7MnXhtnLquWrZ11Z52v7tzMlvEMJ7vHcZPFHlx70lGD+coHC1S6C9QOwypkyHpU07ucA4O5057jfED1alEjHy9EV+eJGyLE+9I0tpVhy1LsGVrK+nmBO316Uh9kIpcjEb4s2g4V+SFI6f40g/30/FUkTVPlUhlKxuXJWj8lSaGtqZ4703rqEvFiceMo8N52utTE7Nb5PzGjz0cGBiFEF54YZDgWJETPVnC/hIMlMgeyZMZdtIn/bzfEABKaQja4nhbgprVadIrU6RWpFl3eSOJ9iTrltWRiJkOLMuSo5LOPAhD54++8wz+szE2/KxEetRJxI3SmiQt72/nzf9iDY01qYVuZuS5O0Ho9A5kiQ8FHH55BO8vcvJAluzhPKWjRXJ9eWIFpxSc+7/9MA65BiPbbCQ6kpTa4tR0ZmhdX0tmRYrV7bUkYjFWNmdIxIymmqS+KciioZLOPHhqZz8r/8swTUec9vo0Xb/SwuqPrqD+mnqFwTwyMxJxY11HHXRA5+azS2PuTulkicMvj5DtyXP45VOUjhYZ68tTPFLATgYkhgPqTjrena88a4wcJ8gaHG4yxpqNsZbyh0JsRZL6NRlWrW/guvWtrGmtoTal/7Vk8dF/lZeoWArYce8e/NtDNJVg1bp6XntPF81vbVbQL1JmRrI5ydrrWuA6uJwVp20P8yGFIwWyfXmOvDRC6UiBE6+MUTycp3i8RCEfEh528ocCQocwLAFZSvFBHmk6RLYlRv2aDOlVKdZvaWb5xjq6NjaSTKg8JAtLgX8JjhwbY++93cR+NATA4DVJ3nffldQ1q3SzlMXSMTLrMmTWZWh5Y9Np28JC+cMg35un0Ff+PfDKKNmePIOHs4yNBGRPBPDyKDBKN4N0A4+njNSqNDWr07RfVkfj2hrWvqaRdGeaRFNCgwOZFwr8GXpy5zF2/rt91J1wSmlo/NgK3vvBLuoyyYVumsyhWCpGZm2GzNrMxH2dld9BLqDQV2Coe4wjL44w1D3G8f2jFI8UCE8G5Ltz5LtzDP30JABPx4xYzKhtSBBfkaJ2TYa6NRmaumpYubmB2rUZ4jX6ViCzR4E/A7kDObr/4BXqTjijbUbfHfX80Uc3Eo9plFbN4pk4NRtqqNlQw8q3t03c7+709o1y+KURDu49ycjBHKM9OThWpHYwpDhYgMECJ/aOnPZ6sZiRbksSX5ksfztYk2bZZfUs31RPy9paEjrxTKZJgT9NY/vGePE/7CfbX2BwTYx1/2Ed77m8TWEv52VmdK6up3N1Pde/tXy8YPycg+FskZdfGWasN8/g/jGCwwVyvXk4WqRm0An7C9BfgN2jAPSOv2jCiHckia1I0rAuQ3p1mhWbG1j9mkYa21PEYvowkLPNa+Cb2T3AXYADzwC/7e65Cz9r8cgdyvHSp1/h4KERjm+IEftXy7jt9Z0Xf6LIGcyM5toUzbUp1rbVwRmT6NydEyMFuvedZLg7y+ihHLmePCOHctjRIvGhkKCvQNBX4MRT5Q+Dw8AvgbDGoCNBanWa+rUZatdkWHNFE6svb6C2TiXHajZvgW9mq4F/C2xx96yZ/U/gDuBL89WGS3HswCiH/vAVeg+OsH9FwN73pLjntSsu/kSRGTAz2hrStF3XAdedvf3kUJ7DL57i+P5RhvaPUThcYPRADj9aJJZ1OFCkdKDI0M9GGAL66AEgbIkTW5EktTpF/doaWi+rY+1rGlm2rlaziKrAfJd0EkCNmRWBWqBvnvc/ZaUgJB4rrzgZFkIe/PgzxHuLnFwZY9970vzx7Vexoilz8RcSmQNNzWmark/D9e2n3R+GIcP9BY7tO8Wxl0YYPpAl25Nn7FAOjpeIDQYwGFDYm+MEw5wAXgLCBATtifI5BesyNK2rpXVDLas2N9DSkSGtD4NImLfAd/deM/tz4CCQBR5y94cmP8bM7gbuBli7du18Ne0se/qG+YuHXuDOG7t486ZlvPS5Q8R7i2Sbjd23JfmVqzoU9rIoxWIxmpdnaF6eYfObl522LSiEHOseof+lUU68MsbIgSwjB3MU+wrEhgNiR0pwpER2V5YsgxwB9gDFWvCOJMGyOA3rakitTtO6oY7WdbW0NKZoq0+RjMdIanmQRW8+SzotwHuB9cAQ8DUz+7C7f2X8Me5+P3A/lJdWmK+2nemXhwYB+MX+Aa7qSXDs+wOEcXj21iTtK2p451Uq5cjSE0/FWLm5kZXnOPs4P1zk+P5Rjr88ypEXR8gdypHrzRMeKUI2hO4idBcpPZGjBIwBhwxylbOOs61GanUaOpLUrU2TbE3SUp+iIZ2kIZNgWUOamlScppokNcm4zjtYIPNZ0vlV4BV37wcws28CNwBfueCzFsDBgfLVn/pePkXPj0qMFUq8+LYEb3n7at5/nQ7SSvSkG5Os3tbM6m3NXDPpfndn7GiBYy+dYuRgjuMvjVI8nCd7KI8PlEiPhNQNO2F3CE+9etW0IAlHmo39rUa2OcZYy6vLUVhtjPrKB8H69jquXdvCpuW6Atp8mM/APwi8wcxqKZd0bgIW3epo7s6hwTFwp+uhImPZBMc2xem7Cn69vW6hmycyr8yMuhVp1q9In7Vt4qzjnjyjB3Oc2D9KrjdP9lCOYCSgkA8Jep3iQacYlC+oUwqdXMYZa8kz1mK80jLMnuYjZFuNtvV1bF7VyObl9WzsaKCpRjOKZtt81vAfM7OvA08BJcozyO6fr/1P1bFTefLFkGUvhbR1h/Q053n8dQkwY70CX2TC5LOOm25oYtWkbaVTpfLyE73l5ScmlqLoy1PKlcO/2B8yfLDISK7EWDHAKTDSNMRPWoyHmsslouWb6una0sSmzU10NGZUCrpE8zpLx90/A3xmPvc5XQdPjBHPO5t+XL5gya7XOqNpZ01rLS21GnGITEWiIUHiNQnqXnP6IMndKfYXT/8QqMwiOtmbYzRfYvRwidHuEv7LMWCMbo7xchKCZXFq19RQ25lm1eUNrNrcQMemehJ1On90qvQvBTz8/DFGCyXevXUVBwbGWPtkidWWIn9Fgqs+2MYHVjVz+YoGjS5ELpGZkeooX5qy4bUNp20LCyGFw+VvBNmeHEf3jdC/f5SRA1myQ0WCvoBS3wjDj40wzADPA2YQb0qQ7kzTsDZDY1d5Kmn7ZXWkVqaIJTRzaLKqD3x35yu/OADA67pa2fviIGt2BTQ0ZXjzH11O3RUq44jMh1jq1VVKm2hiBcsnthWHi/TtO0Xv3lOc6B7lxP4xSocLxI8H+FCJ0lCJ0WdHOcIA+yh/ECSTMaw9QU1nhoZ1NTSuq6FtQy3LNtaTWZaqygFc1Qd+vhRO3H7slRPYj06RLMLy1zcr7EUWiWRjknXbW1m3vfW0+0fzRfpeHuHQC8OVcwtyZHtyJPoDMqdCrK9Avq/A0OPDrz7JIFkTh47yN4PyB0EdKzbV07ahjkR9dGMxuj2bopF8aeL2d3/Wwxt3BTRkkqz8zeUXeJaILAZ16SSbtrSwaUvLaffnigEDgzmO7h/l8AunGD6YJd+bJzhcIHasBGMBdAcUu/OMPDpMH+XFvczAmuLY8iSZzjSNa2to2VDH8o11LL+sgURqaZeIqj7wx/LBxO01vwxIFKDxxnqN7kWWsEwyzuqOOlZ31MEbOk7blisGHOkdY/DAKP0vj3LylTFGD+UIDhdJDATEhwIYChh9IccoJzlM+YxjYuBtCWxFkto1GZrW1dB+WR2rNjfQ1lmzJFYorfrAHx/hx0rO6t0lUokYW+9euGUdRGRuZZJxuroa6OpqgLeevm0sX6L/4CgD+0cZeHmM4QNZcj05SoeLxAYDrL8E/SXGnskyxiCHKX8z8HR5hdL0qjRN62po7qph+cbyTKLkIrqiWdUH/mihHPgd+0I64knWv66Vlq0NF3mWiERRbTrBuk1NrNvUdNa2sdEiA6+M0f/yKAP7RzhVOV4QHC4SGw3hUJHCoSL9j43QD7xI+SI2maYE669opnl9LbWdadKr06RWpUivShOvnd+zixX4lRH+6w+mWNuaZOV7ly2aT2MRWTxq65LUXtXEmqvO/jAYOp7j+Muj9O07xYlXRhnryZPrzRPrLxEOFtn7i+P4z510IsbKphoaMuVRf6I5MRH+qdWV35W/Y3NwvECBnw+oHQjJ9DmxVWma3nz2mykiciHN7Rma2zNsfH3bafcPjRb4fx/aT/eek9QOOTWDIYeGsrSMxFiRjxM7CXbQqE8nqEtNWlTO4MqvXznroa/Az5dYuScgbkma39w8J5+qIlKdmutS/Ov3bOLR1xxnXVsdLx0b4eHnj/HsSB7cSZ+C2qGAmsESzaMxuoIUtYOOu3O1RvizbzRXZPnzIfG00XxT80I3R0QiJp2Ic9MV5WneGzvqufnK5TzXN8yByqq8p3JFnu45yZ7hHHsol5iTMeOWOWhL1Qd+8GKezKiTXJOk9vLahW6OiEScmXHV6iauWv1q+fiO18GRkzl29wxRCp3aVBx3n/XjiVUf+LFdWQBqXt+og7UismBWNGVY0TS3F1eKfMH6B3uO8vDzxyb+zhUDHts/QK4Y4O7UPJsHoOnGs68CJCISJZEe4Qehs+PxgwC87fLydMsHHj/IT148znVdLdzZuQo7GZJrMJq31C9wa0VE5lakR/hB+OplcUuV2zsPlK9X+2T3ICNPjhCEIQNdMerSkf7sExGJduCH/mrgF4PyqpiTr5s59MQw7jC0Pk5a62aLSMRFOuUm5T2F0njgl7ucyDmn9o4SxqG4qTrXxhaR6hLpwD99hF++XVMZ4bccCikWQ4ZXxGhuOfsCzSIiUVNFgX96SaepL6RQChnsjNFWr8AXkeiLduC/ejGricBPVNasbu4JyZUChjpjtNWlFqJ5IiLzKtqBf44RfiEISOSd+uPOWBAyvMJoq1fgi0j0VU3gF0pe+R3S1BtiDkdbQ8KkqaQjIlUh4oH/6u3xEX4xcBqPljcMVK58ppKOiFSDSAe+Tx7hT5R0QhqOlm8PLy93v10jfBGpApEO/HON8AvFgMYj5dunVsSoScWpSc3vZcZERBZCpAM/OK2GXw55OxGQzEGxBnINGt2LSPWIdOCHk9fSqZx4lektX2BgeHkMzHjPNasWpG0iIvMt0iuGnba0QhCWl0M+EgCw7c0dfPD2NXQ0ZhaodSIi8yvSgR+cMQ+/FDq1xx0z2PaGDpoU9iJSRaJd0jljLZ1iEFI/UL5sWHqdavciUl0iHfh+xkHb3HCRzLBjCSO9UoEvItUl0oEfTFpLpxSGjB7IAZBvj2FxLYcsItUl0jX88IwRfravHPiF5Zp3LyLVZ15H+GbWbGZfN7PnzWyvmb1xLvcXnnGmbe5g+YLlwYpIf86JiJzTfCfffcD33f1fmFkKqJ3LnU2ellksOYXeSuB3KPBFpPrMW/KZWSPwFuBOAHcvAIW53OeZyyMXj5R358sU+CJSfeazpLMB6Af+u5n90sz+q5nVTX6Amd1tZjvNbGd/f/8l7/C0tXQKAcGxYnk/7Qp8Eak+8xn4CeBa4O/c/bXAKPCpyQ9w9/vdfbu7b1+2bNkl7zCYlPg+GBAGTq7BSNbooK2IVJ/5DPweoMfdH6v8/XXKHwBzZvI8/PjxEqE72SYjlYj0bFQRkXOat+Rz9yPAITO7vHLXTcCeudzn5JJO/HhAGEKuyUjGFfgiUn3mu5j9u8BXKzN09gO/PZc7m3zQNjEQ4B5jrNlIKfBFpArNa+C7+y5g+3zt77TAPxESEiPbrJKOiFSnSCdfOGlphdRQSBg6uUaVdESkOkU6+SaP8JOnQkIvz9LRCF9EqlGkk2888GNFJ5mDwMqXNkxq4TQRqUKRDvzxAX56pHxjOBOCGa11qQVslYjIwoh04I+feJUeKf99IlUu6q9tndMlfEREFqVIB/54SSdzqvw7X2801iRprtUIX0SqT8QDv/x7vKSTa9DoXkSqV8QDvxz0XYnyxcoL9cYaBb6IVKmqCPy2QnmxtFyD0V6vco6IVKdIrxM8XtKJnwxY21ZL7vJa3rChbWEbJSKyQCId+OOrZcZOhrTUpvjoLRtJJrU0sohUp4uWdMzs82Z2w3w0ZraF7uCOjZWnY8YbFfYiUr2mUsN/EfgLM+s2sz81s21z3KZZE4SQzIE5xOvjxLSkgohUsYsmoLvf5+5vBN4KnKB8icK9ZvZHZrZ5zlt4CUJ3kmOOAfEmje5FpLpNecjr7gfc/U8rlyf8EPA+YO+ctWwWuDupLGCQaIr04QoRkYuacuCbWdLM3mNmXwUeBPYB75+zls2C0CGZLY/wFfgiUu0umoJm9mvAB4FbgceBB4C73X10jtt2ycZLOmAKfBGpelNJwT8E/gH4fXc/McftmVWhQyoLZqrhi4hcNPDd/VcArOzDwAZ3/xMzWwuscPfH57qRMxWGrpKOiEjFdOYp/i3wRsrlHYBTwN/MeotmUehOasx10FZEhOmdaft6d7/WzH4J4O6DZraoF6YpH7QFUw1fRGRaI/yimcUBBzCzZUB44acsLB8f4aMavojIdAL/c8C3gA4z+4/Ao8B/mpNWzZJANXwRkQlTTkF3/6qZPQncBBjw6+6+qE+8CsPyxcuphXiDRvgiUt2mNex19+eB5+eoLbPOsyHmEMvEtI6OiFS9qZx49Y8X2u7ut81ec2ZZZZVMahX2IiJTGeG/ETgE7AAeo1zOWRIsW1kPX4EvIjKlwF8BjC+v8CHg/wN2uPtzc9mwWZEtj/CtRvV7EZGpLI8cuPv33f2jwBuAl4BHzOx357x1l8hy5cCP1WmELyIypYO2ZpamvHjaB4EuylM0vzl3zZodllNJR0Rk3FQO2v49cBXlJZH/2N2fnfNWzZbxEX6tSjoiIlMZ4f8WMApsBj5hZl653wB398a5atylmjhoW6fAFxGZSuC/Cfi5u/tFH7nIxColnbhq+CIiU1pa4SPAk2b2gJndaWYr5rpRs2XioK1q+CIiU1oP/2MAZvYa4F3Al8ysCXgY+D7wU3cPprKzyuJrO4Fed3/3jFs9ReMlnbhq+CIi07qI+fPu/pfu/k7g7ZQXT/sNyidjTdUnmMcLn8fyCnwRkXEzqnW4e9bdv+fuv+vu26fyHDPrpDy187/OZJ8zYRM1fAW+iMh8Frf/Cvh3zOMa+rFKDT9er8AXEZn2IvFm9jWgDkgBAeWpme+8yHPeDRxz9yfN7G0XeNzdwN0Aa9eunW7TzhLPlT9d4vU6aCsiMu0kdPffoHzg9WbgncAPpvC0G4HbzKwbeAB4u5l95Ryvfb+7b3f37cuWLZtu084Sy5dH+AnV8EVEZlzS2Qyspryw2vqLPdjd/8DdO929C7gD+JG7f3iG+54SDxwrgJumZYqIwAxKOhWfAf5t5fZfz1JbZlUwFoBDkIJETIEvIjKjwHf3F4Dfn+FzHwEemclzpyMYDXCcUsqI2ZJZwl9EZM7MKPDN7A+BKymXhNzdPzSrrZoFYWWGTpBEgS8iwsxLOubuvzmrLZllXnDcy4FvquiIiMw48C8zs9+gvIom7v692WvS7AjzIQ6ECZV0RERg5oH/Y6C28rMoV9H0QrlZQQJiynsRkRmdeHUj0M+rFzNflIEf5kPcnTCpEb6ICMxshN8KtFA5YMsiDnyAQCUdERFgZmfa/i/KJ1zdWvlZPtuNmg3lET6EKumIiAAzP9N2mbv/hrt/gEUa+F5ZGjlMQFyJLyIyoxr+rcBKM/styuWcRXkFrCBXviZLkABTSUdE5OIjfDO78oy72oF/rjw3Xrm96ASVGn6YVNiLiMDURvhfBq4FMLO73H3iAiZmVuvuY3PVuEsRVM60JaXAFxGBqdXwJyfmx8/Y9pNZbMusGh/hu0b4IiLA1AJ/8rTLM9Nz0S5aECrwRUROM5WSzgozuxN4mrMDf1HOwQcIKrN0PLnADRERWSSmEvifBbYDvw10mtlzwPOVn/a5a9qlCfPlWToa4YuIlF008N39/sl/m1knsBW4Gvjfc9SuSxbmKl8+FPgiIsAM5uG7ew/QAyy6FTInmzhoq1k6IiLAIj7oeqnCgqZliohMFt3A1ywdEZHTRDbwvRL4llbgi4hAhAN/fFqmpSPbRRGRaYlsGo5fxDyuwBcRASIa+O4+cdA2qcAXEQGiGvgFJ3QnSEAiEckuiohMWyTTcPLVrhKxSHZRRGTaIpmGYSEkxAmSRkojfBERIKKB73mfNMLXtEwREYho4If5cKKGn4xHsosiItMWyTR8tYZvJFXSEREBIhz4oTthApIq6YiIABENfC9VavhxSKikIyICRDTwCcAdPAbJuEb4IiIQ0cD3UvnEq3LgR7KLIiLTFsk09ODVko4CX0SkbN7S0MzWmNnDZrbXzJ4zs0/M1b685DiOx0wlHRGRimlf4vASlID/w92fMrMG4Ekz+2d33zPbO/JAJR0RkTPNWxq6+2F3f6py+xSwF1g9J/sqOaFDGNOZtiIi4xZk+GtmXcBrgcfm4vXHp2V6DJ14JSJSMe9paGb1wDeAT7r78Bnb7jaznWa2s7+/f8b7KB+0rZR0tFqmiAgwz4FvZknKYf9Vd//mmdvd/X533+7u25ctWzbj/YyXdMojfJV0RERgfmfpGPBFYK+7/+c53VlQueqVpmWKiEyYzzS8Efgt4O1mtqvyc8tc7MgDJwTcTCUdEZGKeZuW6e6PAvNSX5mo4ceNhObhi4gAUT3TdnINXyUdEREgqoFfGeGHWjxNRGRCNANfI3wRkbNEMw0rs3Rcs3RERCZEMg3DYlg501aLp4mIjItm4JccAItDefq/iIhEMvBLhRCAWDKS3RMRmZFIJmIQlEf4cS2rICIyIZKBrxG+iMjZIpmIQbEc+HEtjSwiMiGSiTh+0DaRVElHRGRcJAN/fISvko6IyKsimYhBZYQf1whfRGRCJAM/rIzwE6rhi4hMiGQijo/wk6lIdk9EZEYimYjj0zJTqfgCt0REZPGIZOCPj/BTmUh2T0RkRiKZiOOzdNIa4YuITJi3SxzOp/HAT6Uj+XkmsmQVi0V6enrI5XIL3ZQlL5PJ0NnZSTKZnPJzIhn44ydepdKR7J7IktXT00NDQwNdXV1ayfYSuDsDAwP09PSwfv36KT8vkkPg8cDPqIYvsqjkcjna2toU9pfIzGhra5v2N6VIJmJYGp+loxG+yGKjsJ8dM/l3jGbgFzXCFxE5UyQTMaysh59WDV9EzuFb3/oWZsbzzz8/K6/31a9+la1bt7J161ZuuOEGnn766Ylt3//+97n88svZuHEj995778T9X/va17jyyiuJxWLs3LnztNfbvXs3b3zjG7nyyiu5+uqrZ+0gd+QC393xSg0/rRG+iJzDjh07eNOb3sQDDzwwK6+3fv16fvzjH7N7924+/elPc/fddwMQBAG/8zu/w4MPPsiePXvYsWMHe/bsAeCqq67im9/8Jm95y1tOe61SqcSHP/xhvvCFL/Dcc8/xyCOPTGsmzoVEbgjsgRO44wY1quGLLFr/8ktPzMnrfvHO6y+4fWRkhJ/+9Kc8/PDD3HbbbXz2s5/lkUce4TOf+QzLly9n165d3H777Vx99dXcd999ZLNZvv3tb3PZZZfR39/Pxz72MQ4ePAjAX/3VX3HjjTdyww03TLz+G97wBnp6egB4/PHH2bhxIxs2bADgjjvu4Dvf+Q5btmzhiiuuOGf7HnroIbZu3co111wDQFtb2yX/m4yL3hA4gDCEMA5pLZ4mImf49re/zTvf+U42b95Ma2srTz31FABPP/009913H8888wxf/vKX2bdvH48//jh33XUXf/3Xfw3AJz7xCe655x6eeOIJvvGNb3DXXXed9fpf/OIXede73gVAb28va9asmdjW2dlJb2/vBdu3b98+zIybb76Za6+9lj/7sz+bra5HcIRfckJ3PA6ZpM60FVmsLjYSnys7duzgk5/8JFAece/YsYNbb72V66+/npUrVwJw2WWX8Y53vAOAq6++mocffhiAH/zgBxMlGYDh4WFOnTpFQ0MDAA8//DBf/OIXefTRR4FyiflMF5tdUyqVePTRR3niiSeora3lpptu4rrrruOmm266tI4TxcAPKoEfg4xG+CIyycDAAD/60Y949tlnMTOCIMDMuOWWW0in0xOPi8ViE3/HYjFKpRIAYRjy85//nJqamrNee/fu3dx11108+OCDE2WYzs5ODh06NPGYnp4eVq1adcE2dnZ28ta3vpX29nYAbrnlFp566qlZCfzIJWIxH+AOxI1EPHLdE5FL8PWvf52PfOQjHDhwgO7ubg4dOsT69esnRuQX8453vIPPf/7zE3/v2rULgIMHD3L77bfz5S9/mc2bN09sv/7663nxxRd55ZVXKBQKPPDAA9x2220X3MfNN9/M7t27GRsbo1Qq8eMf/5gtW7ZMv7PnELlEzFWWRraETu4QkdPt2LGD973vfafd9/73v59/+Id/mNLzP/e5z7Fz5062bt3Kli1b+MIXvgDAn/zJnzAwMMDHP/5xtm3bxvbt2wFIJBJ8/vOf5+abb+aKK67gAx/4AFdeeSVQnhra2dnJz3/+c2699VZuvvlmAFpaWvi93/s9rr/+erZt28a1117LrbfeOiv9t3PVmBaD7du3+5lzU6fiyMun+MH7dxG2xfnID2+4+BNEZN7s3bv3vLNTZPrO9e9pZk+6+/ZzPT5yI/x8vnIBc43wRUROE73Az5UPrsRUvxcROU3kUjFfqeHHkxrhi4hMNq+Bb2bvNLMXzOwlM/vUXOwjnwsAiCnwRUROM2+Bb2Zx4G+AdwFbgA+a2ezMNZokny8Hflxz8EVETjOfqfg64CV33+/uBeAB4L2zvZPieOAnFfgiIpPNZyquBg5N+runct8EM7vbzHaa2c7+/v4Z7aQxnaCxJklj/eysLici0RKPx9m2bdvET3d398TiZ93d3VOek78UzefSCucqqp92EoC73w/cD+V5+DPZyca2epLtddQvr5vJ00Uk4mpqaibOkB33s5/9DHg18D/0oQ8tQMvm3nwGfg+wZtLfnUDfrO+lXNHB4jpoK7KYPfOeZ+bkda/+X1dP+zn19fWMjIzwqU99ir1797Jt2zY++tGPcs8998xBCxfOfAb+E8AmM1sP9AJ3ALP+MeqVq10p8EXkXLLZLNu2bQPKFy751re+NbHt3nvv5c///M/57ne/u0Ctm1vzFvjuXjKzfwP8ExAH/pu7Pzfr+ykp8EWWgpmMxGfDuUo61WJel0d29+8B35vTfYyP8LW0gojIaSI3d3F8hI+ufSIi09TQ0MCpU6cWuhlzJnqBrxG+iMzQ1q1bSSQSXHPNNfzlX/7lQjdn1kXuileWMOJNceK1GuKLyNlGRkbOe18ymeSHP/zhfDdp3kQu8Ft/tZXWX21d6GaIiCw6kSvpiIjIuSnwRWReLdar7C01M/l3VOCLyLzJZDIMDAwo9C+RuzMwMEAmk5nW8yJXwxeRxauzs5Oenh5mujiivCqTydDZ2Tmt5yjwRWTeJJNJ1q9fv9DNqFoq6YiIVAkFvohIlVDgi4hUCVusR8vNrB84MIOntgPHZ7k5i436uPRFvX8Q/T4u1v6tc/dl59qwaAN/psxsp7tvX+h2zCX1cemLev8g+n1civ1TSUdEpEoo8EVEqkQUA//+hW7APFAfl76o9w+i38cl17/I1fBFROTcojjCFxGRc1Dgi4hUiUgFvpm908xeMLOXzOxTC92e2WBm3Wb2jJntMrOdlftazeyfzezFyu+WhW7ndJjZfzOzY2b27KT7ztsnM/uDynv6gpndvDCtnp7z9PGzZtZbeS93mdktk7YtqT6a2Roze9jM9prZc2b2icr9kXkfL9DHpfs+unskfihftvxlYAOQAp4Gtix0u2ahX91A+xn3/RnwqcrtTwF/utDtnGaf3gJcCzx7sT4BWyrvZRpYX3mP4wvdhxn28bPA75/jsUuuj8BK4NrK7QZgX6UfkXkfL9DHJfs+RmmE/zrgJXff7+4F4AHgvQvcprnyXuDvK7f/Hvj1hWvK9Ln7/wZOnHH3+fr0XuABd8+7+yvAS5Tf60XtPH08nyXXR3c/7O5PVW6fAvYCq4nQ+3iBPp7Pou9jlAJ/NXBo0t89XPjNWSoceMjMnjSzuyv3LXf3w1D+jxLoWLDWzZ7z9Slq7+u/MbPdlZLPeLljSffRzLqA1wKPEdH38Yw+whJ9H6MU+HaO+6Iw5/RGd78WeBfwO2b2loVu0DyL0vv6d8BlwDbgMPAXlfuXbB/NrB74BvBJdx++0EPPcd9S7eOSfR+jFPg9wJpJf3cCfQvUllnj7n2V38eAb1H+injUzFYCVH4fW7gWzprz9Sky76u7H3X3wN1D4P/h1a/7S7KPZpakHIRfdfdvVu6O1Pt4rj4u5fcxSoH/BLDJzNabWQq4A/jHBW7TJTGzOjNrGL8NvAN4lnK/Plp52EeB7yxMC2fV+fr0j8AdZpY2s/XAJuDxBWjfJRsPwor3UX4vYQn20cwM+CKw193/86RNkXkfz9fHJf0+LvRR49n8AW6hfCT9ZeDfL3R7ZqE/Gygf9X8aeG68T0Ab8EPgxcrv1oVu6zT7tYPyV+Ei5VHRv7xQn4B/X3lPXwDetdDtv4Q+fhl4BthNORxWLtU+Am+iXK7YDeyq/NwSpffxAn1csu+jllYQEakSUSrpiIjIBSjwRUSqhAJfRKRKKPBFRKqEAl9EpEoo8EVEqoQCX0SkSijwpSqY2efN7MAlvsa/NjM3sysm3be3srDWTF7vv09aU32XmR0xs6musCkybQp8ibzKae5vA1LjS1XM0FbKZ1veWnndNLAcOOcHiZm9zcy+dL4Xc/ffdvdt7r6N8in6JeDOS2ifyAUp8KUa/DHwfwN7gCsv4XWuBu6lEviV19rrl3i6upm1Ad8H/i93X9LrP8nipsCXSDOzK4GrgP9B+QIWZwW+mf3kjNLK+M+vnvHQLZTXTukwsybKHwDPXGL7aoDvAv/T3f/LpbyWyMUkFroBInPsPwKfdnc3s72Uw/807v7mi72Ima0BBtw9a2b/DNxMucSz+xyPfYzyZe7qgVYz21XZ9H+6+z9Nelyc8gfR8+7+6Wn3TGSaFPgSWWb2esrBvM3M/gbIcO6A/gnla5ae6ffd/QeV21t5dTT/PeA3KV/z9NtnPsndX1953bcBd7r7nedp4t8CSeBfTaU/IpdKgS9R9p+Ad7v7DwHMbDnwyzMfNJURPqeXb34MfAGoZYYlHTP7DHAd8DZ3L83kNUSmSzV8iSQz+zUgPR72UL5SEVBnZq0zeMmJwHf3fOV2wd2HZtC2LuCzlNeOf3TSMYP/MYN2iUyZ1sMXEakSGuGLiFQJBb6ISJVQ4IuIVAkFvohIlVDgi4hUCQW+iEiVUOCLiFSJ/x8WJLOkYjhDSgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "Masses['Eapprox'] = ytilde\n", + "# Generate a plot comparing the experimental with the fitted values values.\n", + "fig, ax = plt.subplots()\n", + "ax.set_xlabel(r'$A = N + Z$')\n", + "ax.set_ylabel(r'$E_\\mathrm{bind}\\,/\\mathrm{MeV}$')\n", + "ax.plot(Masses['A'], Masses['Ebinding'], alpha=0.7, lw=2,\n", + " label='Ame2016')\n", + "ax.plot(Masses['A'], Masses['Eapprox'], alpha=0.7, lw=2, c='m',\n", + " label='Fit')\n", + "ax.legend()\n", + "save_fig(\"Masses2016OLS\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Adding error analysis and training set up\n", + "\n", + "We can easily test our fit by computing the $R2$ score that we discussed in connection with the functionality of **Scikit-Learn** in the introductory slides.\n", + "Since we are not using **Scikit-Learn** here we can define our own $R2$ function as" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def R2(y_data, y_model):\n", + " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and we would be using it as" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.9547578478889096\n" + ] + } + ], + "source": [ + "print(R2(Energies,ytilde))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can easily add our **MSE** score as" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.03787596148305234\n" + ] + } + ], + "source": [ + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "print(MSE(Energies,ytilde))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and finally the relative error as" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "A \n", + "1 0 inf\n", + "2 1 1.123190\n", + "3 2 0.327631\n", + "4 6 0.344172\n", + "5 9 0.044402\n", + " ... \n", + "264 3304 0.009911\n", + "265 3310 0.009154\n", + "266 3317 0.007824\n", + "269 3338 0.011347\n", + "270 3344 0.009790\n", + "Name: Ebinding, Length: 267, dtype: float64\n" + ] + } + ], + "source": [ + "def RelativeError(y_data,y_model):\n", + " return abs((y_data-y_model)/y_data)\n", + "print(RelativeError(Energies, ytilde))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The $\\chi^2$ function\n", + "\n", + "Normally, the response (dependent or outcome) variable $y_i$ is the\n", + "outcome of a numerical experiment or another type of experiment and is\n", + "thus only an approximation to the true value. It is then always\n", + "accompanied by an error estimate, often limited to a statistical error\n", + "estimate given by the standard deviation discussed earlier. In the\n", + "discussion here we will treat $y_i$ as our exact value for the\n", + "response variable.\n", + "\n", + "Introducing the standard deviation $\\sigma_i$ for each measurement\n", + "$y_i$, we define now the $\\chi^2$ function (omitting the $1/n$ term)\n", + "as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\chi^2(\\boldsymbol{\\beta})=\\frac{1}{n}\\sum_{i=0}^{n-1}\\frac{\\left(y_i-\\tilde{y}_i\\right)^2}{\\sigma_i^2}=\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)^T\\frac{1}{\\boldsymbol{\\Sigma^2}}\\left(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}}\\right)\\right\\},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where the matrix $\\boldsymbol{\\Sigma}$ is a diagonal matrix with $\\sigma_i$ as matrix elements.\n", + "\n", + "\n", + "\n", + "## The $\\chi^2$ function\n", + "\n", + "In order to find the parameters $\\beta_i$ we will then minimize the spread of $\\chi^2(\\boldsymbol{\\beta})$ by requiring" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_j} = \\frac{\\partial }{\\partial \\beta_j}\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(\\frac{y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}}{\\sigma_i}\\right)^2\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which results in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_j} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}\\frac{x_{ij}}{\\sigma_i}\\left(\\frac{y_i-\\beta_0x_{i,0}-\\beta_1x_{i,1}-\\beta_2x_{i,2}-\\dots-\\beta_{n-1}x_{i,n-1}}{\\sigma_i}\\right)\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or in a matrix-vector form as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{A}^T\\left( \\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{\\beta}\\right).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we have defined the matrix $\\boldsymbol{A} =\\boldsymbol{X}/\\boldsymbol{\\Sigma}$ with matrix elements $a_{ij} = x_{ij}/\\sigma_i$ and the vector $\\boldsymbol{b}$ with elements $b_i = y_i/\\sigma_i$.\n", + "\n", + "\n", + "\n", + "## The $\\chi^2$ function\n", + "\n", + "We can rewrite" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = 0 = \\boldsymbol{A}^T\\left( \\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{\\beta}\\right),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{A}^T\\boldsymbol{b} = \\boldsymbol{A}^T\\boldsymbol{A}\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and if the matrix $\\boldsymbol{A}^T\\boldsymbol{A}$ is invertible we have the solution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta} =\\left(\\boldsymbol{A}^T\\boldsymbol{A}\\right)^{-1}\\boldsymbol{A}^T\\boldsymbol{b}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The $\\chi^2$ function\n", + "\n", + "If we then introduce the matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{H} = \\left(\\boldsymbol{A}^T\\boldsymbol{A}\\right)^{-1},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have then the following expression for the parameters $\\beta_j$ (the matrix elements of $\\boldsymbol{H}$ are $h_{ij}$)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_j = \\sum_{k=0}^{p-1}h_{jk}\\sum_{i=0}^{n-1}\\frac{y_i}{\\sigma_i}\\frac{x_{ik}}{\\sigma_i} = \\sum_{k=0}^{p-1}h_{jk}\\sum_{i=0}^{n-1}b_ia_{ik}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We state without proof the expression for the uncertainty in the parameters $\\beta_j$ as (we leave this as an exercise)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\sigma^2(\\beta_j) = \\sum_{i=0}^{n-1}\\sigma_i^2\\left( \\frac{\\partial \\beta_j}{\\partial y_i}\\right)^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "resulting in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\sigma^2(\\beta_j) = \\left(\\sum_{k=0}^{p-1}h_{jk}\\sum_{i=0}^{n-1}a_{ik}\\right)\\left(\\sum_{l=0}^{p-1}h_{jl}\\sum_{m=0}^{n-1}a_{ml}\\right) = h_{jj}!\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The $\\chi^2$ function\n", + "The first step here is to approximate the function $y$ with a first-order polynomial, that is we write" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y=y(x) \\rightarrow y(x_i) \\approx \\beta_0+\\beta_1 x_i.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By computing the derivatives of $\\chi^2$ with respect to $\\beta_0$ and $\\beta_1$ show that these are given by" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_0} = -2\\left[ \\frac{1}{n}\\sum_{i=0}^{n-1}\\left(\\frac{y_i-\\beta_0-\\beta_1x_{i}}{\\sigma_i^2}\\right)\\right]=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial \\chi^2(\\boldsymbol{\\beta})}{\\partial \\beta_1} = -\\frac{2}{n}\\left[ \\sum_{i=0}^{n-1}x_i\\left(\\frac{y_i-\\beta_0-\\beta_1x_{i}}{\\sigma_i^2}\\right)\\right]=0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The $\\chi^2$ function\n", + "\n", + "For a linear fit (a first-order polynomial) we don't need to invert a matrix!! \n", + "Defining" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma = \\sum_{i=0}^{n-1}\\frac{1}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_x = \\sum_{i=0}^{n-1}\\frac{x_{i}}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_y = \\sum_{i=0}^{n-1}\\left(\\frac{y_i}{\\sigma_i^2}\\right),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_{xx} = \\sum_{i=0}^{n-1}\\frac{x_ix_{i}}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\gamma_{xy} = \\sum_{i=0}^{n-1}\\frac{y_ix_{i}}{\\sigma_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we obtain" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_0 = \\frac{\\gamma_{xx}\\gamma_y-\\gamma_x\\gamma_y}{\\gamma\\gamma_{xx}-\\gamma_x^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_1 = \\frac{\\gamma_{xy}\\gamma-\\gamma_x\\gamma_y}{\\gamma\\gamma_{xx}-\\gamma_x^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This approach (different linear and non-linear regression) suffers\n", + "often from both being underdetermined and overdetermined in the\n", + "unknown coefficients $\\beta_i$. A better approach is to use the\n", + "Singular Value Decomposition (SVD) method discussed below. Or using\n", + "Lasso and Ridge regression. See below.\n", + "\n", + "\n", + "\n", + "\n", + "## Fitting an Equation of State for Dense Nuclear Matter\n", + "\n", + "Before we continue, let us introduce yet another example. We are going to fit the\n", + "nuclear equation of state using results from many-body calculations.\n", + "The equation of state we have made available here, as function of\n", + "density, has been derived using modern nucleon-nucleon potentials with\n", + "[the addition of three-body\n", + "forces](https://www.sciencedirect.com/science/article/pii/S0370157399001106). This\n", + "time the file is presented as a standard **csv** file.\n", + "\n", + "The beginning of the Python code here is similar to what you have seen\n", + "before, with the same initializations and declarations. We use also\n", + "**pandas** again, rather extensively in order to organize our data.\n", + "\n", + "The difference now is that we use **Scikit-Learn's** regression tools\n", + "instead of our own matrix inversion implementation. Furthermore, we\n", + "sneak in **Ridge** regression (to be discussed below) which includes a\n", + "hyperparameter $\\lambda$, also to be explained below.\n", + "\n", + "## The code" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mean squared error: 12.36\n", + "Variance score: 1.00\n", + "Mean absolute error: 2.83\n", + "[ 0. 618.32047562 -861.13519106 1404.91549644] -11.057088709963523\n", + "Mean squared error: 197.93\n", + "Variance score: 1.00\n", + "Mean absolute error: 11.69\n", + "[ 0. 28.18220995 282.79902342 842.30879705] 12.946893955207997\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEMCAYAAAA1VZrrAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8ZUlEQVR4nO3dd3xVVbbA8d9KSKNLhwQIJfQSIFTpRZEiYAULWJ6ooGAXxxnb6IxPGR0Un44VUBQQRBBBBRTpJWAooUMSCAQCoSWQerPeH/fCRAjJBZLclPX9fPLhnH32uWfdm8NdOfvss7eoKsYYY0xOvDwdgDHGmMLPkoUxxphcWbIwxhiTK0sWxhhjcmXJwhhjTK4sWRhjjMlVKU8HkF+qVKmiwcHBng7DGGOKlI0bNx5X1aoXlxfbZBEcHEx4eLinwzDGmCJFRGKyK7dmKGOMMbmyZGGMMSZXliyMMcbkKt/uWYiIP7Ac8HMdZ7aqviwilYCZQDAQDdyhqidd+7wAPAg4gHGq+rOrvB0wBQgAFgLj9SoGtUpPTyc2NpaUlJRre3OmUPD39ycoKAgfHx9Ph2JMsZefN7hTgd6qmiQiPsBKEVkE3AIsVdU3RWQCMAF4XkSaAcOB5kAtYImINFJVB/AhMBpYizNZ9AcWXWlAsbGxlCtXjuDgYEQkL96j8RBVJSEhgdjYWOrVq+fpcIwp9vKtGUqdklyrPq4fBYYAU13lU4GhruUhwAxVTVXVKGAv0EFEagLlVXWN62piWpZ9rkhKSgqVK1e2RFEMiAiVK1e2q0RjCki+3rMQEW8RiQDigcWqug6orqpxAK5/q7mqBwIHs+we6yoLdC1fXJ7d8UaLSLiIhB87duxyMV39GzKFiv0ujfmzQ+EnWTR2K45UR56/dr4mC1V1qGooEITzKqFFDtWz+5+vOZRnd7yPVTVMVcOqVr3kmRKPS0hIIDQ0lNDQUGrUqEFgYCChoaFUrFiRZs2aFWgs33//Pdu3b7+w/tJLL7FkyZIrfp3o6GhatMjp1+p03333MXv27BzrTJkyhcOHD19xDMYY2Db7EL8/GknCqlP8/klUnr9+gfSGUtVTwDKc9xqOupqWcP0b76oWC9TOslsQcNhVHpRNeZFTuXJlIiIiiIiI4JFHHuHJJ5+8sO7llfe/ioyMjMtuuzhZvPbaa/Tt2zfPY7gSliyMuXKqyu//u4fNb+wnM11x9CxLhwfq5vlx8i1ZiEhVEanoWg4A+gI7gfnAKFe1UcA81/J8YLiI+IlIPSAEWO9qqkoUkU7ibHcYmWWfYsPhcPDQQw/RvHlzbrjhBpKTkwHYt28f/fv3p127dnTr1o2dO3cCEBMTQ58+fWjVqhV9+vThwIEDgPMv+KeeeopevXrx/PPPZ7v/6tWrmT9/Ps8++yyhoaHs27fvT3/5b9iwgS5dutC6dWs6dOhAYmIi0dHRdOvWjbZt29K2bVtWr16d4/tRVR577DGaNWvGwIEDiY+Pv7Dttddeo3379rRo0YLRo0ejqsyePZvw8HDuvvtuQkNDSU5OzraeMea/0s85WDB2Cwe/OUImEDCqKv1fD6Js6bzvIZifvaFqAlNFxBtnUpqlqgtEZA0wS0QeBA4AtwOoaqSIzAK2AxnAWFdPKIBH+W/X2UVcRU+oiz04ZcO1vkS2Pruv/VXtt2fPHr755hs++eQT7rjjDubMmcM999zD6NGj+eijjwgJCWHdunWMGTOGX3/9lccee4yRI0cyatQoPv/8c8aNG8f3338PwO7du1myZAne3t706dMn2/1vvvlmBg0axG233fanONLS0rjzzjuZOXMm7du358yZMwQEBFCtWjUWL16Mv78/e/bsYcSIETkOpzJ37lx27drF1q1bOXr0KM2aNeOBBx4A4LHHHuOll14C4N5772XBggXcdtttTJ48mYkTJxIWFnbZeoMHD76qz9eY4ub04WR+HruNtKgUHP5C0IRa7An6hccX/cq7N75LYPlsb+1etXxLFqq6BWiTTXkC0Ocy+7wBvJFNeTiQe8N4EVavXj1CQ0MBaNeuHdHR0SQlJbF69Wpuv/32C/VSU1MBWLNmDd999x3g/CJ97rnnLtS5/fbb8fb2znH/y9m1axc1a9akfXtn0itfvjwAZ8+e5bHHHiMiIgJvb292796d4+ssX76cESNG4O3tTa1atejdu/eFbb/99htvvfUW586d48SJEzRv3jzbJOBuPWNKmgObTrLyqR1knnKQXtmLJv+sztzTH7Fjzw5KeZUi6lRU0UkWhd3VXgHkFz8/vwvL3t7eJCcnk5mZScWKFYmIiMh1/6w9g8qUKQNwRfufp6rZ9jJ69913qV69Ops3byYzMxN/f/8rium8lJQUxowZQ3h4OLVr1+aVV17Jtvuru/WMKWn+mHuIyH9GQZriaOhL/Ve9+b/dr3Iq5RSVAyrzQtcXaFylcZ4f14b7KMTKly9PvXr1+PbbbwHnF/nmzZsB6NKlCzNmzABg+vTpdO3a9Yr2L1euHImJiZfs06RJEw4fPsyGDc5musTERDIyMjh9+jQ1a9bEy8uLL7/8Eocj56553bt3Z8aMGTgcDuLi4vjtt98ALnzhV6lShaSkpD/1kMoaU071jCmJMjMzWTJxN5Gv7oc0RbqVofRLh3l36+ucSjlFy2otmdR/Ur4kCrBkUehNnz6dzz77jNatW9O8eXPmzXPe23/vvff44osvaNWqFV9++SWTJk26ov2HDx/O22+/TZs2bdi3b9+F+r6+vsycOZPHH3+c1q1b069fvwt/5U+dOpVOnTqxe/fuC1cvlzNs2DBCQkJo2bIljz76KD169ACgYsWKPPTQQ7Rs2ZKhQ4deaO4C5835Rx55hNDQUPz8/C5bz5iSJjU5g+8f38KRr46iAqVHlufw8KVMi/wchzoY1mQYf+/1dyr4V8i3GKS49jAJCwvTi2/A7tixg6ZNm3ooIpMf7HdqirsTccn89NhWMvelkukrXPdsAL+UmUL06Wj8S/kzvuN4uta5tGXhaonIRlUNu7i8xN6zMMaYwm5/xElWPbkDOekg8zpvyvztHNNPTObs6bMElgvkL93+Qp0KdQokFksWxhhTCK2fd4hdb0QhaQoNfEkev51vDzl7QHYM7MiTnZ6kjG/OzcF5yZKFMcYUIpmZmfw0aQ8J0+IRBe3qze4hC4k4tAlBuKfVPdze7PYCHxvNkoUxxhQSKSkZfP/sNjJWJCJA5l1pLG/yDUePH6Wcbzme7fIsbWpe8vhagbBkYYwxhcCxI8ksfGwL3nvTwFc4+3gcv/jNIO1sGvUr1ufF7i9SrUy13F8on1iyMMYYD9sZcYK1T++kVIKDzEpK7GPhrEn9FRzQp14fxrQfg6+3r0djtOcsClhsbCxDhgwhJCSEBg0aMH78eNLS0li2bBmDBg26pP6CBQto06YNrVu3plmzZvznP//xQNTGmPyycv5B1j2ynVIJDlIbpLDx8XmsSf2VUl6lGBM2hvEdx3s8UYBdWRQoVeWWW27h0UcfZd68eTgcDkaPHs2LL77IwIEDL6mfnp7O6NGjWb9+PUFBQaSmphIdHV3wgRtj8lyGI5MFk3aT+OUxvBVOdT3Ouh7f5fuwHVfLkkUB+vXXX/H39+f+++8HnGNAvfvuu9SrV49evXpdUv/8UBuVK1cGnONHNW5ceE4eY8zVSTqbztzntiKrzoIoB+/Ywbr6i3BkOGhRtQXPd32eiv4VPR3mn5TYZLF18NZ8ed2WP7S87LbIyEjatWv3p7Ly5ctTp04d9u7de0n9SpUqcfPNN1O3bl369OnDoEGDGDFiRL5MlGSMKRgHYxJZ8kQkPlHpOALS2fbA7+wsuwkUhjYeyqjQUZTyKnxfzYUvomLsciO6Xq4c4NNPP2Xr1q0sWbKEiRMnsnjxYqZMmZLPkRpj8sPGlUfY8uJefE4rZ2qeJHzEAo75xOHn7cf4juPpVrebp0O8rBKbLHK6AsgvzZs3Z86cOX8qO3PmDAcPHqRBgwaX3a9ly5a0bNmSe++9l3r16lmyMKaIUVUWTtvP8cmH8UmH2NB9bOi7gDTvVALLBfJC1xeoWzHvp0LNS9aeUYD69OnDuXPnmDZtGuCcSvXpp5/mvvvuo3Tp0pfUT0pKYtmyZRfWIyIiqFu3cJ9Qxpg/S07LYPrftnLy3cNIuoOIActZ2Xc2ad6pdAnqwjs3vlPoEwVYsihQIsLcuXP59ttvCQkJoVGjRvj7+/OPf/wDgKVLlxIUFHTh548//uCtt96icePGhIaG8vLLL9tVhTFFyNFjyXz94CZYcJpk/yR+vXcWu1qsxbuUNw+2eZAJXSdQ2ufSPxQLoxLbDOUptWvX5ocffrikvGfPniQnJ19S3q1b4W3DNMZcXuTWBNY8uwv/Iw7iqh9g7bD5pFU4R6WASjzX5TmaV2vu6RCviCULY4zJQ6rKrz8c5MCbB/A7l8nWFuvY2ut3vEsLLaq24Lnrn+O6gOs8HeYVs2RhjDF5JN2RyZxJO0mbnkAmKSztvYBjofvw9hFubXor97a6F28vb0+HeVUsWRhjTB44dSaNOS9uxW/FOU6WPsJvN32Ho0ESZX3L8ETHJ+hcu7OnQ7wmliyMMeYa7Ys+w69PbSdgfzq7a0Sw7saf8K3hRXCFYF7o9gK1ytXydIjXzJKFMcZcg1UrDrPzpSh8TqXxe/NFxHTbgl8FH3oH92ZM+zH4lfLzdIh5It+6zopIbRH5TUR2iEikiIx3lb8iIodEJML1MyDLPi+IyF4R2SUiN2YpbyciW13b3pOCniLKGGMu4shUvpuyhz1P7yPt7AnmdfucQ/0iKXNdAGPbj+WJTk8Um0QB+XtlkQE8raqbRKQcsFFEFru2vauqE7NWFpFmwHCgOVALWCIijVTVAXwIjAbWAguB/sCifIzdGGMuKzElnW9fi8R3YSKHKu7h165zCGgi1CxXgwldJxBSOcTTIea5fEsWqhoHxLmWE0VkBxCYwy5DgBmqmgpEicheoIOIRAPlVXUNgIhMA4ZiycIY4wExh5L4+ZlIAnaksq72r2y9fhXl6vjTrlY7nu78NOX8ynk6xHxRIE9wi0gw0AZY5yp6TES2iMjnInK+w3EgcDDLbrGuskDX8sXl2R1ntIiEi0j4sWPH8vIt5Clvb29CQ0Np0aIFgwcP5tSpUwB06dIl2/qvvPIKEydOzHbbldq6dSt169blww8/vKbX+emnn2jcuDENGzbkzTffvGy9Bx54gGrVqtGiRYtrOp4xhcHatUf5ZeRm2HOCeaFT2Nl/LeXrBnB3q7t5ucfLxTZRQAEkCxEpC8wBnlDVMziblBoAoTivPP51vmo2u2sO5ZcWqn6sqmGqGla1atVrDT3fBAQEEBERwbZt26hUqRIffPABAKtXr873Y7ds2ZIZM2ZcGJ/qajgcDsaOHcuiRYvYvn0733zzDdu3b8+27n333cdPP/101ccypjDIzFTmTtvDzvG7OZG2nxnXf8DZ7keoVrMyr/Z8leEthl925OjiIl+ThYj44EwU01X1OwBVPaqqDlXNBD4BOriqxwK1s+weBBx2lQdlU14sdO7cmUOHDgFQtmzZC+VvvPEGjRs3pm/fvuzatetP+/z973+nSZMm9OvXjxEjRly46vjqq6/o0KEDoaGhPPzwwzgcjmyPWa1aNSIjI6865vXr19OwYUPq16+Pr68vw4cPZ968ednW7d69O5UqVbrqYxnjaYkp6Ux5aTOJ78YRUWkF87tPoXR7aFm7BZP6T6JNzTaeDrFA5Ns9C1ePpc+AHar6Tpbymq77GQDDgG2u5fnA1yLyDs4b3CHAelV1iEiiiHTC2Yw1Enj/WuMb/M3ga32JbP0w4tJxny7H4XCwdOlSHnzwwT+Vb9y4kRkzZvDHH3+QkZFB27ZtL0yaFB4ezpw5cy7ZtmPHDmbOnMmqVavw8fFhzJgxTJ8+nZEjR15y3AkTJpCamkpMTMwlo9h269aNxMTES/aZOHEiffv2BeDQoUPUrv3fvB4UFMS6desu2ceYoi7mcBI/PReJ947TLGw8m7jW+6lUtzTDmg5jZOuRhXKSovySn+/0euBeYKuIRLjK/gKMEJFQnE1J0cDDAKoaKSKzgO04e1KNdfWEAngUmAIE4LyxXaRvbicnJxMaGkp0dDTt2rWjX79+f9q+YsUKhg0bdmHY8ptvvvnCtpUrVzJkyBACAgIAGDzYmfSWLl3Kxo0bad++/YVjVKtW7ZJj//TTT5w9e5aBAwcSGRl5SbJYsWJFrvGrXtoKWNwvwU3Js27DUba8uJekswdZ2PZrpFUqNWtW4YlOT9ApqJOnwytw+dkbaiXZ329YmMM+bwBvZFMeDuTpHdIruQLIa+fvWZw+fZpBgwbxwQcfMG7cuD/VudyXb3Zf1OfLR40axT//+c/LHjclJYXnnnuO+fPn88UXX7Bt2zYGDBjwpzruXFkEBQVx8OB/+yLExsZSq1bRf0LVGHDen5j3zX5Ovn+IPeXXs6LDj1RsEUCjWk14odsL1Chbw9MheoTNZ+FBFSpU4L333mPixImkp6dfKO/evTtz584lOTmZxMTEPw1p3rVrV3744QdSUlJISkrixx9/BJwTK82ePZv4+HgATpw4QUxMzJ+O9/rrrzNy5EiCg4Np2bIl27Zt42IrVqwgIiLikp/ziQKgffv27Nmzh6ioKNLS0pgxY8afrn6MKaqSUtKZ8uoWTrwTxZKgmawIW0DltmUY1HIgb9/wdolNFGDJwuPatGlD69atmTFjxoWytm3bcueddxIaGsqtt976pzkt2rdvz80330zr1q255ZZbCAsLo0KFCjRr1ozXX3+dG264gVatWtGvXz/i4uIu7Ldr1y4WL17ME088AXDZZOGOUqVKMXnyZG688UaaNm3KHXfcQfPm/x2bf8CAARw+7OyDMGLECDp37syuXbsICgris88+u6pjGpPfDh5JYvroTST+socZLT8gtu1OAkOr8Ey3ZxjbYSy+3r6eDtGj5HLNGkVdWFiYhoeH/6lsx44dNG3a1EMR5Z2kpCTKli3LuXPn6N69Ox9//DFt27b1dFgeUVx+p8azNmyM54+/7CHaEc6vjb6nbBNfGjVowAtdX6B2hdq5v0AxIiIbVTXs4vKScyu/GBk9ejTbt28nJSWFUaNGldhEYcy1ysxU5s3Yz/H3Y1hV4we2B4dTpXk5+jbrw9gOY/Ev5e/pEAsNSxZF0Ndff+3pEIwp8hJT0pn1RiTnFkexIGQ6Z2ofI6h5FR7u+DA3NrjRevhdxJKFMabEiTp4hl8mbCf2yEYWt5qDb7DQvEUIL3R9gQaVGng6vELJkoUxpkRZtSKOLa/uYl25H9jadB2VmpShd+uejOs4jjK+ZTwdXqFlycIYUyJkODL57rM9xE3bzsJ60zlZLZ5aLSrxyPUPMyBkgDU75aLEJQtVtZOimCiuPflM3juZlMqclyKJ2byGX5rNwbu60qptIyb0mEDDSg09HV6RUKKShb+/PwkJCVSuXNkSRhGnqiQkJODvb71VTM527zvF4ue2EO6Yz5bGaygXHMANnXszvuN4a3a6AiUqWQQFBREbG0thnuvCuM/f35+goKDcK5oSSVX5bXEsm97eyJIa0zl+XRzVml3H2N6PMDBkoP3BeIVKVLLw8fGhXr16ng7DGJPP0jIymT15J9t//I0lwXPIrJhBi7AQXuz7l2I55WlBKFHJwhhT/B07lcLsFyJYHzebzQ3XEFDDj5t79md8Z2t2uhaWLIwxxcbmLQn88vIqlpb9kmO1DlO5YXnGDxprzU55wJKFMabIU1V+nBXFyqk/8lvgXByl02natiEvD/qrNTvlEUsWxpgiLSklnW/eiGBF5HS21tmA33WlGNr7Rp7s8aQ1O+WhXJOFiDQCPgSqq2oLEWkF3Kyqr+d7dMYYk4PomERm/uU3lnhPIaH6USrWLsszt45jQCN7yC6vuTOfxSfAC0A6gKpuAYbnZ1DGGJObFUtief+Jz5hT5h1OlI+nWdsQPn/0IwY2tvsT+cGdZqjSqrr+og8/I5/iMcaYHKU7Mpn5/mYWrvqMHdX/oFRZb27rO5gn+463IcXzkTvJ4riINAAUQERuA+Jy3sUYY/LesRPJfPbizyxO/oKTVY5TvnpZ/nrXM/Rr1M/ToRV77iSLscDHQBMROQREAffka1TGGHORLX8c49P/ncqaCvPJLOOgadPGvHnvqyVuJjtPyTVZqOp+oK+IlAG8VDUx/8MyxhgnVWXu9Ei+Xvg++ypuw9vfi+F9bmP8AJsXuyBdNlmIyFOXKQdAVd/Jp5iMMQaAxHPpTH51AT/Ff8qZ8qcof105Xh01gZ5Neno6tBInpyuLcgUWhTHGXGT37pO8888P2OD3E+qvNKvflLcffo1a5Wt5OrQS6bLJQlVfvZYXFpHawDSgBpAJfKyqk0SkEjATCAaigTtU9aRrnxeABwEHME5Vf3aVtwOmAAHAQmC82mQGxhRLqsr877by6bx/EVt6L16+wl09hvPEsEcp5WXPEXtKrs9ZiMhUEamYZf06EfncjdfOAJ5W1aZAJ2CsiDQDJgBLVTUEWOpax7VtONAc6A/8n4h4u17rQ2A0EOL66e/e2zPGFCXJaRn849VZvLXoGWJL76VCuQq89+hEnrn1cUsUHubOp99KVU+dX1HVkyLSJredVDUOVxdbVU0UkR1AIDAE6OmqNhVYBjzvKp+hqqlAlIjsBTqISDRQXlXXAIjINGAosMiN2I0xRURU9Ele++dEImQZ4gOt64Tyzri/U6lMJU+HZnAvWXiJyHVZmooqubnfBSISDLQB1uEcNuR8EokTkWquaoHA2iy7xbrK0l3LF5dnd5zROK9AqFOnzpWEaIzxoB9+3MT7373JMZ9DlCrlzX097mPMbffbk9iFiDtf+v8CVovIbNf67cAb7h5ARMoCc4AnVPVMDr/87DZoDuWXFqp+jPOZEMLCwuyehjGFXFqGg3+8PY2FUdNI90mjUkAV3h79d9o2beXp0MxF3HnOYpqIhAO9cX5x36Kq2915cRHxwZkopqvqd67ioyJS03VVUROId5XHAlmfrgkCDrvKg7IpN8YUYTGHEpjwv6+zM2094g3ta3bmX0+/TLky1hGzMLrsDW4RKe/6txJwBPgamA4ccZXlSJyXEJ8BOy56JmM+MMq1PAqYl6V8uIj4iUg9nDey17uarBJFpJPrNUdm2ccYUwTN/WkNI1+7n51p6/Hx8mFsj/H8529vW6IoxHK6svgaGARs5M/NPuJar5/La18P3AtsFZEIV9lfgDeBWSLyIHAAZ7MWqhopIrOA7Th7Uo1VVYdrv0f5b9fZRdjNbWOKpLT0DP72zv+xJOpbMiWTGj6BvPXI67Rs3sjToZlcSHF9XCEsLEzDw8M9HYYxxmXPgUM8885LHEjZCUCP6jfwzwnPERAQ4OHITFYislFVwy4ud2fyo6Wq2ie3MmOMuZzpC35i8sJ3SMk8S2ktw+O9n2T4nTd5OixzBXIaG8ofKA1UEZHr+G+vpPKAPW9vjMnVubQUnn3nLVbH/AwK9bya8Nb4V2jY2EaKLWpyurJ4GHgCZ2LYyH+TxRngg/wNyxhT1G3Zv4enJ/+V4+di8VIvBlS/jZdfHEspP3sSuyjKaWyoSSIyGfiLqv69AGMyxhRhqspH38/ks6Uf4chI57qMyjx543MMvr2rp0Mz1yDHFK+qDhEZAFiyMMbk6uS5U4yb9ArbDmwAhZaZHXljwl+o3bCKp0Mz18id68FfRORW4Dsb6dUYcznLtq7hb1NeJ+nsKfwcftxS8z6e/us9ePvmOl6pKQLcSRZPAWWADBFJwfWchaqWz9fIjDFFQrojnde/mcz8tbMhAwJTgnl62AR6DWnp6dBMHnJnuA97pNIYk62ohBjGf/BXDsbtx0uF6x038ddXx1Gtrn1tFDdudUtwdZ0NAfzPl6nq8vwKyhhTuKkqM1Z/z7tz3iP9XBoVUq/j7pAxPPhcf7xKWbNTceTOQ3n/A4zHOYBfBM6JjNbgHFjQGFPCnEk9w/NTX2ftllWQAc2S2vHk/U8R1ifY06GZfOTOlcV4oD2wVlV7iUgT4JqmXDXGFE3rYzYw4bPXORl/HD+HPzdwJ0/9614qVrchO4o7d5JFiqqmiAgi4qeqO0Wkcb5HZowpNNIcaby76ENmLv0WTVYCk+oyst3j3Da+I15e1uxUEriTLGJdc3B/DywWkZPYfBLGlBj7T+znmSkvE7V3P14OLzqfvpEx4x6meZdque9sig13ekMNcy2+IiK/ARWAn/I1KmOMx6kqMzbO5t9zPiTtZCqVUqowqNwD/M9H/SlX0c/T4ZkC5m5vqLZAV5zzWKxS1bR8jcoY41HHzx3npdn/YE34ekhTWiV04u4BD9HvvsY2L3YJ5U5vqJdwTlB0flrUL0TkW1V9PV8jM8Z4xPKo5bwy/S1OHjpJ6bQy9D17J/e/divBTSt6OjTjQe5cWYwA2qhqCoCIvAlsAixZGFOMnE07y79//YDvlv5AZmIm9U41ZmjDB7njrx3x97eRYks6d86AaJwP46W41v2AffkVkDGm4G2L38bfZr5BzK4DeKeVokf8EO54aDidBgZ5OjRTSLiTLFKBSBFZjPOeRT9gpYi8B6Cq4/IxPmNMPkp3pDM1fBqfLPiKtGNpVD8bSH/vkYz4oBfVA8t4OjxTiLiTLOa6fs5blj+hGGMKUsypGF6Z/w+2bIxEU5UOcb0Z0HsEAx9rQilve3bC/Jk7XWenFkQgxpiCoap8v+N73v3+Q5IOnqVCcmX6nR7B0L/dQLOwyp4OzxRSdtfKmBLk+LnjvLn4LZatXIsj0UHL+Pb0bnAXt74fSrlyvp4OzxRiliyMKQFUld9jfud/57/L0d0JBCSXpnfcrfR76Ca6Da1tz06YXOWYLETEG3hTVZ8toHiMMXnsTOoZ3l85mfnLfiHtWDoNTjalt+9dDPmoA4F1yno6PFNE5HgXS1UdQDu5ij87RORzEYkXkW1Zyl4RkUMiEuH6GZBl2wsisldEdonIjVnK24nIVte2964mFmNKqg2HNnD/tIf4bv5C9KjQN/pW7u70NA990cMShbki7jRD/QHME5FvgbPnC1X1u8vvAsAUYDIw7aLyd1V1YtYCEWkGDAeaA7WAJSLSyJWsPgRGA2uBhUB/YJEbcRtTYiWnJ/Np+KfMWDKX5MOpBJ4JpnfiXfR9vSMtw6p4OjxTBLmTLCoBCfx5siPlv8N/ZEtVl4tIsJtxDAFmqGoqECUie4EOIhINlFfVNQAiMg0YiiULYy4rMj6SN395mx1/7IMkodvBAYSFDmDoiy0oV8bH0+GZIsqdrrP35/ExHxORkUA48LSqngQCcV45nBfrKkt3LV9cni0RGY3zKoQ6derkcdjGFG7pjnS+2vIVU5Z8TWLUOaom1qRP/Ai6PNGR6/sH2k1sc01yffJGRBqJyNLz9x5EpJWI/PUqj/ch0AAIBeKAf50/TDZ1NYfybKnqx6oapqphVatWvcoQjSl69p3Yx2PfP86HM6eQuO8cHQ724raAZxg5tT9dbwqyRGGumTvNUJ8AzwL/AVDVLSLyNVcxkKCqHj2/LCKfAAtcq7FA7SxVg3BOsBTrWr643BgDZGRmMHv7bD5dOpWTexIpn1SJvrF30PqeTtw0sgHeXpYkTN5wJ1mUVtX1F/1lknE1BxORmqoa51odBpzvKTUf+FpE3sF5gzsEWK+qDhFJFJFOwDpgJPD+1RzbmOLm4OmD/GvFv1i7LoK04+mExnWhvd8gek9uTeMmFT0dnilm3EkWx0WkAa7mHxG5DWcTUo5E5BugJ1BFRGKBl4GeIhLqeq1o4GEAVY0UkVnAdpyJaKyrJxTAozh7VgXgvLFtN7dNiZapmczfNZ///PYp8TtPUTaxAoOib6fRwA4MfrwRAb72rK3Je6J62VsAzgoi9YGPgS7ASSAKuFtVY/I/vKsXFham4eHhng7DmDx1JOkI76x6h1Vrw0k+kkrzo2F0yhhCl7+1pE17u09nrp2IbFTVsIvL3ekNtR/oKyJlAC9VTcyPAI0xl6eqLNq7iI9+/5jDkQn4nynDkKjh1O/aiZufb0KFMjauk8lf7kyrWhlnE1JXQEVkJfCaqibkd3DGGIg/G8+kNZNYvn4tZw+m0Oh4S7qeuZXQF5txfe9a1tPJFAh3GjdnAMuBW13rdwMzgb75FZQxxnk1sXj/Yj5c/hGx247jc9qPgVF3EdyiM4M+akr1ygGeDtGUIG49wa2qf8+y/rqIDM2neIwxOIcSf2/te/y+YTVJMck0ON6CbseH0eixJvQbWhcv6xJrCpg7yeI3ERkOzHKt3wb8mH8hGVNyqSq/Rv3Khys/4sDWo5Q66cdNUcOpE9KJAZOaEVTTpjo1nuFOsngYeAr40rXuDZwVkacAVdXy+RWcMSVJwrkEJq+fzG8bVpIYnUy9403ofvRW6o1uxE131rOpTo1HudMbqlxBBGJMSXX+auKjVR9xIDIeSSjFjVG3U7d2J/pOaUqDYPt7zHiePb1jjAcdP3ec99e9z/KNqzkTnUzwscb0PHILgSNDGDSqIb6l7GrCFA6WLIzxAFVlyf4l/Gf1fzgYeQyvBB/n1UTNjvT8tAmNQyp6OkRj/sSShTEF7MLVxKbVnIlKpt6xJvSIG0bNu0MY9EAD/H28PR2iMZdw56G8icAXqhpZAPEYU2ypKj/v+5lP1nzCocjjSIIP/aPuoE71DnT/tAlNG1X0dIjGXJY7VxY7gY9FpBTwBfCNqp7O37CMKV6OJh1l0rpJrInYcOFqouehW6h2TwMGP9DABv8zhZ47vaE+BT4VkcbA/cAWEVkFfKKqv+V3gMYUZarKgt0L+Hzt5xzecQLv474MiBpOYI12dPu0Mc2bVPJ0iMa4xa0/Z0TEG2ji+jkObAaeEpGHVXV4PsZnTJF16MwhJq2bxPqIP0iMTqZRfCuuPzKY6vfUZ9D9djVhihZ37lm8A9wMLAX+oarrXZv+V0R25WdwxhRFjkwH83bNY+q6aRzZfhKfhAAG77+HmrXb0P2zxnZvwhRJ7vxpsw34q6qey2ZbhzyOx5giLfpUNJPWTmLTlq0kxaTQ9EgbuhwbTPX7gxl0bwP8SllPJ1M0uZMsIoAmFw2DfBqIsRvdxjilO9KZFTmLrzfM4NjO0/gnlGXo/uFUq9+K3hMbE1KvgqdDNOaauJMs/g9oC2wBBGjhWq4sIo+o6i/5GJ8xhd6u47v495p/E7l1N2djU2gZ14mOJwZQ83/qMmBEfXsK2xQL7iSLaODB889ZiEgz4Fng78B3gCULUyKlZKTw1ZavmL3hOxJ2JVLuxHXctm8klVs054b3GhNc24ZVM8WHO8miSdYH8lR1u4i0UdX9NkOXKan+iPuD99e8z+5t0aQcTqft4W60PduPOk/U44ahdWyEWFPsuJMsdovIhzhnzAO401XmB6TnW2TGFEKJqYl8sukTFm36hZN7zlI5oTpD999GxU6NGTihMTWrlvZ0iMbkC3eSxShgDPAEznsWK4FncCaKXvkWmTGFiKqyPGY5/1n7H6IiD+OIV7ocvIGm2oOmr9anR59AmwvbFGs5JgvXw3g/qGpf4F/ZVEnKl6iMKUTiz8bzwfoPWBGxhtP7zxGYUI/eMbdQ6YYGDHqyMZXK+3k6RGPyXY7JQlUdInJORCpYN1lT0mRqJgt2L+CLtVM4vCMBSShF35hh1C/bmbb/bkj79tU8HaIxBcadZqgUYKuILAbOni9U1XE57SQinwODgHhVbeEqqwTMBIJx9rK6Q1VPura9ADwIOIBxqvqzq7wdMAUIABYC41VV3X6HxlyFqJNRvLf2PTZu3srZg8k0PNaCbnFDqHZHfQaObkAZPx9Ph2hMgXInWfzo+rlSU4DJwLQsZROApar6pohMcK0/7+qOOxxoDtQClohII1V1AB8Co4G1OJNFf2DRVcRjTK5SM1L5Zts3zFg3i4TdiQScLMfgqNsJrNuGHl80olHDip4O0RiPcGfU2akiEgDUUVW3x4JS1eUiEnxR8RCgp2t5KrAMeN5VPkNVU4EoEdkLdBCRaKC8qq4BEJFpwFAsWZh88EfcH7y/+n12bosi9Ug6reM6EXbmJuo9Up9+t9W17rCmRHNnIMHBwETAF6gnIqHAa6p681Ucr7qqxgGoapyInG/0DcR55XBerKss3bV8cfnlYh2N8yqEOnXqXEV4piQ6nXKajzd+zKLwxZzed47Kp6ozNOoWqnVsRv/nG1PDusMa41Yz1Cs4BwxcBqCqESJSL4/jyK7PoeZQni1V/Rj4GCAsLMzua5gc/Xce7E84sP0oegKuP3gjzb160uqNhnTqVsO6wxrj4k6yyFDV0xf9p7naL+KjIlLTdVVRE4h3lccCtbPUCwIOu8qDsik35pocPH2Qyesms3pjOOdiU6hzIoSesUOpNbQRN40Noay/3cA2Jiu3higXkbsAbxEJAcYBq6/yePNxPuT3puvfeVnKv3bNnVELCAHWu7ruJopIJ2AdMBJ4/yqPbQxpjjRmbpvJ9DUzOb77NH6nA7gpejjBtTvS4/NGhIRU9HSIxhRK7iSLx4EXgVTgG+BnnIMI5khEvsF5M7uKiMQCL+NMErNE5EHgAHA7gKpGisgsYDuQAYx19YQCeJT/dp1dhN3cNldpU9wmJq/6gO2b95F2PJ2WRzrQMXEAjcY0otewOnh7WZOTMZcjxfWRhbCwMA0PD/d0GKYQOJF8gk82fsKPaxaTFJNC5TPV6BMzjDrdWtP/mUZUqujv6RCNKTREZKOqhl1c7k5vqEY4x4IKzlpfVXvnZYDG5LVMzWThnoV8vOJzDu84jtcZL7rF9qdFmV50/ndjWrSp4ukQjSky3GmG+hb4CPgU59PVxhR6exL28P6qyazftIW0o2nUO9GE7seG0nhUM/rcHYyPTW9qzBVxtzfUh/keiTF54GzaWaZunsrM5XNJjE6m7Nny9I8eTkjbTtzwfghVq9kzE8ZcDXeSxQ8iMgaYi/MmNwCqeiLfojLmCqkqy6KXMfm3j4jZfhhNhLC4boSV6k+nfzalVUcb9M+Ya+HufBbgnEr1PAXq5304xly5A6cP8N7KyaxYv570+HRqnalLryO30PLuNvQaaU1OxuQFd8aGyuuntY3JE8npyUzfOp0vl84iMeYcAcml6XVgKKGte9L33cZUqRbg6RCNKTYumyxE5DlVfcu1fLuqfptl2z9U9S8FEaAxF1NVVh1cxbu/fED09lj0XCatjnais88guv2jJc3aWy8nY/JaTlcWw4G3XMsv4OwVdV5/wJKFKXAHTx9k0vIP+H3dGjKOZ1DjbBA944fR4Z6OdLvLRoY1Jr/klCzkMsvZrRuTr5LTk/nqj+lMWTKDc4eSCUgtTc/Ym+nU/gb6vd+I8hVtalNj8lNOyUIvs5zdujH5QlVZHrOciQs+IHbPYSRVaXW0A93KDaXvO60JblbR0yEaUyLklCxai8gZnFcRAa5lXOs2PoLJd9Gnonnr5/dYuzEcPeOgxtkgep++lZ6jrydsYC0bPtyYAnTZZKGq1t/QeERSWhKfrp3C9KVzyIhPIyC9NN3iBtC770B6jWmAr5+dmsYUNHeeszCmQGRqJgt3LeKdeR9x4uAJvDKgzZEu9Kl3Kze93ppKNawrrDGeYsnCFArb47fz2nfvsHPnTiRFqX2mHjd43c7Av3WjfrtKng7PmBLPkoXxqIRzCby18AN+Wb8YzmRSPq0CPU4NZshdg2h7SyBeXtYV1pjCwJKF8Yg0RxpTVs/gs5+/JC3hHKUyvQk71puh3YbT65HG+AbYqWlMYWL/I02BUlWW7l7Om9++x/G4I4gDQk62YHDdEQz5eyfKV7OOdsYURpYsTIHZeWwPr0x/h537tiDpUO1cDW70u4MRL/WnVvMKng7PGJMDSxYm351MPslrMyfz2+ZfkORMSmeUpnvyQO69/06a9alhz0sYUwRYsjD5Js2RxvsLv2LG71+TkZSMt3rR7kx37rlpJF3vaoxXKbt5bUxRYcnC5DlV5du1P/PBvI84c/oYKDRIasrw0FEMHdOFUqXtoTpjihpLFiZPrdz5B29+/R6xx3cjmVAtuQZDg+7i3tcGUtZuXhtTZFmyMHliT9xBXvtiElsPrUEcUCajDDeUGcrYp++lSkg5T4dnjLlGlizMNYk/c4o3PvuIlXsXkenIwEe96Sx9GffQgzTsGOjp8IwxecQjyUJEooFEwAFkqGqYiFQCZgLBQDRwh6qedNV/AXjQVX+cqv7sgbBNFkmpKUyc+iWLtnxLWsZZAFpltGfsLaNpf1NT6+FkTDHjySuLXqp6PMv6BGCpqr4pIhNc68+LSDOcs/Y1B2oBS0Skkao6Cj5kk57h4MNZ3zN79TQSM5y/vrqOBjzUYzQD7roe8bIkYUxxVJiaoYYAPV3LU4FlwPOu8hmqmgpEicheoAOwxgMxlliOTOXrH5bx5ZJPOJYeA0CVzGrc02YUd98/iFL+helUMsbkNU/9D1fgFxFR4D+q+jFQXVXjAFQ1TkSqueoGAmuz7BvrKruEiIwGRgPUqVMnv2IvUVSVeUs2MmX+J8SkbwOgrJZlWKM7GPPw3fiXtR5OxpQEnkoW16vqYVdCWCwiO3Oom127RrbTurqSzscAYWFhNvXrNVBVlqzcyefffsautDUo4IcvN9YZzFMP/w8VKpf3dIjGmALkkWShqodd/8aLyFyczUpHRaSm66qiJhDvqh4L1M6yexBwuEADLkFUlRXropny9RS2pi4jQzLwFi+61+zDM/c/Qq3aNTwdojHGAwo8WYhIGcBLVRNdyzcArwHzgVHAm65/57l2mQ98LSLv4LzBHQKsL+i4iztVZd3GQ0z5cjoR534h1TsFL28Iq9KJ50eNoWGDBp4O0RjjQZ64sqgOzHV1rSwFfK2qP4nIBmCWiDwIHABuB1DVSBGZBWwHMoCx1hMq76gqGzYd4atps9h0dhFnSyXiVQqaVW7BMyMeo03zlp4O0RhTCIhq8WzaDwsL0/DwcE+HUWipKhs3xTNjyveEJ/3Iab8EvATqVq7P07eMpUubjvashDElkIhsVNWwi8utv2MJo6ps2HCUuVMXEZ70A8cDjuLlD4GVgxh382j6te9tScIYcwlLFiVEZqaybs0RfvxyKeFnf+BomVi8SkP1ytV4dOCDDOp4E95eNhqsMSZ7liyKuQxHJmtXHGHxV8vYmLyAQ+Vi8CoHVapU5qEbRjKs8xB8vH08HaYxppCzZFFMpWVksmrJIZZ9s5KI9EUcKL8PrwpQqdp1PND3Lm7tdAv+peyBOmOMeyxZFDPJaRks/yGWtbPX8Ac/EVNhL15lhUrVKzKqz3Bu63ArpX1KezpMY0wRY8mimEhMSef3GTFsmr+eTX4/O5NEKaFSjQqM7HMnt7W7jTK+ZTwdpjGmiLJkUcSdOJXCsi+j2fbLBjaVXUJMtb14+QiVAytyb+87uKX1LZTzs8mHjDHXxpJFEXXocBIrvohm14pwNlZaysHA/Xj7CdVqV+LuXrczrMUwSxLGmDxjyaKI2b3zJOs+i2ZfRDgbqv3G4eAYvEt7UatuVYb3uJVhzYZR1resp8M0xhQzliyKAFXlj1XxREw9QFTUBtbX/JVjDePwLudNnXo1GNHtdgY3Gmz3JIwx+caSRSGWnuFg7fxD7Pr6IHtOryM88HdOhhynVKVSNGhQh7s638FNITdZF1hjTL6zZFEIJZ1JY9VXMcTMi2W71xo21lzB2epn8K3qS/PGDRkRdid96/fF19vX06EaY0oISxaFyJGoJNZ9EcPh3w6wueIqIuqsIS0glYCafnRo2oo7W99Bt7rdKOVlvzZjTMGybx0PU1X2rk4gYupBDm3bx6YaK9nRbBOZZZUygf50a96Z21vcTvta7W2AP2OMx1iy8BBHioNNcw6xb1Yc+0/uYGPNFexvvROvit5UrF2a3i26cUvTW2hatamnQzXGGEsWBe1MbDIbpx0k9qc4tvlt5I+aq4ivfQSfqj4E1q3CgOY3MqTxEALLB3o6VGOMucCSRQFQVQ6vOsnmr2I5FBHLxqqr2dJoHefKJeNfw5em9YMZ0nQwNzW8iQr+FTwdrjHGXMKSRT7KSMxg99wj7P72MDtO72Bz9TXsCY3EURHKBvrTIySMoY2H0LVOVxsm3BhTqFmyyGOqytmd59j69SGilx1mU+kNRFRfQ3zdo3hVLUWloHL0a9ydIY2H0KRKE7tpbYwpEixZ5BFHioP4xSfYPvMwkQd2sbHyGnY0jSC5bDp+1X1pFFyXoU0HcmPDG6lSuoqnwzXGmCtiyeIapRxMIea7o2xfGMMaWceWqus50vIwGRW8KFvLn/6NOjCo0QCur3O9PR9hjCmy7NvrKmRmZHJmzRl2zTrEml0bWV9+HXvqbiOldAZULkWt2jW4tWV/bmrYn9oVans6XGOMuWaWLK5A+ql0jv2YwPL5m1iZtpot123iTMPTpJUVfKv70qNxR+5oOYhOQZ1sKA5jTLFiycINKTEpbJgRwU+bfiU8YCPx1x3B4QvpFb0IqlOH4W0GMLBxP2qUreHpUI0xJl8UmWQhIv2BSYA38Kmqvpmfx1NVIjdHMu/bn1gbt45Yv8NQCdICwKdKefq27sHIdgNpXb2F9WgyxhR7RSJZiIg38AHQD4gFNojIfFXdnpfHSclIYf2h9azbt55ff1/JsWPHcWQq6gdaNoBGDcIYfv1N3Nysiz0XYYwpUYpEsgA6AHtVdT+AiMwAhgB5miyiT8bxzNxXSd+bAhlQNr08Vcu3oGOv7ozq2Zsa5W1yIWNMyVRUkkUgcDDLeizQ8eJKIjIaGA1Qp06dKz5IBe8aBG9oSq0TNQkq34x2Y9vTuXNN/H28rzJsY4wpHopKssjupoBeUqD6MfAxQFhY2CXbc1PxNDx0eBTlavnTYUpzvHy8rjxSY4wphopKsogFsj6wEAQczuuDBNQLoPeCtqTFp1miMMaYLIrKN+IGIERE6omILzAcmJ8fB/Iu401AvYD8eGljjCmyisSVhapmiMhjwM84u85+rqqRHg7LGGNKjCKRLABUdSGw0NNxGGNMSVRUmqGMMcZ4kCULY4wxubJkYYwxJleWLIwxxuTKkoUxxphcieoVP+hcJIjIMSDmKnevAhzPw3CKG/t8cmafT+7sM8qZJz+fuqpa9eLCYpssroWIhKtqmKfjKKzs88mZfT65s88oZ4Xx87FmKGOMMbmyZGGMMSZXliyy97GnAyjk7PPJmX0+ubPPKGeF7vOxexbGGGNyZVcWxhhjcmXJwhhjTK5KdLIQkf4isktE9orIhGy2i4i859q+RUTaeiJOT3Hj8+kpIqdFJML185In4vQUEflcROJFZNtltpf08ye3z6eknz+1ReQ3EdkhIpEiMj6bOoXnHFLVEvmDc16MfUB9wBfYDDS7qM4AYBHOaV07Aes8HXch+3x6Ags8HasHP6PuQFtg22W2l9jzx83Pp6SfPzWBtq7lcsDuwvwdVJKvLDoAe1V1v6qmATOAIRfVGQJMU6e1QEURqVnQgXqIO59Piaaqy4ETOVQpyeePO59Piaaqcaq6ybWcCOwAAi+qVmjOoZKcLAKBg1nWY7n0F+VOneLK3ffeWUQ2i8giEWleMKEVGSX5/HGXnT+AiAQDbYB1F20qNOdQkZkpLx9INmUX9yN2p05x5c5734RzHJkkERkAfA+E5HdgRUhJPn/cYecPICJlgTnAE6p65uLN2ezikXOoJF9ZxAK1s6wHAYevok5xlet7V9UzqprkWl4I+IhIlYILsdAryedPruz8ARHxwZkopqvqd9lUKTTnUElOFhuAEBGpJyK+wHBg/kV15gMjXT0SOgGnVTWuoAP1kFw/HxGpISLiWu6A83xKKPBIC6+SfP7kqqSfP673/hmwQ1XfuUy1QnMOldhmKFXNEJHHgJ9x9vz5XFUjReQR1/aPgIU4eyPsBc4B93sq3oLm5udzG/CoiGQAycBwdXXhKAlE5BucPXqqiEgs8DLgA3b+gFufT4k+f4DrgXuBrSIS4Sr7C1AHCt85ZMN9GGOMyVVJboYyxhjjJksWxhhjcmXJwhhjTK4sWRhjjMmVJQtjjDG5smRhjDEmV5YsjCmkRKSpiHwkIrNF5FFPx2NKNksWxlwDEQkWkeQsD1UhIuNccxRMv5bXVtUdqvoIcAcQ5nrtANfcD2klbWgM41kl9gluY/LQPlUNzbI+BrhJVaPc2VlEWgL/vKj4AVWNF5GbgQnAZABVTQZCRST6mqM25grYlYUxV0BExojINhGJEZHHs9n+Ec4Jo+aLyJMislNEPnXtM11E+orIKhHZ4xoPCVXdqqqDLvqJd22br6pdgLsL9I0acxG7sjDGTSJyK9AP57wDVYCtOGcxu0BVHxGR/kAvoCzwNnA7MBrn4Ix3AV2Bm3GOAzQ0h+P1BG4B/HCOEWSMx1iyMMZ944CHVDUdiBORdHK/Oo9S1a0AIhIJLFVVFZGtQHBOO6rqMmDZtQZtTF6wZihj3OCad6CVqu52rdcEjgNpueyammU5M8t6JvbHmilCLFkY455mQAURqS8iXjhvSL/v4ZiMKTCWLIxxTxtgOvANsAU4oKofezYkYwqOXQYb455QYIGqzsytoqoGuxaPAy2ylN+XZTk66zZjCju7sjDGPaFARDblDpzNU9lty3PnH8rDOeNcZkEc0xiwmfKMMca4wa4sjDHG5MqShTHGmFxZsjDGGJMrSxbGGGNyZcnCGGNMrixZGGOMyZUlC2OMMbmyZGGMMSZXliyMMcbk6v8BFZ4CfsQGepMAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.pyplot as plt\n", + "import sklearn.linear_model as skl\n", + "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organize the data into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "X = np.zeros((len(Density),4))\n", + "X[:,3] = Density**(4.0/3.0)\n", + "X[:,2] = Density\n", + "X[:,1] = Density**(2.0/3.0)\n", + "X[:,0] = 1\n", + "\n", + "# We use now Scikit-Learn's linear regressor and ridge regressor\n", + "# OLS part\n", + "clf = skl.LinearRegression().fit(X, Energies)\n", + "ytilde = clf.predict(X)\n", + "EoS['Eols'] = ytilde\n", + "# The mean squared error \n", + "print(\"Mean squared error: %.2f\" % mean_squared_error(Energies, ytilde))\n", + "# Explained variance score: 1 is perfect prediction \n", + "print('Variance score: %.2f' % r2_score(Energies, ytilde))\n", + "# Mean absolute error \n", + "print('Mean absolute error: %.2f' % mean_absolute_error(Energies, ytilde))\n", + "print(clf.coef_, clf.intercept_)\n", + "\n", + "# The Ridge regression with a hyperparameter lambda = 0.1\n", + "_lambda = 0.1\n", + "clf_ridge = skl.Ridge(alpha=_lambda).fit(X, Energies)\n", + "yridge = clf_ridge.predict(X)\n", + "EoS['Eridge'] = yridge\n", + "# The mean squared error \n", + "print(\"Mean squared error: %.2f\" % mean_squared_error(Energies, yridge))\n", + "# Explained variance score: 1 is perfect prediction \n", + "print('Variance score: %.2f' % r2_score(Energies, yridge))\n", + "# Mean absolute error \n", + "print('Mean absolute error: %.2f' % mean_absolute_error(Energies, yridge))\n", + "print(clf_ridge.coef_, clf_ridge.intercept_)\n", + "\n", + "fig, ax = plt.subplots()\n", + "ax.set_xlabel(r'$\\rho[\\mathrm{fm}^{-3}]$')\n", + "ax.set_ylabel(r'Energy per particle')\n", + "ax.plot(EoS['Density'], EoS['Energy'], alpha=0.7, lw=2,\n", + " label='Theoretical data')\n", + "ax.plot(EoS['Density'], EoS['Eols'], alpha=0.7, lw=2, c='m',\n", + " label='OLS')\n", + "ax.plot(EoS['Density'], EoS['Eridge'], alpha=0.7, lw=2, c='g',\n", + " label='Ridge $\\lambda = 0.1$')\n", + "ax.legend()\n", + "save_fig(\"EoSfitting\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The above simple polynomial in density $\\rho$ gives an excellent fit\n", + "to the data. \n", + "\n", + "We note also that there is a small deviation between the\n", + "standard OLS and the Ridge regression at higher densities. We discuss this in more detail\n", + "below.\n", + "\n", + "\n", + "## Splitting our Data in Training and Test data\n", + "\n", + "It is normal in essentially all Machine Learning studies to split the\n", + "data in a training set and a test set (sometimes also an additional\n", + "validation set). **Scikit-Learn** has an own function for this. There\n", + "is no explicit recipe for how much data should be included as training\n", + "data and say test data. An accepted rule of thumb is to use\n", + "approximately $2/3$ to $4/5$ of the data as training data. We will\n", + "postpone a discussion of this splitting to the end of these notes and\n", + "our discussion of the so-called **bias-variance** tradeoff. Here we\n", + "limit ourselves to repeat the above equation of state fitting example\n", + "but now splitting the data into a training set and a test set." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training R2\n", + "0.9999936913787534\n", + "Training MSE\n", + "2.4029973213885385\n", + "Test R2\n", + "0.9999140621935813\n", + "Test MSE\n", + "60.480439204729514\n" + ] + } + ], + "source": [ + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "def R2(y_data, y_model):\n", + " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organized into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "X = np.zeros((len(Density),5))\n", + "X[:,0] = 1\n", + "X[:,1] = Density**(2.0/3.0)\n", + "X[:,2] = Density\n", + "X[:,3] = Density**(4.0/3.0)\n", + "X[:,4] = Density**(5.0/3.0)\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)\n", + "# matrix inversion to find beta\n", + "beta = np.linalg.inv(X_train.T.dot(X_train)).dot(X_train.T).dot(y_train)\n", + "# and then make the prediction\n", + "ytilde = X_train @ beta\n", + "print(\"Training R2\")\n", + "print(R2(y_train,ytilde))\n", + "print(\"Training MSE\")\n", + "print(MSE(y_train,ytilde))\n", + "ypredict = X_test @ beta\n", + "print(\"Test R2\")\n", + "print(R2(y_test,ypredict))\n", + "print(\"Test MSE\")\n", + "print(MSE(y_test,ypredict))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## The Boston housing data example\n", + "\n", + "The Boston housing \n", + "data set was originally a part of UCI Machine Learning Repository\n", + "and has been removed now. The data set is now included in **Scikit-Learn**'s \n", + "library. There are 506 samples and 13 feature (predictor) variables\n", + "in this data set. The objective is to predict the value of prices of\n", + "the house using the features (predictors) listed here.\n", + "\n", + "The features/predictors are\n", + "1. CRIM: Per capita crime rate by town\n", + "\n", + "2. ZN: Proportion of residential land zoned for lots over 25000 square feet\n", + "\n", + "3. INDUS: Proportion of non-retail business acres per town\n", + "\n", + "4. CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n", + "\n", + "5. NOX: Nitric oxide concentration (parts per 10 million)\n", + "\n", + "6. RM: Average number of rooms per dwelling\n", + "\n", + "7. AGE: Proportion of owner-occupied units built prior to 1940\n", + "\n", + "8. DIS: Weighted distances to five Boston employment centers\n", + "\n", + "9. RAD: Index of accessibility to radial highways\n", + "\n", + "10. TAX: Full-value property tax rate per USD10000\n", + "\n", + "11. B: $1000(Bk - 0.63)^2$, where $Bk$ is the proportion of [people of African American descent] by town\n", + "\n", + "12. LSTAT: Percentage of lower status of the population\n", + "\n", + "13. MEDV: Median value of owner-occupied homes in USD 1000s\n", + "\n", + "## Housing data, the code\n", + "We start by importing the libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt \n", + "\n", + "import pandas as pd \n", + "import seaborn as sns" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and load the Boston Housing DataSet from **Scikit-Learn**" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import load_boston\n", + "\n", + "boston_dataset = load_boston()\n", + "\n", + "# boston_dataset is a dictionary\n", + "# let's check what it contains\n", + "boston_dataset.keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we invoke Pandas" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)\n", + "boston.head()\n", + "boston['MEDV'] = boston_dataset.target" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and preprocess the data" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# check for missing values in all the columns\n", + "boston.isnull().sum()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then visualize the data" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# set the size of the figure\n", + "sns.set(rc={'figure.figsize':(11.7,8.27)})\n", + "\n", + "# plot a histogram showing the distribution of the target values\n", + "sns.distplot(boston['MEDV'], bins=30)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is now useful to look at the correlation matrix" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# compute the pair wise correlation for all columns \n", + "correlation_matrix = boston.corr().round(2)\n", + "# use the heatmap function from seaborn to plot the correlation matrix\n", + "# annot = True to print the values inside the square\n", + "sns.heatmap(data=correlation_matrix, annot=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the above coorelation plot we can see that **MEDV** is strongly correlated to **LSTAT** and **RM**. We see also that **RAD** and **TAX** are stronly correlated, but we don't include this in our features together to avoid multi-colinearity" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(20, 5))\n", + "\n", + "features = ['LSTAT', 'RM']\n", + "target = boston['MEDV']\n", + "\n", + "for i, col in enumerate(features):\n", + " plt.subplot(1, len(features) , i+1)\n", + " x = boston[col]\n", + " y = target\n", + " plt.scatter(x, y, marker='o')\n", + " plt.title(col)\n", + " plt.xlabel(col)\n", + " plt.ylabel('MEDV')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we start training our model" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "X = pd.DataFrame(np.c_[boston['LSTAT'], boston['RM']], columns = ['LSTAT','RM'])\n", + "Y = boston['MEDV']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We split the data into training and test sets" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "# splits the training and test data set in 80% : 20%\n", + "# assign random_state to any value.This ensures consistency.\n", + "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state=5)\n", + "print(X_train.shape)\n", + "print(X_test.shape)\n", + "print(Y_train.shape)\n", + "print(Y_test.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we use the linear regression functionality from **Scikit-Learn**" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.metrics import mean_squared_error, r2_score\n", + "\n", + "lin_model = LinearRegression()\n", + "lin_model.fit(X_train, Y_train)\n", + "\n", + "# model evaluation for training set\n", + "\n", + "y_train_predict = lin_model.predict(X_train)\n", + "rmse = (np.sqrt(mean_squared_error(Y_train, y_train_predict)))\n", + "r2 = r2_score(Y_train, y_train_predict)\n", + "\n", + "print(\"The model performance for training set\")\n", + "print(\"--------------------------------------\")\n", + "print('RMSE is {}'.format(rmse))\n", + "print('R2 score is {}'.format(r2))\n", + "print(\"\\n\")\n", + "\n", + "# model evaluation for testing set\n", + "\n", + "y_test_predict = lin_model.predict(X_test)\n", + "# root mean square error of the model\n", + "rmse = (np.sqrt(mean_squared_error(Y_test, y_test_predict)))\n", + "\n", + "# r-squared score of the model\n", + "r2 = r2_score(Y_test, y_test_predict)\n", + "\n", + "print(\"The model performance for testing set\")\n", + "print(\"--------------------------------------\")\n", + "print('RMSE is {}'.format(rmse))\n", + "print('R2 score is {}'.format(r2))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# plotting the y_test vs y_pred\n", + "# ideally should have been a straight line\n", + "plt.scatter(Y_test, y_test_predict)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reducing the number of degrees of freedom, overarching view\n", + "\n", + "Many Machine Learning problems involve thousands or even millions of\n", + "features for each training instance. Not only does this make training\n", + "extremely slow, it can also make it much harder to find a good\n", + "solution, as we will see. This problem is often referred to as the\n", + "curse of dimensionality. Fortunately, in real-world problems, it is\n", + "often possible to reduce the number of features considerably, turning\n", + "an intractable problem into a tractable one.\n", + "\n", + "Later we will discuss some of the most popular dimensionality reduction\n", + "techniques: the principal component analysis (PCA), Kernel PCA, and\n", + "Locally Linear Embedding (LLE). \n", + "\n", + "\n", + "Principal component analysis and its various variants deal with the\n", + "problem of fitting a low-dimensional [affine\n", + "subspace](https://en.wikipedia.org/wiki/Affine_space) to a set of of\n", + "data points in a high-dimensional space. With its family of methods it\n", + "is one of the most used tools in data modeling, compression and\n", + "visualization.\n", + "\n", + "\n", + "\n", + "\n", + "## Preprocessing our data\n", + "\n", + "Before we proceed however, we will discuss how to preprocess our\n", + "data. Till now and in connection with our previous examples we have\n", + "not met so many cases where we are too sensitive to the scaling of our\n", + "data. Normally the data may need a rescaling and/or may be sensitive\n", + "to extreme values. Scaling the data renders our inputs much more\n", + "suitable for the algorithms we want to employ.\n", + "\n", + "**Scikit-Learn** has several functions which allow us to rescale the\n", + "data, normally resulting in much better results in terms of various\n", + "accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n", + "ensures that for each feature/predictor we study the mean value is\n", + "zero and the variance is one (every column in the design/feature\n", + "matrix). This scaling has the drawback that it does not ensure that\n", + "we have a particular maximum or minimum in our data set. Another\n", + "function included in **Scikit-Learn** is the **MinMaxScaler** which\n", + "ensures that all features are exactly between $0$ and $1$. The\n", + "\n", + "## More preprocessing\n", + "\n", + "\n", + "The **Normalizer** scales each data\n", + "point such that the feature vector has a euclidean length of one. In other words, it\n", + "projects a data point on the circle (or sphere in the case of higher dimensions) with a\n", + "radius of 1. This means every data point is scaled by a different number (by the\n", + "inverse of it’s length).\n", + "This normalization is often used when only the direction (or angle) of the data matters,\n", + "not the length of the feature vector.\n", + "\n", + "The **RobustScaler** works similarly to the StandardScaler in that it\n", + "ensures statistical properties for each feature that guarantee that\n", + "they are on the same scale. However, the RobustScaler uses the median\n", + "and quartiles, instead of mean and variance. This makes the\n", + "RobustScaler ignore data points that are very different from the rest\n", + "(like measurement errors). These odd data points are also called\n", + "outliers, and might often lead to trouble for other scaling\n", + "techniques.\n", + "\n", + "\n", + "\n", + "## Simple preprocessing examples, Franke function and regression" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import sklearn.linear_model as skl\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "\n", + "def FrankeFunction(x,y):\n", + "\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n", + "\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n", + "\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n", + "\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n", + "\treturn term1 + term2 + term3 + term4\n", + "\n", + "\n", + "def create_X(x, y, n ):\n", + "\tif len(x.shape) > 1:\n", + "\t\tx = np.ravel(x)\n", + "\t\ty = np.ravel(y)\n", + "\n", + "\tN = len(x)\n", + "\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n", + "\tX = np.ones((N,l))\n", + "\n", + "\tfor i in range(1,n+1):\n", + "\t\tq = int((i)*(i+1)/2)\n", + "\t\tfor k in range(i+1):\n", + "\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n", + "\n", + "\treturn X\n", + "\n", + "\n", + "# Making meshgrid of datapoints and compute Franke's function\n", + "n = 5\n", + "N = 1000\n", + "x = np.sort(np.random.uniform(0, 1, N))\n", + "y = np.sort(np.random.uniform(0, 1, N))\n", + "z = FrankeFunction(x, y)\n", + "X = create_X(x, y, n=n) \n", + "# split in training and test data\n", + "X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)\n", + "\n", + "\n", + "clf = skl.LinearRegression().fit(X_train, y_train)\n", + "\n", + "# The mean squared error and R2 score\n", + "print(\"MSE before scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test), y_test)))\n", + "print(\"R2 score before scaling {:.2f}\".format(clf.score(X_test,y_test)))\n", + "\n", + "scaler = StandardScaler()\n", + "scaler.fit(X_train)\n", + "X_train_scaled = scaler.transform(X_train)\n", + "X_test_scaled = scaler.transform(X_test)\n", + "\n", + "print(\"Feature min values before scaling:\\n {}\".format(X_train.min(axis=0)))\n", + "print(\"Feature max values before scaling:\\n {}\".format(X_train.max(axis=0)))\n", + "\n", + "print(\"Feature min values after scaling:\\n {}\".format(X_train_scaled.min(axis=0)))\n", + "print(\"Feature max values after scaling:\\n {}\".format(X_train_scaled.max(axis=0)))\n", + "\n", + "clf = skl.LinearRegression().fit(X_train_scaled, y_train)\n", + "\n", + "\n", + "print(\"MSE after scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))\n", + "print(\"R2 score for scaled data: {:.2f}\".format(clf.score(X_test_scaled,y_test)))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Friday August 28\n", + "\n", + "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK4155/h20/forelesningsvideoer/LectureAug28.mp4?vrtx=view-as-webpage) and [handwritten notes](https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/NotesAugust28.pdf)\n", + "\n", + "More material will be added here, see handwritten notes also." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/doc/pub/week35/ipynb/Results/FigureFiles/EoSfitting.png b/doc/pub/week35/ipynb/Results/FigureFiles/EoSfitting.png new file mode 100644 index 000000000..c8fb46b2c Binary files /dev/null and b/doc/pub/week35/ipynb/Results/FigureFiles/EoSfitting.png differ diff --git a/doc/pub/week35/ipynb/Results/FigureFiles/Masses2016OLS.png b/doc/pub/week35/ipynb/Results/FigureFiles/Masses2016OLS.png new file mode 100644 index 000000000..bd56e3e94 Binary files /dev/null and b/doc/pub/week35/ipynb/Results/FigureFiles/Masses2016OLS.png differ