\documentclass[amssymb,twocolumn,aps]{revtex4}

% allows special characters (including æøå)
\usepackage[utf8]{inputenc}
%\usepackage [norsk]{babel} %if you write norwegian
\usepackage[english]{babel}  %if you write english

\usepackage{amsmath,amssymb}  % mathematical symbols (physics imports amsmath)
\usepackage{tikz}
\usepackage{graphicx}         % include graphics such as plots
\usepackage{xcolor}
\usepackage{hyperref}         % automagic cross-referencing 
\usepackage{float}			  % force placement of tables and figures
\usepackage[noabbrev,nameinlink]{cleveref}
\usepackage{siunitx}  

\usepackage{listofitems} % for \readlist to create arrays
\usetikzlibrary{arrows.meta,decorations.pathreplacing} % for arrow size
\usepackage[outline]{contour} % glow around text
\contourlength{1.4pt}

\definecolor{myred}{HTML}{FF220C}  
\definecolor{myblue}{HTML}{70D6FF}  
\definecolor{mygreen}{HTML}{8AAA79}  
\definecolor{mydarkred}{HTML}{666370} 
\definecolor{mydarkblue}{HTML}{1C1F33} 
\colorlet{myorange}{orange!70!red!60!black}
\colorlet{mydarkgreen}{green!30!black}
\DeclareSIUnit{\std}{\sigma}

\tikzset{
  >=latex, % for default LaTeX arrow head
  node/.style={thick,circle,draw=myblue,minimum size=22,inner sep=0.5,outer sep=0.6},
  node in/.style={node,green!20!black,draw=mygreen!30!black,fill=mygreen!75},
  node hidden/.style={node,blue!20!black,draw=myblue!30!black,fill=myblue!80},
  node convol/.style={node,orange!20!black,draw=myorange!30!black,fill=myorange!50},
  node out/.style={node,red!20!black,draw=myred!30!black,fill=myred!80},
  connect/.style={thick,mydarkblue}, %,line cap=round
  connect arrow/.style={-{Latex[length=4,width=3.5]},thick,mydarkblue,shorten <=0.5,shorten >=1},
  node 1/.style={node in}, % node styles, numbered for easy mapping with \nstyle
  node 2/.style={node hidden},
  node 3/.style={node out}
}
\def\nstyle{int(\lay<\Nnodlen?min(2,\lay):3)}

\newcommand{\pred}{\hat y}
\newcommand{\true}{y}

\begin{document}
	
\title{Using Feedforward Neural Networks for Breast Cancer Detection on Few Features}
\date{\today}               
\author{Lars Bogner}
\affiliation{University of Oslo}

\newpage
	
\begin{abstract}
TODO
\end{abstract}

\maketitle

\tableofcontents

\section{Introduction}
Sampling the cell nuclei of tissue extracted using a fine needle aspirate from the breast allows for a very easy separation between benign and malignant tissue. While very good performance is already shown with straightforward approaches, like decision trees \cite{bennettDecisionTreeConstruction1992}, we try to improve the efficiency of the detection process in this paper.

Using a very small subset of features which can be obtained using basic image recognition, we enhance them to a larger set of nuclei features using a regression model, based on which the straightforward classification can be continued using the large set of features.

Starting from the features of radius and area from the Wisconsin Breast Cancer Diagnostic dataset \cite{williamwolbergBreastCancerWisconsin1993} we obtain the full set of features as presented in the dataset using an optimized regression model. Using this regression model it will be possible in the future to use standard classifiers while only needing to measure the radius and area of the cell nuclei in diagnosis. In \cref{section:methods} we present the methods necessary to implement the corresponding neural networks and how to optimize them. The results and a discussion thereof is presented in \cref{section:results}. Finally, our findings are concluded in \cref{section:conclusion}.
    
\section{Methods}\label{section:methods}

\subsection{Neural Networks}

\begin{figure}
    \newcommand\setAngles[3]{
  \pgfmathanglebetweenpoints{\pgfpointanchor{#2}{center}}{\pgfpointanchor{#1}{center}}
  \pgfmathsetmacro\angmin{\pgfmathresult}
  \pgfmathanglebetweenpoints{\pgfpointanchor{#2}{center}}{\pgfpointanchor{#3}{center}}
  \pgfmathsetmacro\angmax{\pgfmathresult}
  \pgfmathsetmacro\dang{\angmax-\angmin}
  \pgfmathsetmacro\dang{\dang<0?\dang+360:\dang}
}
\begin{tikzpicture}[x=2.2cm,y=1.4cm]
  \readlist\Nnod{3,4,4,2} % array of number of nodes per layer
  
  \foreachitem \N \in \Nnod{ % loop over layers
    \def\lay{\Ncnt} % alias of index of current layer
    \pgfmathsetmacro\prev{int(\Ncnt-1)} % number of previous layer
    \message{^^J Layer \lay, N=\N, prev=\prev ->}
    
    % NODES
    \foreach \i [evaluate={\y=\N/2-\i; \x=\lay; \n=\nstyle;}] in {1,...,\N}{ % loop over nodes
      \message{N\lay-\i, }
      \node[node \n] (N\lay-\i) at (\x,\y) {$a_\i^{(\prev)}$};
    }
    
    % CONNECTIONS
    \foreach \i in {1,...,\N}{ % loop over nodes
      \ifnum\lay>1 % connect to previous layer
        \setAngles{N\prev-1}{N\lay-\i}{N\prev-\Nnod[\prev]} % angles in current node
        %\draw[red,thick] (N\lay-\i)++(\angmin:0.2) --++ (\angmin:-0.5) node[right,scale=0.5] {\dang};
        %\draw[blue,thick] (N\lay-\i)++(\angmax:0.2) --++ (\angmax:-0.5) node[right,scale=0.5] {\angmin, \angmax};
        \foreach \j [evaluate={\ang=\angmin+\dang*(\j-1)/(\Nnod[\prev]-1);}] %-180+(\angmax-\angmin)*\j/\Nnod[\prev]
                    in {1,...,\Nnod[\prev]}{ % loop over nodes in previous layer
          \setAngles{N\lay-1}{N\prev-\j}{N\lay-\N} % angles out from previous node
          \pgfmathsetmacro\angout{\angmin+(\dang-360)*(\i-1)/(\N-1)} % number of previous layer
          %\draw[connect arrow,white,line width=1.1] (N\prev-\j.{\angout}) -- (N\lay-\i.{\ang});
          \draw[connect arrow] (N\prev-\j.{\angout}) -- (N\lay-\i.{\ang}); % connect arrows uniformly
        }
      \fi % else: nothing to connect first layer
    }
  }
  
  % LABELS
  \node[above=5,align=center,mygreen!60!black] at (N1-1.90) {input\\[-0.2em]layer};
  \node[above=2,align=center,myblue!60!black] at (N2-1.90) {hidden layers};
  \node[above=8,align=center,myred!60!black] at (N\Nnodlen-1.90) {output\\[-0.2em]layer};
  
\end{tikzpicture}
\caption{Visualization of a neural network, modified work of \cite{NeuralNetworksTikZnet2024}.}
\label{fig:neural_network}
\end{figure}

Neural networks are the continuation of a linear combination by introducing nonlinearity. While linear regression uses a linear combination of input features to create an output, neural networks add a nonlinear function to the output of the linear combination. This is the entire secret behind the powerful possibilities neural networks have shown over the last years.

A feedforward neural network (FFNN) uses a layered structure as shown in \cref{fig:neural_network}, where each node represents a separate linear combination of it's inputs. A general node uses the inputs $x_i$ to create a weighted sum $z$, which is then activated using a so called activation function $g(z)$. More detail on the activation function is provided in \cref{subsec:activation}. The output of the general node can then be calculated as
\begin{equation}
    y = g(z) = g\left(\sum_i (w_i x_i) + b\right),
\end{equation}
where $b$ is the so-called bias of the node which in turn leads to a shift of the weighted sum, irrespective of the inputs. The linear combination $z = \sum_i (w_i x_i) + b$, can also be rewritten as a vector-vector multiplication of vectors $\vec a_{j}^{(k)} = (b, \vec{w})$ and $\vec X = (1, \vec x)$. This leads to a simple expression for the output of a node of $y = g(\vec a \cdot \vec X)$. All parameters of a single node in layer $k$ and position $j$ in the layer are contained within $\vec a_{j}^{(k)}$.

We use the term layer, as the output of each layer will be used as the input of the following layer, thus presenting a clear hierarchy. This hierachy is also useful for calculating the gradient of the final output of the neural network, with respect to all the weights in the layers, as we can use chained differentiation. Calculating the gradient with respect to all the weights is necessary as the optimization techniques presented in \cref{subsec:optimization} rely on the knowledge of the gradients to find the optimal set of weights. The gradient of a single node given the input $\vec{x}$ is given by
\begin{equation}
    \frac{\partial y}{\partial \vec a} = \vec{X} g'(\vec a \vec X),
\end{equation}
where $g'$ is the derivative of the activation function with respect to it's input. Then given the input of a node in the previous layer we can calculate the gradient with respect to it's weights using the chain rule of derivatives.

\subsection{Activation Functions} \label{subsec:activation}
Activation functions provied a source of nonlinearity in the otherwise linear combination of inputs of a neural network. This nonlinearity is key, as it provides the basis for the famous Universal Approximation theorem, which states, that FFNN can approximate any arbitrary real function given a large enough number of nodes \cite{elstnerLectureMachineLearning2025}. While one might come to the conclusion, that if this nonlinearity is of uttermost importance, the right choice of activation is complicated, it is proven by many practical applications, that already very simple activation functions may provide superb performance.

\paragraph{Rectified Linear Unit}
For simplicity this paper will therefore focus on a set of three activation functions. A very simple non-linear function is provided by the Rectified Linear Unit (ReLU), which shall be defined as
\begin{equation}
    \mathrm{ReLU}(z) = \begin{cases}
        0 & z < 0\\ z & z \geq 0
    \end{cases}.
\end{equation}
While this function provides all necessary properties of an activation function, we will provide an explanation, why it is not well suited for the optimization procedure of FFNN. The issue with the ReLU is that for $z < 0$ the gradient of the function vanishes. This poses a problem, as mentioned above, the optimization algorithm relies on the gradient of the output to adjust and optimize the weights of the neural network. A vanishing gradient disables a possible optimization. This problem is further amplified by the fact, that the gradient of weights in the first layers of the network depends on the product of the gradients with the previous layers, thus a vanishing gradient in one of the last layers has adverse effects on the optimization of weights in the previous layers.

\paragraph{Leaky Rectified Linear Unit}
As the ReLU is otherwise quite optimal due to its simplicity, we will mailny use a slightly modified version of the ReLU for the majority of nodes in the studied neural networks, namely the Leaky ReLU (LReLU). The leaky ReLU modifies the standard ReLU by introducing a very small but non-vanishing gradient for $z < 0$. It is defined as
\begin{equation}
    \mathrm{LReLU}(z) = \begin{cases}
        \epsilon z & z < 0\\ z & z \geq 0
    \end{cases},
\end{equation}
with $\epsilon$ being the leak coefficient, a small number, set to $\epsilon = \num{e-5}$ for all studies. Due to the step in the derivative at $z = 0$ our function is still nonlinear, but providing a positive derivative over the whole domain.

\paragraph{Softmax}
The last activation to introduce for our studies will be the Softmax function. It shall be defined as
\begin{equation}
    \mathrm{Softmax}(z_{j}^{(k)}) = \frac{\exp z_{j}^{(k)}}{\sum_{j'} \exp z_{j'}^{(k)}}.
\end{equation}
Additionally, to providing a source of nonlinearity, the softmax function has the special property, that the total output of a layer, i.e. the sum of its outputs is regularized to a value of 1. This property is especially useful in the case, where it is used as a classifier, where each of the layers outputs corresponds to a probability of being memeber in a certain class. While there is no gurantee, that the output of such a layer corresponds to a frequency in the frequentist interpretation, it is very useful in combination with the Cross Entropy introduced in the next section, as we have a nice behaving gradient, useful for optimization.


\subsection{Loss Functions}
The function of the loss function is to quantize the discrepancy between the prediction of a numerical model and the true values. It is also sometimes called cost function. It must be a differentiable function which is minimal when, the prediction aligns with the true values. The choice of loss function needs to be specific to the task for which the model should be optimized for. For the regressional part of this paper we will use the mean squared error (MSE) as a loss function. The MSE is defined using the prediction $\pred$ and the true values $\true$ as
\begin{equation}
    \mathrm{MSE} = \frac{\sum_i (\pred_i - \true_i)^2}{N}.
\end{equation}
Its derivative is therefore defined as
\begin{equation}
    \frac{\partial \mathrm{MSE}}{\partial \pred_i} = \frac{2 (\pred_i - \true_i)}{N}.
\end{equation}
The advantage of the MSE as a loss function is, that the loss value can be directly interpreted in the context of the metric on which we optimize. Furthermore, it gives the advantage, that compared to the mean absolute error, which uses the absolute deviation between the target and the prediction, that it is continuous differentiable and that large deviations are penalized more than small deviations. Especially in the context of small measurement uncertainties this is important.

The second loss function used is the cross entropy loss. It is used for the case of classification tasks and is defined as
\begin{equation}
    \mathrm{CE} = \sum_i \true_i \log \pred_i.
\end{equation}
The cross entropy loss is inspired by the definition of the physical entropy of stochastic systems and is useful, as it is minimal when the probabilities $\pred$ and $\true$ align. Combined with the softmax activation function from \cref{subsec:activation}, the derivative simplifies to a proportional relation with respect to $\pred - \true$.


\subsection{Regularization}
As the number of parameters in a neural network grows very quickly, special attention needs to be payed to avoid overfitting the training dataset. Overfitting means, that the available degress of freedom of the model will be used to represent all the datapoints in the training dataset perfectly. This will lead to very poor generalization ability of the model. This phenomenon is part of the bias-variance trade-off, which is ubiquitous in machine learning. With increasing numbers of parameters in the model, i.e. due to an increase in layer size and/or increase in the number of layers, the number of degrees of freedom also increases. With a larger number of degrees of freedom the variance of the model increases. If there are too few parameters, the bias of the model will increase. The goal is therefore to find the middle point, where neither the bias nor variance are increased. 

A useful tool to control the bias variance trade-off is regularization in neural networks. Using regularization we can decrease the effective number of degrees of freedom, therefore decreasing the tendency to overfit the data. The decrease is thereby controlled by the regularization coefficient $\lambda$. In the limit of $\lambda \to 0$, the solution approaches the unregularized model, while there is a steady decrease of the number of effective degrees of freedom with an increase of $\lambda$. Regularization works by adding a term to the cost function of the form
\begin{equation} \label{eq:regularization}
    C_\text{regularized} = C + \lambda \sum_k \sum_j \lVert a_j^{(k)}\rVert_n^n.
\end{equation}
\Cref{eq:regularization} is the generalized expression for a L$n$-regularization, where $n$ is the degree of the norm in \cref{eq:regularization}. The most common types of regularization are L1- and L2-regularization. Using a 1-norm for regularization leads to the benefit that it has the tendency to force an ever-increasing number of parameters of the model towards zero. It thus effecitvely reduces the number of weights that influence the final result. Especially in highly optimized architectures this may be advantageous, as it reduces the number of computations needed for each evaluation. On the other hand it is known to produce less stable results, as the L2-regularization. 

\subsection{Optimization Algorithms} \label{subsec:optimization}
The goal of an optimization algorithm is to tune the weights of a FFNN in a way, so that the cost function will be minimized, i.e. the predictions and the targets align. Most algorithms use a way of gradient descent to update the weights and find the optimum via iterative optimization. Gradient descent uses the derivatives of the cost function
\begin{equation}
    \delta_j^{(k)} \equiv \frac{\partial C}{\partial a_j^{(k)}}
\end{equation}
and update the weights given a learning rate $\alpha$ using
\begin{equation}
    a_j^{(k)} \to a_j^{(k)} - \alpha \cdot \delta_j^{(k)}.
\end{equation}
To decrease the computational resources needed we will use a more advanced version of this algorithm in this paper. The Adam optimizer updates the weights according to
\begin{equation}
    \begin{aligned}
        m_j^{(k)} &\to \beta_1 m_j^{(k)} + (1 - \beta_1) \delta_j^{(k)}, \\
        v_j^{(k)} &\to \beta_2 v_j^{(k)} + (1 - \beta_2) (\delta_j^{(k)})^2, \\
        \hat{m}_j^{(k)} & = \frac{m_j^{(k)}}{1 - \beta_1^t}, \\
        \hat{v}_j^{(k)} & = \frac{v_j^{(k)}}{1 - \beta_2^t}, \\
        a_j^{(k)} &\to a_j^{(k)} - \alpha \frac{\hat{m}_j^{(k)}}{\sqrt{\hat{v}_j^{(k)}} + \epsilon},
    \end{aligned}
\end{equation}
with the optimization parameters $\beta_{1/2}$. As shown in our previous article \cite{bognerRegularizationOptimizationAll2025}, Adam tends to converge faster towards an optimal solution. It is also thus widely used in machine learning optimizations and will be used for all training procedures throughout the studies.

\subsection{Data Splitting Techniques}
\subsubsection{Train-Test Splitting}
As we mentioned before, there is the possibility for our model to overadjust for the datapoints present during the optimization process. To still get an accurate measure for the models performance we will only use a subset of the dataset for the process of training. A random subset of \qty{20}{\percent} is reserved for testing the model only. This means, that all performance metrics are evaluated in the same way as new unseen data would perform. Due to the random nature of the subset, the performance between runs may differ stochastically. This may be counteracted by redoing the tests with different choices of training and testing datasets and averaging the metrics.

\subsubsection{Out-of-Fold prediction}
\begin{figure}
    \centering
    \begin{tikzpicture}[scale=0.8]
  % parameters
  \def\nfolds{5}
  \def\squaresize{0.8}
  \def\vspace{0.15} % extra space between rows (relative to squaresize)

  % loop over folds
  \foreach \row in {0,...,4} {
    % vertical position with extra spacing
    \pgfmathsetmacro{\y}{-(\row*(\squaresize+\vspace))}
    \foreach \col in {0,...,4} {
      % decide color
      \ifnum\row=\col
        \fill[myblue!80] (\col*\squaresize, \y) 
             rectangle ++(\squaresize,\squaresize);
      \else
        \fill[myred!75] (\col*\squaresize, \y) 
             rectangle ++(\squaresize,\squaresize);
      \fi
      % draw border
      \draw[black,thick] (\col*\squaresize, \y) 
           rectangle ++(\squaresize,\squaresize);
    }
    % label fold on the left
    \node[left] at (-0.2, \y+0.5*\squaresize) {Fold \the\numexpr\row+1\relax};
    % MSE_i label on the right
    \node[right] at (5*\squaresize+0.2, \y+0.5*\squaresize) {$\pred_{\the\numexpr\row+1\relax}$};
  }

  % curly brace summarizing MSEs
   % compute top and bottom y positions for the brace
  \pgfmathsetmacro{\ytop}{0.5*\squaresize}
  \pgfmathsetmacro{\ybottom}{-(4*(\squaresize+\vspace))+0.5*\squaresize}

  % curly brace summarizing MSEs
  \draw[decorate,decoration={brace,amplitude=10pt}] 
    (6.8*\squaresize,\ytop) -- 
    (6.8*\squaresize,\ybottom)
    node[midway,xshift=1.4cm]{$\{\pred_1, \dots, \pred_5\}$};
\end{tikzpicture}
    \caption{Illustration of predicting the full dataset using out-of-fold prediction to avoid contamination of training data. Modified from \cite{bognerRegularizationOptimizationAll2025}.  Red represents the training dataset, while the features from the blue part are used to make the prediction. Finally, all predictions are combined into one dataset of the same size as the input dataset.}
    \label{fig:outoffold}
\end{figure}
A further hindrance in our approach with using the output of a regression model to make classifications is, that for an efficient workflow, we need a full but independent prediction of the dataset. To achieve this, we use a workflow inspired by cross-validation. In cross-validation you use multiple folds of the dataset, whereby for every fold, a different part of the dataset is used as testing data, and the rest of the dataset is used for training purposes. This provides independance of training and testing data while being able to test the models general performance for the entire dataset. For out-of-fold prediction, we use the same approach of changing the training data to leave out a specific part each time, but furthermore we use the model trained on each training set, to make predictions for the remaining set. Combining all the predicted sets into one set, allows for having independent predictions for the entire dataset. Having the predictions be independent is important, as otherwise no statement can be made on the generalization ability of the workflow. Using this approach we can expect the same performance on new and yet unseen data.

\subsection{Implementation}
All the code necessary to use a FFNN is implemented from scratch in \texttt{Python3} as the \texttt{easynn} library. The source code therefore is located in the \texttt{/src/easynn/} folder of the accompanying git repostiory. It is separated into two main modules, called \texttt{schedulers} and \texttt{feedforward}. The scheduler modules use modified versions of our previous work on using gradient descent techniques for ordinary least squares techniques. It is also structured in a modular way, so that scheduling tasks like early stopping and dynamic learning rate adjustment are easy to implement in further versions. The base scheduler implements an update method of signature \texttt{def update(self, gradients: gradient\_\-type) -> gradient\_\-type:}, with \texttt{gradient\_\-type = List[\-Tuple[\-np.ndarray, np.ndarray]]
}. This can be overwritten to implement any optimization algorithm. The method takes the gradients of the cost functions with respect to the weights and biases of each layer. Each layer is assigned a tuple of two arrays of the gradients. It should then return the updates for each of the weights and biases in the same format. The training process which fetches new updates after each evaluation of the loss and gradients will continue will the \texttt{cont} property of the base class is true.

The main structures of the neural network are implemented as classes in the \texttt{feedforward} module. Again it is structured in a way, where it is easy to be extended in the future. The FFNN can be constructed using the arguments \texttt{class FFNN: def \_\_init\_\_(self, layers: List[Layer], scheduler: Scheduler, loss\_fn: LossFunction)}. The layers are constructed using \texttt{class Layer: def \_\_init\_\_(self, input\_dim: int, num\_nodes: int, activation\_function: ActivationFunction, regularization: Optional[Regularization] = None)}. Using this structure it is easy to control the hyperparameters of each hidden layer carefully and for example choose the activation and regularization of each layer precisely. The weights and biases of each layer are initialized from a uniform random distribution in the bounds $[-s, s]$ with 
\begin{equation}
    s = \frac{1}{\sqrt{I \cdot N}},
\end{equation}
where $I$ is the input dimension and $N$ is the number of nodes. This initialization is taken from the standard implementation in the widely used \texttt{pyTorch} library.

For the entire numerical computation of the neural network, the \texttt{numpy} \cite{harrisArrayProgrammingNumPy2020} library is used. Numpy is well-known for its performance in linear algebra performance and vectorization of array computations. Other libraries are solely used for plotting purposes. The libraries used during the plotting procedure include \texttt{pandas} \cite{teamPandasdevPandasPandas2025}, \texttt{scikit-learn}\footnote{Solely used for data splitting and performance metrics.} \cite{pedregosaScikitlearnMachineLearning2011}, \texttt{matplotlib} \cite{hunterMatplotlib2DGraphics2007} and \texttt{seaborn} \cite{Waskom2021}.

To check the functionality of the developed code, in a first stage, the loss history is evaluated visually showing a clear convergence after a high enough number of epochs. Furthermore, the quality of the models is evaluated using toy-datasets created using \texttt{scikit-learn}. In detail, both a regression toy-dataset as well as a classification dataset are validated. A MSE of \num{0.000631} on a dataset created from \texttt{make\_regression(n\_samples=1000, n\_features=5, n\_informative=3, noise=0.1)} is achieved. For classification the \texttt{make\_classification(n\_samples=1000, n\_features=5, n\_informative=3, n\_classes=2)} function is used achieveing a testing accuracy of \num{0.944}. For the classification task a one-hot encoding is used. No hyperparameter tuning was used for the toy-datasets. The performance is assessed to be more than sufficient. Similar performance would be expected using a widespread machine learning framework.


\subsection{Use of AI tools}
During the writing of this paper artifical intelligence in the form of large-language models (LLMs) has been used. Two different LLMs have been used for a faster writing of the code used. Mainly for short rewrites of code structure and brain storming the model GPT-5 by OpenAI and the model DeepSeek by Hangzhou DeepSeek Artificial Intelligence have been used. At no point was artifical intelligence used for writing the report. All output by LLMs has be checked by the author.
	
\section{Results and Discussion}\label{section:results} 
\subsection{Introduction to the Dataset}
\begin{figure}
  \centering
  \includegraphics[width=\columnwidth]{../notebooks/violin_plot_features_0.pdf}
  \caption{Violin plot of the distribution of all features in the dataset, separated by benign and malignant samples. A target value of 0 corresponds to malignant samples, while a target value of 1 corresponds to benign samples.}
  \label{fig:violinplot}
\end{figure}

For the entire study we use the dataset presented by Wolberg \cite{williamwolbergBreastCancerWisconsin1993}. It includes features recorded from nuclei in breast tissue and their geometric properties. Each physical feature is present three times, where the means and standard deviations of the measured physical features are used as separate features. Furthermore, the mean of the three largest measurements of each physical feature is present in the dataset. The physical features include: the radius, the standard deviation of the images gray-scale values (called texture), the perimeter, the area and the local variation in the radius length, called smoothness. Additionally, the concavity, the number of concave points, the compactness, the symmetry and the fractal dimension are present. For normalization, all features are rescaled to be centered around 0 and have a standard deviation of 1. The normalizaiton is achieved using the \texttt{StandardScaler} class of the \texttt{scikit-learn} library. A quantitative overview on the average of each of the physical features is presented in \cref{fig:violinplot}. The violinplot shows the feature distribution for both classes of benign and malignant cell nuclei. 

The separability of the two classes is easily visible given all features in the class. On the other hand, there is no single feature which allows for very good separation. Thus reproducing separability using only a limited set of features is the challenge of this study.

\subsection{Regression on Limited Feature Set}
We use a FFNN with six input nodes and hidden layers of equal node count. For the output we use a layer using linear activation $g(z) = z$, i.e. no activation function or in other words a linear regression. To measure the performance of the model, the MSE of the models outputs is used as a loss metric. The model is fitted using the Adam optimizer algorithm with a learning rate of $\lambda = \num{e-2}$ and decay parameters $\beta_1 = \num{0.9}$ and $\beta_2 = \num{0.999}$. The mean squared error per target variable is evaluated. For a simple model without any hyperparameter optimization the MSE is in the range of \qtyrange{2e-4}{2e1}{\std \squared}, where \unit{\std} signifies the standard deviation of the feature values.

\subsubsection{Hyperparameter Tuning}
\begin{figure}
  \centering
  \includegraphics[width=\columnwidth]{../notebooks/nodenumber_tuning_regression.pdf}
  \caption{$\log_{10} \mathrm{MSE}$ for different numbers of hidden layers in the regression network and nodes per hidden layer.}
  \label{fig:regression_hyperparameter}
\end{figure}
As mentioned in \cref{subsec:activation}, the Leaky ReLU combines the disadvantages of the ReLU while conserving non-vanishing gradients on the entire domain. Thus it is not surprising, that they perform similar in our studies with the LReLU providing a more stable training process. Due to this finding we will use the LReLU for all of the hidden layers.
\Cref{fig:regression_hyperparameter} shows the resulting MSE for different numbers of hidden layers and numbers of nodes in the layers. We can observe a general trend of decrease in MSE with a decrease in the number of hidden layers and an increase in the number of neurons per hidden layer. This general trend is to be expected as with an increase in the number of hidden layers the gradients in the last layers can converge towards zero with deep networks due to the multiplicic nature of the gradients. Additionally, can an increase number of nodes per layer better capture even complicated relationships in the data. Due to the adverse effects of an increase in node number, we chose a conservative set of parameters with 32 neurons per hidden layer and two hidden layers.

In a second substudy we observe the influence of the regularization constant with and L2-regularization on all layers. Further studies on variable regularization per layer were not carried out. The studies show a steady MSE for a range of \numrange{0}{e-3} with a sharp increase by a factor of \num{10} with the step to $\lambda = \num{e-2}$. Thus, we choose to use $\lambda = \num{e-3}$ for the regressional model. This provides a stable insurance against overfitting of the training data while at the same time not negatively effecting the models' generalization ability.

\begin{figure}
  \centering
  \includegraphics[width=\columnwidth]{../notebooks/regression_final_model_performance.pdf}
  \caption{Average MSE of the predicted features using the regression model by feature label.}
  \label{fig:regression_errors}
\end{figure}

The final set of hyperparameters is then again tested for MSE on all of the features of the dataset it is optimized to predict. The results of the per target MSE are displayed in \cref{fig:regression_errors}, while we see an increase in the upper bound of the per target MSE, to \qty{e2}{\square \std}, in comparison to the unoptimized model, the general trend for the MSE shows a lower baseline. The lowest accuracy show the predictions for the texture and perimeter. Both show an MSE of \qty{100}{\square \std} in the mean and worst values. Especially the perimeter values are surprising, as one would expect the perimeter to follow from a simple relation with the nuclei's radius. All the other physical features behave similar with respect to the prediction error of \qty{e-3}{\square \std}. In general the prediction uncertainties are smaller on the error values of the physical features.



\subsubsection{Comparison against Linear Regression}

\subsection{Classification on Regression Output}
\subsubsection{Using Logistic Regression for Classification}


\begin{itemize}
    \item Present your results
    \item Give a critical discussion of your work and place it in the correct context.
    \item Relate your work to other calculations/studies
    \item An eventual reader should be able to reproduce your calculations if she/he wants to do so. All input variables should be properly explained.
    \item Make sure that figures\ref{fig:puppy} and tables contain enough information in their captions, axis labels etc. so that an eventual reader can gain a good impression of your work by studying figures and tables only.
\end{itemize}


\section{Conclusion}\label{section:conclusion} 
\begin{itemize}
    \item State your main findings and interpretations
    \item Try to discuss the pros and cons of the methods and possible improvements
    \item State limitations of the study
    \item Try as far as possible to present perspectives for future work
\end{itemize}

\bibliography{biblio}

\end{document}
