Intermediate commmit, close to finishin the results

2025-11-03 17:36:50 +01:00
parent 1f036a0ff0
commit 2d7af415f6
16 changed files with 372 additions and 64 deletions
@@ -2529,7 +2529,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
   "id": "c94858f9",
   "metadata": {},
   "outputs": [
@@ -2546,7 +2546,6 @@
   ],
   "source": [
    "sns.heatmap(cm, annot=True, fmt='d')\n",
-    "plt.title(\"Confusion Matrix\")\n",
    "plt.xlabel(\"Predicted Label\")\n",
    "plt.ylabel(\"True Label\")\n",
    "plt.tight_layout()\n",
@@ -2556,7 +2555,7 @@
  },
  {
   "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": null,
   "id": "66af4e3e",
   "metadata": {},
   "outputs": [
@@ -2574,7 +2573,6 @@
   "source": [
    "sns.lineplot(x=fpr, y=tpr)\n",
    "plt.plot([0, 1], [0, 1], linestyle='--', color='gray')\n",
-    "plt.title(\"ROC Curve\")\n",
    "plt.xlabel(\"False Positive Rate\")\n",
    "plt.ylabel(\"True Positive Rate\")\n",
    "plt.tight_layout()\n",
@@ -10,6 +10,7 @@ dependencies = [
    "mypy>=1.18.2",
    "numpy>=2.3.4",
    "pandas>=2.3.3",
+    "project1",
    "pytest>=8.4.2",
    "ruff>=0.14.1",
    "scikit-learn>=1.7.2",
@@ -22,3 +23,6 @@ dependencies = [
 [build-system]
 requires = ["uv_build>=0.9.5,<0.10.0"]
 build-backend = "uv_build"
+
+[tool.uv.sources]
+project1 = { path = "../project1" }
@@ -0,0 +1,150 @@
+@article{bennettDecisionTreeConstruction1992,
+  title = {Decision Tree Construction via Linear Programming},
+  author = {Bennett, Kristin},
+  year = 1992,
+  month = jan,
+  journal = {Proceedings of the 4th Midwest Artificial Intelligence and Cognitive Science Society Conference, Utica, Illinois}
+}
+
+@book{bishopPatternRecognitionMachine2006,
+  title = {Pattern Recognition and Machine Learning},
+  author = {Bishop, Christopher M.},
+  year = 2006,
+  series = {Information Science and Statistics},
+  publisher = {Springer},
+  address = {New York},
+  isbn = {978-0-387-31073-2},
+  langid = {english},
+  lccn = {006.4},
+  file = {/home/lars/Zotero/storage/9H5W9BGC/Bishop - 2006 - Pattern recognition and machine learning.pdf}
+}
+
+@article{bognerRegularizationOptimizationAll2025,
+  title = {Regularization and {{Optimization}} Is {{All You Need}}?},
+  author = {Bogner, Lars},
+  year = 2025,
+  month = oct
+}
+
+@misc{elstnerLectureMachineLearning2025,
+  type = {{Lecture}},
+  title = {{Lecture: Machine Learning for Chemistry}},
+  author = {Elstner, Marcus and Kubar, Tomas},
+  year = 2025,
+  month = may,
+  address = {Karlsruhe Institute for Technology, Karlsruhe},
+  langid = {ngerman},
+  file = {/home/lars/Zotero/storage/5LSLJMK8/Elstner and Kubar - 2025 - Lecture Machine Learning for Chemistry.pdf}
+}
+
+@book{goodfellowDeepLearning2016,
+  title = {Deep {{Learning}}},
+  author = {Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron},
+  year = 2016,
+  publisher = {MIT Press}
+}
+
+@article{harrisArrayProgrammingNumPy2020,
+  title = {Array Programming with {{NumPy}}},
+  author = {Harris, Charles R. and Millman, K. Jarrod and van der Walt, St{\'e}fan J. and Gommers, Ralf and Virtanen, Pauli and Cournapeau, David and Wieser, Eric and Taylor, Julian and Berg, Sebastian and Smith, Nathaniel J. and Kern, Robert and Picus, Matti and Hoyer, Stephan and van Kerkwijk, Marten H. and Brett, Matthew and Haldane, Allan and del R{\'i}o, Jaime Fern{\'a}ndez and Wiebe, Mark and Peterson, Pearu and {G{\'e}rard-Marchant}, Pierre and Sheppard, Kevin and Reddy, Tyler and Weckesser, Warren and Abbasi, Hameer and Gohlke, Christoph and Oliphant, Travis E.},
+  year = 2020,
+  month = sep,
+  journal = {Nature},
+  volume = {585},
+  number = {7825},
+  pages = {357--362},
+  publisher = {{Springer Science and Business Media LLC}},
+  doi = {10.1038/s41586-020-2649-2}
+}
+
+@book{hastieElementsStatisticalLearning2009,
+  title = {The {{Elements}} of {{Statistical Learning}}},
+  author = {Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
+  year = 2009,
+  series = {Springer {{Series}} in {{Statistics}}},
+  publisher = {Springer},
+  address = {New York, NY},
+  doi = {10.1007/978-0-387-84858-7},
+  urldate = {2025-09-22},
+  copyright = {http://www.springer.com/tdm},
+  isbn = {978-0-387-84857-0 978-0-387-84858-7},
+  keywords = {Averaging,Boosting,classification,clustering,data mining,machine learning,Projection pursuit,Random Forest,supervised learning,Support Vector Machine,unsupervised learning},
+  file = {/home/lars/Zotero/storage/D3N4DVY9/Hastie et al. - 2009 - The Elements of Statistical Learning.pdf}
+}
+
+@article{hunterMatplotlib2DGraphics2007,
+  title = {Matplotlib: {{A 2D}} Graphics Environment},
+  author = {Hunter, J. D.},
+  year = 2007,
+  journal = {Computing in Science \& Engineering},
+  volume = {9},
+  number = {3},
+  pages = {90--95},
+  publisher = {IEEE COMPUTER SOC},
+  doi = {10.1109/MCSE.2007.55},
+  abstract = {Matplotlib is a 2D graphics package used for Python for application development, interactive scripting, and publication-quality image generation across user interfaces and operating systems.}
+}
+
+@misc{mostafaBreastCancerPrediction,
+  title = {Breast {{Cancer Prediction}} (F1 = 0.99, {{AUC}} = 0.99)},
+  author = {Mostafa, Omar},
+  urldate = {2025-11-03},
+  abstract = {Explore and run machine learning code with Kaggle Notebooks \textbar{} Using data from Breast Cancer Wisconsin (Diagnostic) Data Set},
+  howpublished = {https://kaggle.com/code/omarmostafataha/breast-cancer-prediction-f1-0-99-auc-0-99},
+  langid = {english},
+  file = {/home/lars/Zotero/storage/NFAZ887D/breast-cancer-prediction-f1-0-99-auc-0-99.html}
+}
+
+@misc{NeuralNetworksTikZnet2024,
+  title = {Neural Networks -- {{TikZ}}.Net},
+  year = 2024,
+  month = apr,
+  urldate = {2025-11-01},
+  abstract = {Some examples of neural network architectures: deep neural networks (DNNs), a deep convolutional neural network (CNN), an autoencoders (encoder+decoder), and the illustration of an activation function in neurons. Basic idea The full LaTeX code at the bottom of this post uses the listofitems library, so one can pre-define an array of the number of nodes\dots},
+  langid = {american},
+  file = {/home/lars/Zotero/storage/7F7GGAEU/neural_networks.html}
+}
+
+@article{pedregosaScikitlearnMachineLearning2011,
+  title = {Scikit-Learn: {{Machine Learning}} in {{Python}}},
+  author = {Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
+  year = 2011,
+  journal = {Journal of Machine Learning Research},
+  volume = {12},
+  pages = {2825--2830}
+}
+
+@misc{teamPandasdevPandasPandas2025,
+  title = {Pandas-Dev/Pandas: {{Pandas}}},
+  shorttitle = {Pandas-Dev/Pandas},
+  author = {pandas development {team}, The},
+  year = 2025,
+  month = sep,
+  doi = {10.5281/zenodo.17229934},
+  urldate = {2025-10-16},
+  abstract = {Pandas is a powerful data structures for data analysis, time series, and statistics.},
+  howpublished = {Zenodo},
+  keywords = {data science,python},
+  file = {/home/lars/Zotero/storage/QV289HHN/17229934.html}
+}
+
+@article{Waskom2021,
+  title = {Seaborn: Statistical Data Visualization},
+  author = {Waskom, Michael L.},
+  year = 2021,
+  journal = {Journal of Open Source Software},
+  volume = {6},
+  number = {60},
+  pages = {3021},
+  publisher = {The Open Journal},
+  doi = {10.21105/joss.03021}
+}
+
+@misc{williamwolbergBreastCancerWisconsin1993,
+  title = {Breast {{Cancer Wisconsin}} ({{Diagnostic}})},
+  author = {William Wolberg, Olvi Mangasarian},
+  year = 1993,
+  publisher = {UCI Machine Learning Repository},
+  doi = {10.24432/C5DW2B},
+  urldate = {2025-11-01}
+}
@@ -85,12 +85,22 @@
  abstract = {Matplotlib is a 2D graphics package used for Python for application development, interactive scripting, and publication-quality image generation across user interfaces and operating systems.}
 }

+@misc{mostafaBreastCancerPrediction,
+  title = {Breast {{Cancer Prediction}} (F1 = 0.99, {{AUC}} = 0.99)},
+  author = {Mostafa, Omar},
+  urldate = {2025-11-03},
+  abstract = {Explore and run machine learning code with Kaggle Notebooks \textbar{} Using data from Breast Cancer Wisconsin (Diagnostic) Data Set},
+  howpublished = {https://kaggle.com/code/omarmostafataha/breast-cancer-prediction-f1-0-99-auc-0-99},
+  langid = {english},
+  file = {/home/lars/Zotero/storage/NFAZ887D/breast-cancer-prediction-f1-0-99-auc-0-99.html}
+}
+
@misc{NeuralNetworksTikZnet2024,
  title = {Neural Networks -- {{TikZ}}.Net},
  year = 2024,
  month = apr,
  urldate = {2025-11-01},
-  abstract = {Some examples of neural network architectures: deep neural networks (DNNs), a deep convolutional neural network (CNN), an autoencoders (encoder+decoder), and the illustration of an activation function in neurons. Basic idea The full LaTeX code at the bottom of this post uses the listofitems library, so one can pre-define an array of the number of nodes{\dots}},
+  abstract = {Some examples of neural network architectures: deep neural networks (DNNs), a deep convolutional neural network (CNN), an autoencoders (encoder+decoder), and the illustration of an activation function in neurons. Basic idea The full LaTeX code at the bottom of this post uses the listofitems library, so one can pre-define an array of the number of nodes\dots},
  langid = {american},
  file = {/home/lars/Zotero/storage/7F7GGAEU/neural_networks.html}
 }
@@ -1 +1 @@
-\@doendnote{endnote11}{Solely used for data splitting and performance metrics.}
+\@doendnote{endnote12}{Solely used for data splitting and performance metrics.}
@@ -219,7 +219,7 @@ with the optimization parameters $\beta_{1/2}$. As shown in our previous article
 \subsubsection{Train-Test Splitting}
 As we mentioned before, there is the possibility for our model to overadjust for the datapoints present during the optimization process. To still get an accurate measure for the models performance we will only use a subset of the dataset for the process of training. A random subset of \qty{20}{\percent} is reserved for testing the model only. This means, that all performance metrics are evaluated in the same way as new unseen data would perform. Due to the random nature of the subset, the performance between runs may differ stochastically. This may be counteracted by redoing the tests with different choices of training and testing datasets and averaging the metrics.

-\subsubsection{Out-of-Fold prediction}
+\subsubsection{Out-of-Fold prediction} \label{subsec:outoffold}
 \begin{figure}
    \centering
    \begin{tikzpicture}[scale=0.8]
@@ -281,6 +281,8 @@ For the entire numerical computation of the neural network, the \texttt{numpy} \

 To check the functionality of the developed code, in a first stage, the loss history is evaluated visually showing a clear convergence after a high enough number of epochs. Furthermore, the quality of the models is evaluated using toy-datasets created using \texttt{scikit-learn}. In detail, both a regression toy-dataset as well as a classification dataset are validated. A MSE of \num{0.000631} on a dataset created from \texttt{make\_regression(n\_samples=1000, n\_features=5, n\_informative=3, noise=0.1)} is achieved. For classification the \texttt{make\_classification(n\_samples=1000, n\_features=5, n\_informative=3, n\_classes=2)} function is used achieveing a testing accuracy of \num{0.944}. For the classification task a one-hot encoding is used. No hyperparameter tuning was used for the toy-datasets. The performance is assessed to be more than sufficient. Similar performance would be expected using a widespread machine learning framework.

+For the comparison against ordinary least squares regression, we use the framework presented before by the same author \cite{bognerRegularizationOptimizationAll2025}. It uses the analytical solution to the ordinary least squares regression to find the optimal parameters.
+

 \subsection{Use of AI tools}
 During the writing of this paper artifical intelligence in the form of large-language models (LLMs) has been used. Two different LLMs have been used for a faster writing of the code used. Mainly for short rewrites of code structure and brain storming the model GPT-5 by OpenAI and the model DeepSeek by Hangzhou DeepSeek Artificial Intelligence have been used. At no point was artifical intelligence used for writing the report. All output by LLMs has be checked by the author.
@@ -305,7 +307,7 @@ We use a FFNN with six input nodes and hidden layers of equal node count. For th
 \begin{figure}
  \centering
  \includegraphics[width=\columnwidth]{../notebooks/nodenumber_tuning_regression.pdf}
-  \caption{$\log_{10} \mathrm{MSE}$ for different numbers of hidden layers in the regression network and nodes per hidden layer.}
+  \caption{MSE for different numbers of hidden layers in the regression network and nodes per hidden layer.}
  \label{fig:regression_hyperparameter}
 \end{figure}
 As mentioned in \cref{subsec:activation}, the Leaky ReLU combines the disadvantages of the ReLU while conserving non-vanishing gradients on the entire domain. Thus it is not surprising, that they perform similar in our studies with the LReLU providing a more stable training process. Due to this finding we will use the LReLU for all of the hidden layers.
@@ -320,32 +322,56 @@ In a second substudy we observe the influence of the regularization constant wit
  \label{fig:regression_errors}
 \end{figure}

-The final set of hyperparameters is then again tested for MSE on all of the features of the dataset it is optimized to predict. The results of the per target MSE are displayed in \cref{fig:regression_errors}, while we see an increase in the upper bound of the per target MSE, to \qty{e2}{\square \std}, in comparison to the unoptimized model, the general trend for the MSE shows a lower baseline. The lowest accuracy show the predictions for the texture and perimeter. Both show an MSE of \qty{100}{\square \std} in the mean and worst values. Especially the perimeter values are surprising, as one would expect the perimeter to follow from a simple relation with the nuclei's radius. All the other physical features behave similar with respect to the prediction error of \qty{e-3}{\square \std}. In general the prediction uncertainties are smaller on the error values of the physical features.
-
+The final set of hyperparameters is then again tested for MSE on all of the features of the dataset it is optimized to predict. The results of the per target MSE are displayed in \cref{fig:regression_errors}, while we see an increase in the upper bound of the per target MSE, to \qty{e1}{\square \std}, in comparison to the unoptimized model, the general trend for the MSE shows a lower baseline. The lowest accuracy show the predictions for the texture and perimeter. Both show an MSE in excess of \qty{5}{\square \std} in the mean and worst values. Especially the perimeter values are surprising, as one would expect the perimeter to follow from a simple relation with the nuclei's radius. All the other physical features behave similar with respect to the prediction error of \qtyrange{e-3}{e-4}{\square \std}. In general the prediction uncertainties are smaller on the error values of the physical features.

+Based on these hyperparameters we trained models using 5-fold out-of-fold prediction as introduced in \cref{subsec:outoffold}, to create a full version of the dataset using only radii and areas as inputs. For the classification task we will use this dataset.

 \subsubsection{Comparison against Linear Regression}

+\begin{figure}
+  \centering
+  \includegraphics[width=\columnwidth]{../notebooks/regression_nn_vs_ols.pdf}
+  \caption{Comparison of the MSE per target feature between the neural network regression and ordinary least squares regression.}
+  \label{fig:regression_comparison}
+\end{figure}
+
+\Cref{fig:regression_comparison} shows the performance comparison in the MSE between an ordinary least squares (OLS) regression and the optimized neural network. Both show very similar performance with only slightest differences in the MSE. In general ordinary least squares underperform, compared to the FFNN by a small margin. The very good results of the ordinary least squares approach is surprising.
+
+The excellent performance of the linear regression suggests that the additional challenges posed by using the FFNN are not worth the extra development costs. We thus recommend using this approach only for datasets with complex relations.
+
 \subsection{Classification on Regression Output}
+\begin{figure}
+  \centering
+  \includegraphics[width=\columnwidth]{../notebooks/classification_confusion_matrix.pdf}
+  \caption{Confusion Matrix of the classifier for the regression output. A target value of 0 indicates malignant nuclei, while 1 is indicative of benign samples.}
+  \label{fig:confusion_matrix}
+\end{figure}
+Similar to the structure of the regression model, we use a FFNN using a variable number of nodes per hidden layer in configurations with different numbers of hidden layers to create a classifier. To use the model as a classifier, the loss function of the model is changed to the cross-entropy loss introduced in \cref{section:methods}. Additionally, we change the activation of the output layer to the softmax function. The leaky rectified linear unit is again used for the purpose in all other neurons, following the results from the previous study. The parameters are optimized using loss minimization using the Adam optimization algorithm. 
+
+As above, a parameter scan is used to find the best parameter combination for accurate prediction, whether the nuclei are part of benign or malignant tissue. The results of the parameter scan are displayed in \cref{fig:classification_hyperparameters}. For the number of hidden layers and neurons, there is a clear maximum in the area-under-curve (AUC) score for the case of 64 neurons in either a single or two hidden layers. The AUC measures the integral of the receiver-operating-characteristic, which is the graph of true positive rate against the false positive rate. A score of 1 indicates perfect classification, while a score of \num{0.5} is equivalent to a random classifier. Using the full dataset in the form without regression AUC scores of \num{0.9962} have been shown using logisitic regression \cite{mostafaBreastCancerPrediction}. To account for stochastic variations in the training success the AUC is averaged over a total of \num{50} runs for each entry in \cref{fig:classification_hyperparameters}. The regularization constant is chosen to be $\lambda = \num{e-6}$ according to the figure. During \num{50} runs this combination of hyperparameters is able to achieve an average accuracy of \qty{92.70}{\percent} and an AUC score of \num{0.9781}. While this performance is not as good as the performance using logistic regression on the full dataset \cite{mostafaBreastCancerPrediction}, it is to be expected, that the performance decreases when using only a limited set of features to create the regression output and following that the classification.
+
+The performance is further evaluated using the confusion matrix in \cref{fig:confusion_matrix}. Potential asymmetric biases for false identification, i.e. wether type 1 or 2 errors are more pronounced, can be detected using a confusion matrix. It shows which relative amount of each class (malignant or benign) is identified as which by the classifier. No signifcant asymmetry is present in our FFNN based classifier.
+
+
 \subsubsection{Using Logistic Regression for Classification}
+Thus, we also study the use of logistic regression to create a classification based on the outputs of the regression model. The logistic regression model uses the same framework code as the FFNN for comparability. For creating a logistic regression classification, a singular layer with two neurons and a softmax activation is used. The cross-entropy-loss is used as an optimization metric, which is then minimized using the Adam optimizer. 

+After the minimization a test split is evaluated for accuracy and receiver-operating-characteristic leading to an accuracy value of \qty{94.74}{\percent} and an AUC score of \num{0.99105}. The logistic regression thus outperforms the classification models by far. It is especially noteworthy, that nearly the same performance as for full original dataset data can be reached. While it might seem counterintuitive, that the more complex models are not able to match the performance of the arguably very simple logistic regression is due to the fact, that even with regularization, the more complex models have a higher tendency to overfit simple relations between the features and the classification. 
+
+The good performance of the linear regression also helps fur future diagnostic purposes, as we have shown, that with the correct combination of neural networks and logistic regression it is possible to use a very small, easy to extract subset of the physical features to achieve nuclei identification at the highest level.

-\begin{itemize}
-    \item Present your results
-    \item Give a critical discussion of your work and place it in the correct context.
-    \item Relate your work to other calculations/studies
-    \item An eventual reader should be able to reproduce your calculations if she/he wants to do so. All input variables should be properly explained.
-    \item Make sure that figures\ref{fig:puppy} and tables contain enough information in their captions, axis labels etc. so that an eventual reader can gain a good impression of your work by studying figures and tables only.
-\end{itemize}


 \section{Conclusion}\label{section:conclusion} 
-\begin{itemize}
-    \item State your main findings and interpretations
-    \item Try to discuss the pros and cons of the methods and possible improvements
-    \item State limitations of the study
-    \item Try as far as possible to present perspectives for future work
-\end{itemize}
+
+\onecolumngrid
+\appendix
+\begin{figure}
+  \centering
+  \includegraphics[width=.8\textwidth]{../notebooks/classification_hyperparameter_scan.pdf}
+  \caption{Area-under-Curve score dependance on the number of hidden layers, the number of nodes per hidden layer and the regularization strength $\lambda$ for a L2 regularization. The off-diagonal plots show the average AUC per combination of two parameter values. The diagonal values show the average score per single parameter value. Optimization using Adam with $\alpha = \num{0.05}$ and 500 epochs. LReLU activation in the hidden layers, softmax activation in the output layer. Optimized on training cross-entropy loss.}
+  \label{fig:classification_hyperparameters}
+\end{figure}

 \bibliography{biblio}

@@ -183,6 +183,7 @@ dependencies = [
    { name = "mypy" },
    { name = "numpy" },
    { name = "pandas" },
+    { name = "project1" },
    { name = "pytest" },
    { name = "ruff" },
    { name = "scikit-learn" },
@@ -197,6 +198,7 @@ requires-dist = [
    { name = "mypy", specifier = ">=1.18.2" },
    { name = "numpy", specifier = ">=2.3.4" },
    { name = "pandas", specifier = ">=2.3.3" },
+    { name = "project1", directory = "../project1" },
    { name = "pytest", specifier = ">=8.4.2" },
    { name = "ruff", specifier = ">=0.14.1" },
    { name = "scikit-learn", specifier = ">=1.7.2" },
@@ -731,6 +733,31 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
 ]

+[[package]]
+name = "project1"
+version = "0.1.0"
+source = { directory = "../project1" }
+dependencies = [
+    { name = "ipykernel" },
+    { name = "matplotlib" },
+    { name = "mypy" },
+    { name = "numpy" },
+    { name = "pandas" },
+    { name = "scikit-learn" },
+    { name = "scipy" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "ipykernel", specifier = ">=6.30.1" },
+    { name = "matplotlib", specifier = ">=3.10.6" },
+    { name = "mypy", specifier = ">=1.17.1" },
+    { name = "numpy", specifier = ">=2.3.2" },
+    { name = "pandas", specifier = ">=2.3.2" },
+    { name = "scikit-learn", specifier = ">=1.7.1" },
+    { name = "scipy", specifier = ">=1.16.1" },
+]
+
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.52"