{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6dbeb580",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import plotting\n",
    "\n",
    "\n",
    "data = load_breast_cancer()\n",
    "\n",
    "\n",
    "feature_names = [n for n in data.feature_names if 'radius' in n or 'area' in n]\n",
    "\n",
    "X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n",
    "y = data.data[:, [data.feature_names.tolist().index(n) for n in data.feature_names if n not in feature_names]]\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "83deb258",
   "metadata": {},
   "outputs": [],
   "source": [
    "from easynn.feedforward import Layer, FFNN, ReLU, LeakyReLU, Linear, MSELoss, Regularization\n",
    "from easynn.schedulers import AdamScheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "382a6b64",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_dim = X.shape[1]\n",
    "target_dim = y.shape[1]\n",
    "def get_regression_model(n_hidden_layers: int, n_neurons: int, activation: type=ReLU) -> list[Layer]:\n",
    "    layers = []\n",
    "    layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n",
    "    for _ in range(n_hidden_layers - 1):\n",
    "        layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n",
    "    layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n",
    "    return layers\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "08a53cf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# First network test\n",
    "\n",
    "model = FFNN(get_regression_model(n_hidden_layers=3, n_neurons=32, activation=LeakyReLU), AdamScheduler(epochs=10000, learning_rate=1e-2), MSELoss())\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f2983ade",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Mean Squared Error')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 132524 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "y_pred = model.predict(X_test)\n",
    "mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n",
    "\n",
    "sns.barplot(x=np.arange(len(mse)), y=mse)\n",
    "plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n",
    "plt.yscale(\"log\")\n",
    "plt.ylabel(\"Mean Squared Error\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2df46970",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 36/36 [18:43<00:00, 31.21s/it]\n"
     ]
    }
   ],
   "source": [
    "# Hyperparameter tuning; number of layers and neurons\n",
    "from tqdm import tqdm\n",
    "from itertools import product\n",
    "\n",
    "n_layers_list = [1, 2, 3, 4, 5, 6]\n",
    "n_neurons_list = [4, 8, 16, 32, 64, 128]\n",
    "mse_results = np.zeros((len(n_layers_list), len(n_neurons_list)))\n",
    "for idx, (n_layers, n_neurons) in enumerate(\n",
    "    tqdm(product(n_layers_list, n_neurons_list),\n",
    "         total=len(n_layers_list) * len(n_neurons_list))\n",
    "):\n",
    "    i, j = divmod(idx, len(n_neurons_list))\n",
    "    model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=15000, learning_rate=1e-3), MSELoss())\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mse = mean_squared_error(y_test, y_pred)\n",
    "    mse_results[i, j] = mse\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "5148a48a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 156124 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.heatmap(mse_results, xticklabels=n_neurons_list, yticklabels=n_layers_list, annot=True, fmt=\".2f\")\n",
    "plt.xlabel(\"Number of Neurons\")\n",
    "plt.ylabel(\"Number of Layers\")\n",
    "plt.savefig(\"nodenumber_tuning_regression.pdf\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fcdc7a9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Identify a set of good hyperparameters and evaluate\n",
    "parameters = [(1, 32), (1, 64), (2, 32), (2, 64), (3, 64), (3, 128)]\n",
    "mse_results_slow_scan = np.zeros((len(parameters), target_dim))\n",
    "for k, (n_layers, n_neurons) in enumerate(parameters):\n",
    "    model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=20000, learning_rate=5e-4), MSELoss())\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n",
    "    mse_results_slow_scan[k, :] = mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "faa1838c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 311508 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.heatmap(np.log10(mse_results_slow_scan), xticklabels=[n for n in data.feature_names if n not in feature_names], yticklabels=[f\"L{l}_N{n}\" for l, n in parameters],)\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.xlabel(\"Target Features\")\n",
    "plt.ylabel(\"Model (Layers_Neurons)\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"regression_detailed_hyperparameter_scan.pdf\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bc255d9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 91572 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib.colors import ListedColormap\n",
    "\n",
    "log_mse = np.log10(mse_results_slow_scan)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "# Compute a 2D histogram: x = log10(MSE), y = parameter index\n",
    "# 'bins' can be adjusted for smoother or more detailed output\n",
    "cmap = sns.color_palette(\"rocket\", as_cmap=True)\n",
    "h = ax.hist2d(\n",
    "    x=log_mse.ravel(),\n",
    "    y=np.repeat(np.arange(len(parameters)), log_mse.shape[1]),\n",
    "    bins=[np.linspace(-5, 2, 15), len(parameters)],\n",
    "    cmap=cmap\n",
    ")\n",
    "\n",
    "# Label axes\n",
    "ax.set_xlabel(\"log10(MSE)\")\n",
    "ax.set_ylabel(\"Parameter set\")\n",
    "ax.set_yticks(np.arange(len(parameters)))\n",
    "ax.set_yticklabels([f\"L{l}_N{n}\" for l, n in parameters])\n",
    "ax.set_title(\"2D Histogram of log10(MSE) per Parameter Set\")\n",
    "\n",
    "# Add colorbar\n",
    "fig.colorbar(h[3], ax=ax, label=\"Count\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9ea48022",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_hidden_layers = 2\n",
    "n_neurons = 32\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2b4c6c07",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# regularization test\n",
    "layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n",
    "regularization_strengths = [0.0, 1e-5, 1e-4, 1e-3, 1e-2]\n",
    "mse_results_reg = np.zeros(len(regularization_strengths))\n",
    "for i, reg_strength in enumerate(regularization_strengths):\n",
    "    for layer in layers:\n",
    "        layer.regularization = Regularization(reg_strength, 'l2')\n",
    "    model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n",
    "    model.fit(X_train, y_train)\n",
    "    y_pred = model.predict(X_test)\n",
    "    mse = mean_squared_error(y_test, y_pred)\n",
    "    mse_results_reg[i] = mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dc0c201d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 40088 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.barplot(x=[str(r) for r in regularization_strengths], y=mse_results_reg)\n",
    "plt.yscale(\"log\")\n",
    "plt.xlabel(\"L2 Regularization Strength\")\n",
    "plt.ylabel(\"Mean Squared Error\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"regression_regularization_tuning.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f228fe14",
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_strength = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3802a70b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check final model training\n",
    "layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n",
    "for layer in layers:\n",
    "    layer.regularization = Regularization(reg_strength, 'l2')\n",
    "model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cac82a2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 44956 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.lineplot(x=np.arange(20000), y=model.scheduler.loss_history)\n",
    "plt.yscale(\"log\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d860a38e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 121644 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred = model.predict(X_test)\n",
    "mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n",
    "\n",
    "sns.barplot(x=np.arange(len(mse)), y=mse)\n",
    "plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n",
    "plt.yscale(\"log\")\n",
    "plt.ylabel(\"Mean Squared Error\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"regression_final_model_performance.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0ba6ac8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyoptim.optimizers as optimizers\n",
    "import pyoptim.datamanip as datamanip\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7aa98299",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 260640 characters]",
      "text/plain": [
       "<Figure size 1350x900 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "beta = optimizers.OLS_parameters(X_train, y_train)\n",
    "y_ols_pred = X_test @ beta\n",
    "mse_ols = mean_squared_error(y_test, y_ols_pred, multioutput='raw_values')\n",
    "\n",
    "mse_df = pd.DataFrame({\n",
    "    \"Feature\": [n for n in data.feature_names if n not in feature_names],\n",
    "    \"Neural Network\": mse,\n",
    "    \"OLS\": mse_ols\n",
    "})\n",
    "\n",
    "sns.barplot(x=\"Feature\", y=\"value\", hue=\"Model\", data=pd.melt(mse_df, id_vars=[\"Feature\"]).rename(columns={\"variable\": \"Model\"}))\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.yscale(\"log\")\n",
    "plt.ylabel(\"Mean Squared Error\")\n",
    "plt.xlabel(\"Target Feature\")\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"regression_nn_vs_ols.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d2314a6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "project2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
