lars/FYSSTK-Project2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run ruff

Browse Source
This commit is contained in:
Lars Bogner
2025-11-05 08:43:18 +01:00
parent 6466f07935
commit e6f4a7369f
11 changed files with 314 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/10_regression-analysis.ipynb
+103 -39
View File
@@ -17,11 +17,18 @@
"data = load_breast_cancer()\n",
"\n",
"\n",
"feature_names = [n for n in data.feature_names if 'radius' in n or 'area' in n]\n",
"feature_names = [n for n in data.feature_names if \"radius\" in n or \"area\" in n]\n",
"\n",
"X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n",
"y = data.data[:, [data.feature_names.tolist().index(n) for n in data.feature_names if n not in feature_names]]\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n"
"y = data.data[\n",
" :,\n",
" [\n",
" data.feature_names.tolist().index(n)\n",
" for n in data.feature_names\n",
" if n not in feature_names\n",
" ],\n",
"]\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
]
},
{
@@ -31,7 +38,15 @@
"metadata": {},
"outputs": [],
"source": [
"from easynn.feedforward import Layer, FFNN, ReLU, LeakyReLU, Linear, MSELoss, Regularization\n",
"from easynn.feedforward import (\n",
" Layer,\n",
" FFNN,\n",
" ReLU,\n",
" LeakyReLU,\n",
" Linear,\n",
" MSELoss,\n",
" Regularization,\n",
")\n",
"from easynn.schedulers import AdamScheduler"
]
},
@@ -44,14 +59,17 @@
"source": [
"feature_dim = X.shape[1]\n",
"target_dim = y.shape[1]\n",
"def get_regression_model(n_hidden_layers: int, n_neurons: int, activation: type=ReLU) -> list[Layer]:\n",
"\n",
"\n",
"def get_regression_model(\n",
" n_hidden_layers: int, n_neurons: int, activation: type = ReLU\n",
") -> list[Layer]:\n",
" layers = []\n",
" layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n",
" for _ in range(n_hidden_layers - 1):\n",
" layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n",
" layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n",
" return layers\n",
"\n"
" return layers"
]
},
{
@@ -63,7 +81,11 @@
"source": [
"# First network test\n",
"\n",
"model = FFNN(get_regression_model(n_hidden_layers=3, n_neurons=32, activation=LeakyReLU), AdamScheduler(epochs=10000, learning_rate=1e-2), MSELoss())\n",
"model = FFNN(\n",
" get_regression_model(n_hidden_layers=3, n_neurons=32, activation=LeakyReLU),\n",
" AdamScheduler(epochs=10000, learning_rate=1e-2),\n",
" MSELoss(),\n",
")\n",
"model.fit(X_train, y_train)"
]
},
@@ -100,10 +122,14 @@
"import matplotlib.pyplot as plt\n",
"\n",
"y_pred = model.predict(X_test)\n",
"mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n",
"mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n",
"\n",
"sns.barplot(x=np.arange(len(mse)), y=mse)\n",
"plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n",
"plt.xticks(\n",
" ticks=np.arange(len(mse)),\n",
" labels=[n for n in data.feature_names if n not in feature_names],\n",
" rotation=90,\n",
")\n",
"plt.yscale(\"log\")\n",
"plt.ylabel(\"Mean Squared Error\")"
]
@@ -131,15 +157,23 @@
"n_neurons_list = [4, 8, 16, 32, 64, 128]\n",
"mse_results = np.zeros((len(n_layers_list), len(n_neurons_list)))\n",
"for idx, (n_layers, n_neurons) in enumerate(\n",
" tqdm(product(n_layers_list, n_neurons_list),\n",
" total=len(n_layers_list) * len(n_neurons_list))\n",
" tqdm(\n",
" product(n_layers_list, n_neurons_list),\n",
" total=len(n_layers_list) * len(n_neurons_list),\n",
" )\n",
"):\n",
" i, j = divmod(idx, len(n_neurons_list))\n",
" model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=15000, learning_rate=1e-3), MSELoss())\n",
" model = FFNN(\n",
" get_regression_model(\n",
" n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
" ),\n",
" AdamScheduler(epochs=15000, learning_rate=1e-3),\n",
" MSELoss(),\n",
" )\n",
" model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n",
" mse = mean_squared_error(y_test, y_pred)\n",
" mse_results[i, j] = mse\n"
" mse_results[i, j] = mse"
]
},
{
@@ -160,10 +194,16 @@
}
],
"source": [
"sns.heatmap(mse_results, xticklabels=n_neurons_list, yticklabels=n_layers_list, annot=True, fmt=\".2f\")\n",
"sns.heatmap(\n",
" mse_results,\n",
" xticklabels=n_neurons_list,\n",
" yticklabels=n_layers_list,\n",
" annot=True,\n",
" fmt=\".2f\",\n",
")\n",
"plt.xlabel(\"Number of Neurons\")\n",
"plt.ylabel(\"Number of Layers\")\n",
"plt.savefig(\"nodenumber_tuning_regression.pdf\")\n"
"plt.savefig(\"nodenumber_tuning_regression.pdf\")"
]
},
{
@@ -177,10 +217,16 @@
"parameters = [(1, 32), (1, 64), (2, 32), (2, 64), (3, 64), (3, 128)]\n",
"mse_results_slow_scan = np.zeros((len(parameters), target_dim))\n",
"for k, (n_layers, n_neurons) in enumerate(parameters):\n",
" model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=20000, learning_rate=5e-4), MSELoss())\n",
" model = FFNN(\n",
" get_regression_model(\n",
" n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
" ),\n",
" AdamScheduler(epochs=20000, learning_rate=5e-4),\n",
" MSELoss(),\n",
" )\n",
" model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n",
" mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n",
" mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n",
" mse_results_slow_scan[k, :] = mse"
]
},
@@ -202,12 +248,16 @@
}
],
"source": [
"sns.heatmap(np.log10(mse_results_slow_scan), xticklabels=[n for n in data.feature_names if n not in feature_names], yticklabels=[f\"L{l}_N{n}\" for l, n in parameters],)\n",
"plt.xticks(rotation=45, ha='right')\n",
"sns.heatmap(\n",
" np.log10(mse_results_slow_scan),\n",
" xticklabels=[n for n in data.feature_names if n not in feature_names],\n",
" yticklabels=[f\"L{l}_N{n}\" for l, n in parameters],\n",
")\n",
"plt.xticks(rotation=45, ha=\"right\")\n",
"plt.xlabel(\"Target Features\")\n",
"plt.ylabel(\"Model (Layers_Neurons)\")\n",
"plt.tight_layout()\n",
"plt.savefig(\"regression_detailed_hyperparameter_scan.pdf\")\n"
"plt.savefig(\"regression_detailed_hyperparameter_scan.pdf\")"
]
},
{
@@ -241,7 +291,7 @@
" x=log_mse.ravel(),\n",
" y=np.repeat(np.arange(len(parameters)), log_mse.shape[1]),\n",
" bins=[np.linspace(-5, 2, 15), len(parameters)],\n",
" cmap=cmap\n",
" cmap=cmap,\n",
")\n",
"\n",
"# Label axes\n",
@@ -266,7 +316,7 @@
"outputs": [],
"source": [
"n_hidden_layers = 2\n",
"n_neurons = 32\n"
"n_neurons = 32"
]
},
{
@@ -276,14 +326,15 @@
"metadata": {},
"outputs": [],
"source": [
"\n",
"# regularization test\n",
"layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n",
"layers = get_regression_model(\n",
" n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
")\n",
"regularization_strengths = [0.0, 1e-5, 1e-4, 1e-3, 1e-2]\n",
"mse_results_reg = np.zeros(len(regularization_strengths))\n",
"for i, reg_strength in enumerate(regularization_strengths):\n",
" for layer in layers:\n",
" layer.regularization = Regularization(reg_strength, 'l2')\n",
" layer.regularization = Regularization(reg_strength, \"l2\")\n",
" model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n",
" model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n",
@@ -335,9 +386,11 @@
"outputs": [],
"source": [
"# Check final model training\n",
"layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n",
"layers = get_regression_model(\n",
" n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
")\n",
"for layer in layers:\n",
" layer.regularization = Regularization(reg_strength, 'l2')\n",
" layer.regularization = Regularization(reg_strength, \"l2\")\n",
"model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n",
"model.fit(X_train, y_train)"
]
@@ -361,7 +414,7 @@
],
"source": [
"sns.lineplot(x=np.arange(20000), y=model.scheduler.loss_history)\n",
"plt.yscale(\"log\")\n"
"plt.yscale(\"log\")"
]
},
{
@@ -383,10 +436,14 @@
],
"source": [
"y_pred = model.predict(X_test)\n",
"mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n",
"mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n",
"\n",
"sns.barplot(x=np.arange(len(mse)), y=mse)\n",
"plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n",
"plt.xticks(\n",
" ticks=np.arange(len(mse)),\n",
" labels=[n for n in data.feature_names if n not in feature_names],\n",
" rotation=90,\n",
")\n",
"plt.yscale(\"log\")\n",
"plt.ylabel(\"Mean Squared Error\")\n",
"plt.tight_layout()\n",
@@ -425,16 +482,23 @@
"source": [
"beta = optimizers.OLS_parameters(X_train, y_train)\n",
"y_ols_pred = X_test @ beta\n",
"mse_ols = mean_squared_error(y_test, y_ols_pred, multioutput='raw_values')\n",
"mse_ols = mean_squared_error(y_test, y_ols_pred, multioutput=\"raw_values\")\n",
"\n",
"mse_df = pd.DataFrame({\n",
" \"Feature\": [n for n in data.feature_names if n not in feature_names],\n",
" \"Neural Network\": mse,\n",
" \"OLS\": mse_ols\n",
"})\n",
"mse_df = pd.DataFrame(\n",
" {\n",
" \"Feature\": [n for n in data.feature_names if n not in feature_names],\n",
" \"Neural Network\": mse,\n",
" \"OLS\": mse_ols,\n",
" }\n",
")\n",
"\n",
"sns.barplot(x=\"Feature\", y=\"value\", hue=\"Model\", data=pd.melt(mse_df, id_vars=[\"Feature\"]).rename(columns={\"variable\": \"Model\"}))\n",
"plt.xticks(rotation=45, ha='right')\n",
"sns.barplot(\n",
" x=\"Feature\",\n",
" y=\"value\",\n",
" hue=\"Model\",\n",
" data=pd.melt(mse_df, id_vars=[\"Feature\"]).rename(columns={\"variable\": \"Model\"}),\n",
")\n",
"plt.xticks(rotation=45, ha=\"right\")\n",
"plt.yscale(\"log\")\n",
"plt.ylabel(\"Mean Squared Error\")\n",
"plt.xlabel(\"Target Feature\")\n",