lars/FYSSTK-Project1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement new ideas for plots. Heatmap for ridge poly-deg vs. lambda and combined bias variance tradeoff

Browse Source
This commit is contained in:
Lars Bogner
2025-09-10 18:09:14 +02:00
parent f79b4c01f6
commit 977059e8c2
17 changed files with 170 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
figures/bias_variance_tradeoff.pdf
BIN
View File
Binary file not shown.
figures/k_fold_bootstrapping_comparision.pdf → figures/bias_variance_tradeoff_combined_plot.pdf
BIN
View File
Binary file not shown.
figures/cost_function_comparison.pdf
BIN
View File
Binary file not shown.
figures/data_scatter.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 551 KiB

After

Width:  |  Height:  |  Size: 581 KiB

figures/gradient_descent_convergence.pdf
BIN
View File
Binary file not shown.
figures/kfold_mse_comparison_per_cost_function.pdf
BIN
View File
Binary file not shown.
figures/ols_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ols_parameter_plot.pdf
BIN
View File
Binary file not shown.
figures/optimization_performance.pdf
BIN
View File
Binary file not shown.
figures/optimizer_comparison.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_heatmap.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2_lambda.pdf
BIN
View File
Binary file not shown.
figures/ridge_parameter_plot.pdf
BIN
View File
Binary file not shown.
figures/stochastic_gradient_descent_convergence.pdf
BIN
View File
Binary file not shown.
src/main.ipynb
+164 -77
View File
@@ -258,6 +258,54 @@
"id": "12",
"metadata": {},
"outputs": [],
"source": [
"lambda_values = np.logspace(-5, 3, 60)\n",
"polynomial_degrees = np.arange(1, 31, dtype=int)\n",
"\n",
"test_mse_ridge_list = np.zeros((len(polynomial_degrees), len(lambda_values)))\n",
"\n",
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
"\n",
" for j, lambda_ in enumerate(lambda_values):\n",
" beta = optimizers.Ridge_parameters(X_train_scaled, y_train_scaled, lambda_)\n",
"\n",
" y_pred = X_test_scaled @ beta\n",
" mse, r2 = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
" test_mse_ridge_list[i, j] = mse"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "13",
"metadata": {},
"outputs": [],
"source": [
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
"c = ax.pcolormesh(\n",
" lambda_values,\n",
" polynomial_degrees,\n",
" test_mse_ridge_list,\n",
" shading=\"auto\",\n",
" cmap=\"inferno\",\n",
")\n",
"fig.colorbar(c, ax=ax, label=\"Test MSE\")\n",
"ax.set_xscale(\"log\")\n",
"ax.set_xlabel(\"$\\\\lambda$\")\n",
"ax.set_ylabel(\"Polynomial Degree\")\n",
"fig.savefig(os.path.join(FIG_DIR, \"ridge_mse_heatmap.pdf\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "14",
"metadata": {},
"outputs": [],
"source": [
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
@@ -298,7 +346,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "13",
"id": "15",
"metadata": {},
"outputs": [],
"source": [
@@ -352,7 +400,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "14",
"id": "16",
"metadata": {},
"outputs": [],
"source": [
@@ -394,7 +442,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "15",
"id": "17",
"metadata": {},
"outputs": [],
"source": [
@@ -447,7 +495,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "16",
"id": "18",
"metadata": {},
"outputs": [],
"source": [
@@ -523,13 +571,17 @@
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
")\n",
"plotting.plot_optimizers(\n",
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=None, labels=labels\n",
")\n",
"plotting.plot_optimizers(\n",
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=None, labels=labels\n",
")\n",
"for ax in axs:\n",
" ax.set_xlabel(\"Epoch\")\n",
"\n",
"axs[0].set_title(\"OLS\")\n",
"axs[1].set_title(\"Ridge\")\n",
"axs[2].set_title(\"LASSO\")\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"stochastic_gradient_descent_convergence.pdf\"))"
]
@@ -537,13 +589,15 @@
{
"cell_type": "code",
"execution_count": null,
"id": "17",
"id": "19",
"metadata": {},
"outputs": [],
"source": [
"x = np.linspace(-1, 1, 300)\n",
"y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"y = datamanip.runge_function(x)\n",
"x_train, x_test, y_train, y_test_noise_free = train_test_split(x, y, test_size=0.2)\n",
"y_train = datamanip.noise_data(y_train, 1.0)\n",
"y_test = datamanip.noise_data(y_test_noise_free, 1.0)\n",
"\n",
"polynomial_degrees = np.arange(1, 50)\n",
"n_bootstraps = len(x_train)\n",
@@ -555,6 +609,7 @@
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
" y_train_scaled, y_test_scaled_nf = datamanip.scale_data(y_train, y_test_noise_free)\n",
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
"\n",
" for b, (X_, y_) in enumerate(\n",
@@ -564,55 +619,60 @@
" y_pred = X_test_scaled @ beta\n",
" mse, _ = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
" mses[i, b] = mse\n",
" biases[i, b] = np.sqrt(np.mean((y_test_scaled - np.mean(y_pred)) ** 2))\n",
" biases[i, b] = np.mean((y_test_scaled_nf - np.mean(y_pred)) ** 2)\n",
" variances[i, b] = np.var(y_pred)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18",
"id": "20",
"metadata": {},
"outputs": [],
"source": [
"FILL_BETWEEN = True\n",
"fig, axs = plt.subplots(1, 2, figsize=plotting.get_figsize(0.4))\n",
"axs[0].plot(polynomial_degrees, np.mean(mses, axis=1), label=\"MSE (Test)\", color=\"C0\")\n",
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
"mse_mean = np.mean(mses, axis=1)\n",
"bias_mean = np.mean(biases, axis=1)\n",
"var_mean = np.mean(variances, axis=1)\n",
"\n",
"\n",
"axs[1].plot(polynomial_degrees, np.mean(biases, axis=1), label=\"Bias$^2$\", color=\"C1\")\n",
"axs[1].plot(\n",
" polynomial_degrees, np.mean(variances, axis=1), label=\"Variance\", color=\"C2\"\n",
"ax.plot(polynomial_degrees, bias_mean, label=\"Bias$^2$\", color=\"C2\", ls=\"-\")\n",
"ax.fill_between(\n",
" polynomial_degrees,\n",
" np.zeros_like(bias_mean),\n",
" bias_mean,\n",
" color=\"C2\",\n",
" alpha=0.3,\n",
")\n",
"if FILL_BETWEEN:\n",
" axs[0].fill_between(\n",
" polynomial_degrees,\n",
" np.mean(mses, axis=1) - np.std(mses, axis=1),\n",
" np.mean(mses, axis=1) + np.std(mses, axis=1),\n",
" color=\"C0\",\n",
" alpha=0.3,\n",
" )\n",
" axs[1].fill_between(\n",
" polynomial_degrees,\n",
" np.mean(biases, axis=1) - np.std(biases, axis=1),\n",
" np.mean(biases, axis=1) + np.std(biases, axis=1),\n",
" color=\"C1\",\n",
" alpha=0.3,\n",
" )\n",
" axs[1].fill_between(\n",
" polynomial_degrees,\n",
" np.mean(variances, axis=1) - np.std(variances, axis=1),\n",
" np.mean(variances, axis=1) + np.std(variances, axis=1),\n",
" color=\"C2\",\n",
" alpha=0.3,\n",
" )\n",
"axs[0].set_ylabel(\"Mean Squared Error\")\n",
"axs[1].set_ylabel(\"Bias and Variance\")\n",
"ax.plot(\n",
" polynomial_degrees,\n",
" var_mean + bias_mean,\n",
" label=\"Variance + Bias$^2$\",\n",
" color=\"C1\",\n",
" ls=\"-\",\n",
")\n",
"ax.fill_between(\n",
" polynomial_degrees,\n",
" bias_mean,\n",
" bias_mean + var_mean,\n",
" color=\"C1\",\n",
" alpha=0.3,\n",
")\n",
"ax.fill_between(\n",
" polynomial_degrees,\n",
" bias_mean + var_mean,\n",
" mse_mean,\n",
" color=\"C0\",\n",
" alpha=0.3,\n",
" label=\"Irreducible Error\",\n",
" hatch=\"//\",\n",
" edgecolor=\"white\",\n",
")\n",
"ax.plot(polynomial_degrees, mse_mean, label=\"MSE\", color=\"C0\", ls=\"-\")\n",
"\n",
"for ax in axs:\n",
" ax.set_xlabel(\"Polynomial Degree\")\n",
" ax.set_yscale(\"log\")\n",
" ax.legend()\n",
"\n",
"ax.set_xlabel(\"Polynomial Degree\")\n",
"ax.set_ylabel(\"Error\")\n",
"ax.legend()\n",
"\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff.pdf\"))"
@@ -621,13 +681,13 @@
{
"cell_type": "code",
"execution_count": null,
"id": "19",
"id": "21",
"metadata": {},
"outputs": [],
"source": [
"k_folds = 5\n",
"k_fold_mses = np.zeros((len(polynomial_degrees), k_folds))\n",
"splits = datamanip.k_fold_split(x, y, k_folds)\n",
"splits = datamanip.k_fold_split(x, datamanip.noise_data(y), k_folds)\n",
"\n",
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
" for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n",
@@ -645,50 +705,77 @@
{
"cell_type": "code",
"execution_count": null,
"id": "20",
"id": "22",
"metadata": {},
"outputs": [],
"source": [
"FILL_BETWEEN = False\n",
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
"fig, (ax, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5), sharey=True)\n",
"mse_mean = np.mean(mses, axis=1)\n",
"bias_mean = np.mean(biases, axis=1)\n",
"var_mean = np.mean(variances, axis=1)\n",
"\n",
"ax.plot(\n",
" polynomial_degrees, np.mean(mses, axis=1), label=\"Bootstrapping (Test)\", color=\"C0\"\n",
"ax.plot(polynomial_degrees, bias_mean, label=\"Bias$^2$\", color=\"C2\", ls=\"-\")\n",
"ax.fill_between(\n",
" polynomial_degrees,\n",
" np.zeros_like(bias_mean),\n",
" bias_mean,\n",
" color=\"C2\",\n",
" alpha=0.3,\n",
")\n",
"ax.plot(\n",
" polynomial_degrees,\n",
" np.mean(k_fold_mses, axis=1),\n",
" label=f\"{k_folds}-Fold (Test)\",\n",
" var_mean + bias_mean,\n",
" label=\"Variance + Bias$^2$\",\n",
" color=\"C1\",\n",
" ls=\"-\",\n",
")\n",
"if FILL_BETWEEN:\n",
" ax.fill_between(\n",
" polynomial_degrees,\n",
" np.mean(mses, axis=1) - np.std(mses, axis=1),\n",
" np.mean(mses, axis=1) + np.std(mses, axis=1),\n",
" color=\"C0\",\n",
" alpha=0.3,\n",
" )\n",
" ax.fill_between(\n",
" polynomial_degrees,\n",
" np.mean(k_fold_mses, axis=1) - np.std(k_fold_mses, axis=1),\n",
" np.mean(k_fold_mses, axis=1) + np.std(k_fold_mses, axis=1),\n",
" color=\"C1\",\n",
" alpha=0.3,\n",
" )\n",
"ax.fill_between(\n",
" polynomial_degrees,\n",
" bias_mean,\n",
" bias_mean + var_mean,\n",
" color=\"C1\",\n",
" alpha=0.3,\n",
")\n",
"ax.fill_between(\n",
" polynomial_degrees,\n",
" bias_mean + var_mean,\n",
" mse_mean,\n",
" color=\"C0\",\n",
" alpha=0.3,\n",
" label=\"Irreducible Error\",\n",
" hatch=\"//\",\n",
" edgecolor=\"white\",\n",
")\n",
"ax.plot(polynomial_degrees, mse_mean, label=\"MSE\", color=\"C0\", ls=\"-\")\n",
"\n",
"\n",
"ax.set_xlabel(\"Polynomial Degree\")\n",
"ax.set_ylabel(\"Mean Squared Error\")\n",
"ax.set_yscale(\"log\")\n",
"ax.set_ylabel(\"Error\")\n",
"ax.legend()\n",
"\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"k_fold_bootstrapping_comparision.pdf\"))"
"\n",
"\n",
"ax2.plot(polynomial_degrees, mse_mean, label=\"Bootstrapping MSE\", color=\"C0\")\n",
"ax2.plot(\n",
" polynomial_degrees,\n",
" np.mean(k_fold_mses, axis=1),\n",
" label=f\"{k_folds}-Fold Crossvalidation MSE\",\n",
" color=\"C1\",\n",
")\n",
"\n",
"\n",
"ax2.set_xlabel(\"Polynomial Degree\")\n",
"# ax2.set_ylabel(\"Mean Squared Error\")\n",
"ax2.legend()\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff_combined_plot.pdf\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "21",
"id": "23",
"metadata": {},
"outputs": [],
"source": [
@@ -699,7 +786,7 @@
"k_fold_mses_ridge = np.zeros_like(k_fold_mses_ols)\n",
"k_fold_mses_lasso = np.zeros_like(k_fold_mses_ols)\n",
"\n",
"splits = datamanip.k_fold_split(x, y, k_folds)\n",
"splits = datamanip.k_fold_split(x, datamanip.noise_data(y), k_folds)\n",
"\n",
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
" for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n",
@@ -726,7 +813,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "22",
"id": "24",
"metadata": {},
"outputs": [],
"source": [
@@ -760,7 +847,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "23",
"id": "25",
"metadata": {},
"outputs": [],
"source": []
src/pyoptim/plotting.py
+6 -2
View File
@@ -45,14 +45,18 @@ def set_rc_params():
set_rc_params()
def get_figsize(rel_height: float = 2 / 3) -> tuple[float, float]:
def get_figsize(
rel_height: float = 2 / 3, full_width: bool = False
) -> tuple[float, float]:
"""Returns a figure size tuple based on the global FIG_WIDTH and a relative height.
Args:
rel_height: The relative height of the figure compared to FIG_WIDTH.
Returns:
A tuple (width, height) for the figure size.
"""
return (FIG_WIDTH, FIG_WIDTH * rel_height)
if full_width:
return (FIG_WIDTH, FIG_WIDTH * rel_height)
return (FIG_WIDTH * 0.5, FIG_WIDTH * 0.5 * rel_height)
def mse_r2_plot(