diff --git a/figures/bias_variance_tradeoff.pdf b/figures/bias_variance_tradeoff.pdf index bbef9ab..0cd3e4c 100644 Binary files a/figures/bias_variance_tradeoff.pdf and b/figures/bias_variance_tradeoff.pdf differ diff --git a/figures/k_fold_bootstrapping_comparision.pdf b/figures/bias_variance_tradeoff_combined_plot.pdf similarity index 79% rename from figures/k_fold_bootstrapping_comparision.pdf rename to figures/bias_variance_tradeoff_combined_plot.pdf index 63f390a..39569e4 100644 Binary files a/figures/k_fold_bootstrapping_comparision.pdf and b/figures/bias_variance_tradeoff_combined_plot.pdf differ diff --git a/figures/cost_function_comparison.pdf b/figures/cost_function_comparison.pdf index 24af9f3..e92df9c 100644 Binary files a/figures/cost_function_comparison.pdf and b/figures/cost_function_comparison.pdf differ diff --git a/figures/data_scatter.png b/figures/data_scatter.png index 0072da0..28a6677 100644 Binary files a/figures/data_scatter.png and b/figures/data_scatter.png differ diff --git a/figures/gradient_descent_convergence.pdf b/figures/gradient_descent_convergence.pdf index 02bc16c..6913623 100644 Binary files a/figures/gradient_descent_convergence.pdf and b/figures/gradient_descent_convergence.pdf differ diff --git a/figures/kfold_mse_comparison_per_cost_function.pdf b/figures/kfold_mse_comparison_per_cost_function.pdf index 66f69d4..0f69870 100644 Binary files a/figures/kfold_mse_comparison_per_cost_function.pdf and b/figures/kfold_mse_comparison_per_cost_function.pdf differ diff --git a/figures/ols_mse_r2.pdf b/figures/ols_mse_r2.pdf index abc2703..ec853fd 100644 Binary files a/figures/ols_mse_r2.pdf and b/figures/ols_mse_r2.pdf differ diff --git a/figures/ols_parameter_plot.pdf b/figures/ols_parameter_plot.pdf index c2775c1..a42d3c7 100644 Binary files a/figures/ols_parameter_plot.pdf and b/figures/ols_parameter_plot.pdf differ diff --git a/figures/optimization_performance.pdf b/figures/optimization_performance.pdf index 4e8f9cf..d8bd6e8 100644 Binary files a/figures/optimization_performance.pdf and b/figures/optimization_performance.pdf differ diff --git a/figures/optimizer_comparison.pdf b/figures/optimizer_comparison.pdf index 3fc71a2..1fad5b5 100644 Binary files a/figures/optimizer_comparison.pdf and b/figures/optimizer_comparison.pdf differ diff --git a/figures/ridge_mse_heatmap.pdf b/figures/ridge_mse_heatmap.pdf new file mode 100644 index 0000000..735ebd0 Binary files /dev/null and b/figures/ridge_mse_heatmap.pdf differ diff --git a/figures/ridge_mse_r2.pdf b/figures/ridge_mse_r2.pdf index 3bee7eb..bfeec55 100644 Binary files a/figures/ridge_mse_r2.pdf and b/figures/ridge_mse_r2.pdf differ diff --git a/figures/ridge_mse_r2_lambda.pdf b/figures/ridge_mse_r2_lambda.pdf index 028184b..0d5bcbb 100644 Binary files a/figures/ridge_mse_r2_lambda.pdf and b/figures/ridge_mse_r2_lambda.pdf differ diff --git a/figures/ridge_parameter_plot.pdf b/figures/ridge_parameter_plot.pdf index 8d8e8f3..3a2b22a 100644 Binary files a/figures/ridge_parameter_plot.pdf and b/figures/ridge_parameter_plot.pdf differ diff --git a/figures/stochastic_gradient_descent_convergence.pdf b/figures/stochastic_gradient_descent_convergence.pdf index 2df036c..b577b8a 100644 Binary files a/figures/stochastic_gradient_descent_convergence.pdf and b/figures/stochastic_gradient_descent_convergence.pdf differ diff --git a/src/main.ipynb b/src/main.ipynb index bc1d9d6..37dd488 100644 --- a/src/main.ipynb +++ b/src/main.ipynb @@ -258,6 +258,54 @@ "id": "12", "metadata": {}, "outputs": [], + "source": [ + "lambda_values = np.logspace(-5, 3, 60)\n", + "polynomial_degrees = np.arange(1, 31, dtype=int)\n", + "\n", + "test_mse_ridge_list = np.zeros((len(polynomial_degrees), len(lambda_values)))\n", + "\n", + "for i, polynomial_degree in enumerate(polynomial_degrees):\n", + " X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n", + " X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n", + " X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n", + " y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n", + "\n", + " for j, lambda_ in enumerate(lambda_values):\n", + " beta = optimizers.Ridge_parameters(X_train_scaled, y_train_scaled, lambda_)\n", + "\n", + " y_pred = X_test_scaled @ beta\n", + " mse, r2 = datamanip.evaluate_model(y_test_scaled, y_pred)\n", + " test_mse_ridge_list[i, j] = mse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", + "c = ax.pcolormesh(\n", + " lambda_values,\n", + " polynomial_degrees,\n", + " test_mse_ridge_list,\n", + " shading=\"auto\",\n", + " cmap=\"inferno\",\n", + ")\n", + "fig.colorbar(c, ax=ax, label=\"Test MSE\")\n", + "ax.set_xscale(\"log\")\n", + "ax.set_xlabel(\"$\\\\lambda$\")\n", + "ax.set_ylabel(\"Polynomial Degree\")\n", + "fig.savefig(os.path.join(FIG_DIR, \"ridge_mse_heatmap.pdf\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], "source": [ "X_train = datamanip.polynomial_features(x_train, 10, False)\n", "X_test = datamanip.polynomial_features(x_test, 10, False)\n", @@ -298,7 +346,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -352,7 +400,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -394,7 +442,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -447,7 +495,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -523,13 +571,17 @@ " axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n", ")\n", "plotting.plot_optimizers(\n", - " axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n", + " axs[1], optimizers_ridge, X_tr, y_tr, ylabel=None, labels=labels\n", ")\n", "plotting.plot_optimizers(\n", - " axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n", + " axs[2], optimizers_lasso, X_tr, y_tr, ylabel=None, labels=labels\n", ")\n", "for ax in axs:\n", " ax.set_xlabel(\"Epoch\")\n", + "\n", + "axs[0].set_title(\"OLS\")\n", + "axs[1].set_title(\"Ridge\")\n", + "axs[2].set_title(\"LASSO\")\n", "fig.tight_layout()\n", "fig.savefig(os.path.join(FIG_DIR, \"stochastic_gradient_descent_convergence.pdf\"))" ] @@ -537,13 +589,15 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "19", "metadata": {}, "outputs": [], "source": [ "x = np.linspace(-1, 1, 300)\n", - "y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n", - "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "y = datamanip.runge_function(x)\n", + "x_train, x_test, y_train, y_test_noise_free = train_test_split(x, y, test_size=0.2)\n", + "y_train = datamanip.noise_data(y_train, 1.0)\n", + "y_test = datamanip.noise_data(y_test_noise_free, 1.0)\n", "\n", "polynomial_degrees = np.arange(1, 50)\n", "n_bootstraps = len(x_train)\n", @@ -555,6 +609,7 @@ " X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n", " X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n", " X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n", + " y_train_scaled, y_test_scaled_nf = datamanip.scale_data(y_train, y_test_noise_free)\n", " y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n", "\n", " for b, (X_, y_) in enumerate(\n", @@ -564,55 +619,60 @@ " y_pred = X_test_scaled @ beta\n", " mse, _ = datamanip.evaluate_model(y_test_scaled, y_pred)\n", " mses[i, b] = mse\n", - " biases[i, b] = np.sqrt(np.mean((y_test_scaled - np.mean(y_pred)) ** 2))\n", + " biases[i, b] = np.mean((y_test_scaled_nf - np.mean(y_pred)) ** 2)\n", " variances[i, b] = np.var(y_pred)" ] }, { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "20", "metadata": {}, "outputs": [], "source": [ - "FILL_BETWEEN = True\n", - "fig, axs = plt.subplots(1, 2, figsize=plotting.get_figsize(0.4))\n", - "axs[0].plot(polynomial_degrees, np.mean(mses, axis=1), label=\"MSE (Test)\", color=\"C0\")\n", + "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", + "mse_mean = np.mean(mses, axis=1)\n", + "bias_mean = np.mean(biases, axis=1)\n", + "var_mean = np.mean(variances, axis=1)\n", "\n", - "\n", - "axs[1].plot(polynomial_degrees, np.mean(biases, axis=1), label=\"Bias$^2$\", color=\"C1\")\n", - "axs[1].plot(\n", - " polynomial_degrees, np.mean(variances, axis=1), label=\"Variance\", color=\"C2\"\n", + "ax.plot(polynomial_degrees, bias_mean, label=\"Bias$^2$\", color=\"C2\", ls=\"-\")\n", + "ax.fill_between(\n", + " polynomial_degrees,\n", + " np.zeros_like(bias_mean),\n", + " bias_mean,\n", + " color=\"C2\",\n", + " alpha=0.3,\n", ")\n", - "if FILL_BETWEEN:\n", - " axs[0].fill_between(\n", - " polynomial_degrees,\n", - " np.mean(mses, axis=1) - np.std(mses, axis=1),\n", - " np.mean(mses, axis=1) + np.std(mses, axis=1),\n", - " color=\"C0\",\n", - " alpha=0.3,\n", - " )\n", - " axs[1].fill_between(\n", - " polynomial_degrees,\n", - " np.mean(biases, axis=1) - np.std(biases, axis=1),\n", - " np.mean(biases, axis=1) + np.std(biases, axis=1),\n", - " color=\"C1\",\n", - " alpha=0.3,\n", - " )\n", - " axs[1].fill_between(\n", - " polynomial_degrees,\n", - " np.mean(variances, axis=1) - np.std(variances, axis=1),\n", - " np.mean(variances, axis=1) + np.std(variances, axis=1),\n", - " color=\"C2\",\n", - " alpha=0.3,\n", - " )\n", - "axs[0].set_ylabel(\"Mean Squared Error\")\n", - "axs[1].set_ylabel(\"Bias and Variance\")\n", + "ax.plot(\n", + " polynomial_degrees,\n", + " var_mean + bias_mean,\n", + " label=\"Variance + Bias$^2$\",\n", + " color=\"C1\",\n", + " ls=\"-\",\n", + ")\n", + "ax.fill_between(\n", + " polynomial_degrees,\n", + " bias_mean,\n", + " bias_mean + var_mean,\n", + " color=\"C1\",\n", + " alpha=0.3,\n", + ")\n", + "ax.fill_between(\n", + " polynomial_degrees,\n", + " bias_mean + var_mean,\n", + " mse_mean,\n", + " color=\"C0\",\n", + " alpha=0.3,\n", + " label=\"Irreducible Error\",\n", + " hatch=\"//\",\n", + " edgecolor=\"white\",\n", + ")\n", + "ax.plot(polynomial_degrees, mse_mean, label=\"MSE\", color=\"C0\", ls=\"-\")\n", "\n", - "for ax in axs:\n", - " ax.set_xlabel(\"Polynomial Degree\")\n", - " ax.set_yscale(\"log\")\n", - " ax.legend()\n", + "\n", + "ax.set_xlabel(\"Polynomial Degree\")\n", + "ax.set_ylabel(\"Error\")\n", + "ax.legend()\n", "\n", "fig.tight_layout()\n", "fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff.pdf\"))" @@ -621,13 +681,13 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "21", "metadata": {}, "outputs": [], "source": [ "k_folds = 5\n", "k_fold_mses = np.zeros((len(polynomial_degrees), k_folds))\n", - "splits = datamanip.k_fold_split(x, y, k_folds)\n", + "splits = datamanip.k_fold_split(x, datamanip.noise_data(y), k_folds)\n", "\n", "for i, polynomial_degree in enumerate(polynomial_degrees):\n", " for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n", @@ -645,50 +705,77 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "22", "metadata": {}, "outputs": [], "source": [ - "FILL_BETWEEN = False\n", - "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", + "fig, (ax, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5), sharey=True)\n", + "mse_mean = np.mean(mses, axis=1)\n", + "bias_mean = np.mean(biases, axis=1)\n", + "var_mean = np.mean(variances, axis=1)\n", "\n", - "ax.plot(\n", - " polynomial_degrees, np.mean(mses, axis=1), label=\"Bootstrapping (Test)\", color=\"C0\"\n", + "ax.plot(polynomial_degrees, bias_mean, label=\"Bias$^2$\", color=\"C2\", ls=\"-\")\n", + "ax.fill_between(\n", + " polynomial_degrees,\n", + " np.zeros_like(bias_mean),\n", + " bias_mean,\n", + " color=\"C2\",\n", + " alpha=0.3,\n", ")\n", "ax.plot(\n", " polynomial_degrees,\n", - " np.mean(k_fold_mses, axis=1),\n", - " label=f\"{k_folds}-Fold (Test)\",\n", + " var_mean + bias_mean,\n", + " label=\"Variance + Bias$^2$\",\n", " color=\"C1\",\n", + " ls=\"-\",\n", ")\n", - "if FILL_BETWEEN:\n", - " ax.fill_between(\n", - " polynomial_degrees,\n", - " np.mean(mses, axis=1) - np.std(mses, axis=1),\n", - " np.mean(mses, axis=1) + np.std(mses, axis=1),\n", - " color=\"C0\",\n", - " alpha=0.3,\n", - " )\n", - " ax.fill_between(\n", - " polynomial_degrees,\n", - " np.mean(k_fold_mses, axis=1) - np.std(k_fold_mses, axis=1),\n", - " np.mean(k_fold_mses, axis=1) + np.std(k_fold_mses, axis=1),\n", - " color=\"C1\",\n", - " alpha=0.3,\n", - " )\n", + "ax.fill_between(\n", + " polynomial_degrees,\n", + " bias_mean,\n", + " bias_mean + var_mean,\n", + " color=\"C1\",\n", + " alpha=0.3,\n", + ")\n", + "ax.fill_between(\n", + " polynomial_degrees,\n", + " bias_mean + var_mean,\n", + " mse_mean,\n", + " color=\"C0\",\n", + " alpha=0.3,\n", + " label=\"Irreducible Error\",\n", + " hatch=\"//\",\n", + " edgecolor=\"white\",\n", + ")\n", + "ax.plot(polynomial_degrees, mse_mean, label=\"MSE\", color=\"C0\", ls=\"-\")\n", + "\n", "\n", "ax.set_xlabel(\"Polynomial Degree\")\n", - "ax.set_ylabel(\"Mean Squared Error\")\n", - "ax.set_yscale(\"log\")\n", + "ax.set_ylabel(\"Error\")\n", "ax.legend()\n", + "\n", "fig.tight_layout()\n", - "fig.savefig(os.path.join(FIG_DIR, \"k_fold_bootstrapping_comparision.pdf\"))" + "\n", + "\n", + "ax2.plot(polynomial_degrees, mse_mean, label=\"Bootstrapping MSE\", color=\"C0\")\n", + "ax2.plot(\n", + " polynomial_degrees,\n", + " np.mean(k_fold_mses, axis=1),\n", + " label=f\"{k_folds}-Fold Crossvalidation MSE\",\n", + " color=\"C1\",\n", + ")\n", + "\n", + "\n", + "ax2.set_xlabel(\"Polynomial Degree\")\n", + "# ax2.set_ylabel(\"Mean Squared Error\")\n", + "ax2.legend()\n", + "fig.tight_layout()\n", + "fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff_combined_plot.pdf\"))" ] }, { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -699,7 +786,7 @@ "k_fold_mses_ridge = np.zeros_like(k_fold_mses_ols)\n", "k_fold_mses_lasso = np.zeros_like(k_fold_mses_ols)\n", "\n", - "splits = datamanip.k_fold_split(x, y, k_folds)\n", + "splits = datamanip.k_fold_split(x, datamanip.noise_data(y), k_folds)\n", "\n", "for i, polynomial_degree in enumerate(polynomial_degrees):\n", " for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n", @@ -726,7 +813,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -760,7 +847,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "25", "metadata": {}, "outputs": [], "source": [] diff --git a/src/pyoptim/plotting.py b/src/pyoptim/plotting.py index de5582d..8aedc1d 100644 --- a/src/pyoptim/plotting.py +++ b/src/pyoptim/plotting.py @@ -45,14 +45,18 @@ def set_rc_params(): set_rc_params() -def get_figsize(rel_height: float = 2 / 3) -> tuple[float, float]: +def get_figsize( + rel_height: float = 2 / 3, full_width: bool = False +) -> tuple[float, float]: """Returns a figure size tuple based on the global FIG_WIDTH and a relative height. Args: rel_height: The relative height of the figure compared to FIG_WIDTH. Returns: A tuple (width, height) for the figure size. """ - return (FIG_WIDTH, FIG_WIDTH * rel_height) + if full_width: + return (FIG_WIDTH, FIG_WIDTH * rel_height) + return (FIG_WIDTH * 0.5, FIG_WIDTH * 0.5 * rel_height) def mse_r2_plot(