diff --git a/figures/bias_variance_tradeoff.pdf b/figures/bias_variance_tradeoff.pdf index 5ecdcd5..bbef9ab 100644 Binary files a/figures/bias_variance_tradeoff.pdf and b/figures/bias_variance_tradeoff.pdf differ diff --git a/figures/cost_function_comparison.pdf b/figures/cost_function_comparison.pdf index 4a5dd90..24af9f3 100644 Binary files a/figures/cost_function_comparison.pdf and b/figures/cost_function_comparison.pdf differ diff --git a/figures/data_scatter.png b/figures/data_scatter.png index 7b3b69e..0072da0 100644 Binary files a/figures/data_scatter.png and b/figures/data_scatter.png differ diff --git a/figures/gradient_descent_convergence.pdf b/figures/gradient_descent_convergence.pdf index 186f889..02bc16c 100644 Binary files a/figures/gradient_descent_convergence.pdf and b/figures/gradient_descent_convergence.pdf differ diff --git a/figures/k_fold_bootstrapping_comparision.pdf b/figures/k_fold_bootstrapping_comparision.pdf new file mode 100644 index 0000000..63f390a Binary files /dev/null and b/figures/k_fold_bootstrapping_comparision.pdf differ diff --git a/figures/kfold_mse_comparison_per_cost_function.pdf b/figures/kfold_mse_comparison_per_cost_function.pdf new file mode 100644 index 0000000..66f69d4 Binary files /dev/null and b/figures/kfold_mse_comparison_per_cost_function.pdf differ diff --git a/figures/ols_mse_r2.pdf b/figures/ols_mse_r2.pdf index 0f132ff..abc2703 100644 Binary files a/figures/ols_mse_r2.pdf and b/figures/ols_mse_r2.pdf differ diff --git a/figures/ols_parameter_plot.pdf b/figures/ols_parameter_plot.pdf index 935f78a..c2775c1 100644 Binary files a/figures/ols_parameter_plot.pdf and b/figures/ols_parameter_plot.pdf differ diff --git a/figures/optimization_performance.pdf b/figures/optimization_performance.pdf new file mode 100644 index 0000000..4e8f9cf Binary files /dev/null and b/figures/optimization_performance.pdf differ diff --git a/figures/optimizer_comparison.pdf b/figures/optimizer_comparison.pdf index 39a54b0..3fc71a2 100644 Binary files a/figures/optimizer_comparison.pdf and b/figures/optimizer_comparison.pdf differ diff --git a/figures/ridge_mse_r2.pdf b/figures/ridge_mse_r2.pdf index 51199ef..3bee7eb 100644 Binary files a/figures/ridge_mse_r2.pdf and b/figures/ridge_mse_r2.pdf differ diff --git a/figures/ridge_mse_r2_lambda.pdf b/figures/ridge_mse_r2_lambda.pdf index 609a41f..028184b 100644 Binary files a/figures/ridge_mse_r2_lambda.pdf and b/figures/ridge_mse_r2_lambda.pdf differ diff --git a/figures/ridge_parameter_plot.pdf b/figures/ridge_parameter_plot.pdf index 15bbf7f..8d8e8f3 100644 Binary files a/figures/ridge_parameter_plot.pdf and b/figures/ridge_parameter_plot.pdf differ diff --git a/figures/stochastic_gradient_descent_convergence.pdf b/figures/stochastic_gradient_descent_convergence.pdf new file mode 100644 index 0000000..2df036c Binary files /dev/null and b/figures/stochastic_gradient_descent_convergence.pdf differ diff --git a/src/main.ipynb b/src/main.ipynb index 284b15c..bc1d9d6 100644 --- a/src/main.ipynb +++ b/src/main.ipynb @@ -18,15 +18,14 @@ "metadata": {}, "outputs": [], "source": [ - "import optimizers\n", - "import datamanip\n", - "import plotting\n", + "import pyoptim.optimizers as optimizers\n", + "import pyoptim.datamanip as datamanip\n", + "import pyoptim.plotting as plotting\n", "\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from sklearn.model_selection import train_test_split\n", - "import os\n", - "import time" + "import os" ] }, { @@ -37,7 +36,7 @@ "outputs": [], "source": [ "FIG_DIR = os.path.abspath(\n", - " os.path.join(os.path.dirname(plotting.__file__), \"..\", \"figures\")\n", + " os.path.join(os.path.dirname(plotting.__file__), \"../..\", \"figures\")\n", ")\n", "print(FIG_DIR)" ] @@ -53,12 +52,7 @@ "y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n", "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", "\n", - "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", - "ax.scatter(x_train, y_train, label=\"Train Data\", s=1)\n", - "ax.scatter(x_test, y_test, label=\"Test Data\", s=1)\n", - "ax.set_xlabel(\"$x$\")\n", - "ax.set_ylabel(\"$y$\")\n", - "ax.legend()\n", + "fig, ax = plotting.scatter_dataset(x_train, x_test, y_train, y_test)\n", "fig.savefig(os.path.join(FIG_DIR, \"data_scatter.png\"), dpi=300)" ] }, @@ -273,29 +267,30 @@ "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5))\n", "num_iters = 1_000\n", "\n", - "for learning_rate in np.logspace(-4, 0, 5):\n", - " OLSGrad_Desc = optimizers.OLSGradientDescent(\n", - " learning_rate=learning_rate, num_iterations=num_iters\n", + "learning_rates = np.logspace(-4, 0, 5)\n", + "ols_optimizers = [\n", + " (\n", + " optimizers.OLSGradientDescent,\n", + " {\"learning_rate\": learning_rate, \"num_iterations\": num_iters},\n", " )\n", - " OLSGrad_Desc.fit(X_tr, y_tr)\n", - " cost_history = OLSGrad_Desc.cost_history\n", - " RidgeGrad_Desc = optimizers.RidgeGradientDescent(\n", - " learning_rate=learning_rate, num_iterations=num_iters, lam=0.1\n", + " for learning_rate in learning_rates\n", + "]\n", + "ridge_optimizers = [\n", + " (\n", + " optimizers.RidgeGradientDescent,\n", + " {\"learning_rate\": learning_rate, \"num_iterations\": num_iters, \"lam\": 0.1},\n", " )\n", - " RidgeGrad_Desc.fit(X_tr, y_tr)\n", - " cost_history_ridge = RidgeGrad_Desc.cost_history\n", - " ax1.plot(cost_history, label=f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\")\n", - " ax2.plot(\n", - " cost_history_ridge, label=f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\"\n", - " )\n", - "ax1.set_xlabel(\"Iteration\")\n", - "ax1.set_ylabel(\"OLS Cost\")\n", - "ax1.legend()\n", + " for learning_rate in learning_rates\n", + "]\n", + "labels = [\n", + " f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\"\n", + " for learning_rate in learning_rates\n", + "]\n", + "\n", + "plotting.plot_optimizers(ax1, ols_optimizers, X_tr, y_tr, \"OLS Cost\", labels=labels)\n", + "plotting.plot_optimizers(ax2, ridge_optimizers, X_tr, y_tr, \"Ridge Cost\", labels=labels)\n", "ax1.set_ylim(bottom=0.465, top=0.505)\n", "ax2.set_ylim(bottom=0.4775, top=0.505)\n", - "ax2.set_xlabel(\"Iteration\")\n", - "ax2.set_ylabel(\"Ridge Cost\")\n", - "ax2.legend()\n", "fig.tight_layout()\n", "fig.savefig(os.path.join(FIG_DIR, \"gradient_descent_convergence.pdf\"))" ] @@ -306,20 +301,6 @@ "id": "13", "metadata": {}, "outputs": [], - "source": [ - "def get_cost_history(optimizer, X, y, **kwargs):\n", - " opt = optimizer(**kwargs)\n", - " opt.fit(X, y)\n", - " iterations = opt.get_epochs()\n", - " return iterations, opt.cost_history" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], "source": [ "fig, (ax1, ax2) = plt.subplots(2, 1, figsize=plotting.get_figsize(0.8))\n", "\n", @@ -333,87 +314,36 @@ "learning_rate_ridge = 0.01\n", "lam = 0.1\n", "\n", + "general_kwargs = {\n", + " \"num_iterations\": num_iters,\n", + "}\n", + "optimizer_kwargs = [\n", + " {},\n", + " {\"delta\": 0.9},\n", + " {},\n", + " {\"gamma\": 0.995},\n", + " {\"beta1\": 0.9, \"beta2\": 0.999},\n", + "]\n", + "ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n", + "ridge_kwargs = {\n", + " \"learning_rate\": learning_rate_ridge,\n", + " \"lam\": lam,\n", + " **general_kwargs,\n", + "}\n", "optimizers_ols = [\n", - " (\n", - " optimizers.OLSGradientDescent,\n", - " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n", - " ),\n", - " (\n", - " optimizers.OLSMomentum,\n", - " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"delta\": 0.9},\n", - " ),\n", - " (\n", - " optimizers.OLSAdaGrad,\n", - " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n", - " ),\n", - " (\n", - " optimizers.OLSRMSProp,\n", - " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"gamma\": 0.9},\n", - " ),\n", - " (\n", - " optimizers.OLSAdam,\n", - " {\n", - " \"num_iterations\": num_iters,\n", - " \"learning_rate\": learning_rate_ols,\n", - " \"beta1\": 0.9,\n", - " \"beta2\": 0.999,\n", - " },\n", - " ),\n", + " (opt, {**ols_kwargs, **opt_kwargs})\n", + " for opt, opt_kwargs in zip(optimizers.OLS_GD_OPTIMIZERS, optimizer_kwargs)\n", "]\n", - "\n", "optimizers_ridge = [\n", - " (\n", - " optimizers.RidgeGradientDescent,\n", - " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n", - " ),\n", - " (\n", - " optimizers.RidgeMomentum,\n", - " {\n", - " \"num_iterations\": num_iters,\n", - " \"learning_rate\": learning_rate_ridge,\n", - " \"delta\": 0.9,\n", - " \"lam\": lam,\n", - " },\n", - " ),\n", - " (\n", - " optimizers.RidgeAdaGrad,\n", - " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n", - " ),\n", - " (\n", - " optimizers.RidgeRMSProp,\n", - " {\n", - " \"num_iterations\": num_iters,\n", - " \"learning_rate\": learning_rate_ridge,\n", - " \"gamma\": 0.9,\n", - " \"lam\": lam,\n", - " },\n", - " ),\n", - " (\n", - " optimizers.RidgeAdam,\n", - " {\n", - " \"num_iterations\": num_iters,\n", - " \"learning_rate\": learning_rate_ridge,\n", - " \"beta1\": 0.9,\n", - " \"beta2\": 0.999,\n", - " \"lam\": lam,\n", - " },\n", - " ),\n", + " (opt, {**ridge_kwargs, **opt_kwargs})\n", + " for opt, opt_kwargs in zip(optimizers.RIDGE_GD_OPTIMIZERS, optimizer_kwargs)\n", "]\n", "\n", "\n", - "def plot_optimizers(ax, optimizer_list, X, y, ylabel=\"Cost\"):\n", - " for opt_class, params in optimizer_list:\n", - " iter, history = get_cost_history(opt_class, X, y, **params)\n", - " ax.plot(iter, history, label=f\"{opt_class.__name__}\")\n", - " ax.set_xlabel(\"Iteration\")\n", - " ax.set_ylabel(ylabel)\n", - " ax.legend()\n", + "plotting.plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n", + "plotting.plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n", "\n", - "\n", - "plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n", "ax1.set_ylim(bottom=0.465, top=0.505)\n", - "\n", - "plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n", "ax2.set_ylim(bottom=0.4775, top=0.505)\n", "fig.tight_layout()\n", "fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))" @@ -422,7 +352,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -464,7 +394,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -478,6 +408,7 @@ " learning_rate=0.1,\n", " num_iterations=num_epochs * batches_per_epoch,\n", " batch_size=batch_size,\n", + " batches_per_epoch=batches_per_epoch,\n", ")\n", "ridge_gd = optimizers.RidgeGradientDescent(\n", " learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n", @@ -487,6 +418,7 @@ " lam=0.1,\n", " num_iterations=num_epochs * batches_per_epoch,\n", " batch_size=batch_size,\n", + " batches_per_epoch=batches_per_epoch,\n", ")\n", "\n", "x = np.linspace(-1, 1, X_size)\n", @@ -498,50 +430,28 @@ "X_tr, X_te = datamanip.scale_data(X_train, X_test)\n", "y_tr, y_te = datamanip.scale_data(y_train, y_test)\n", "\n", - "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", - "\n", - "for optimizer, label in zip(\n", - " [ols_sgd, ols_gd, ridge_sgd, ridge_gd],\n", - " [\n", - " \"Stochastic GD\",\n", - " \"Gradient Descent\",\n", - " \"Ridge Stochastic GD\",\n", - " \"Ridge Gradient Descent\",\n", - " ],\n", - "):\n", - " start_time = time.time()\n", - " optimizer.fit(X_tr, y_tr)\n", - " time_taken = time.time() - start_time\n", - " cost_history = optimizer._cost_history\n", - " if \"Stochastic\" in label:\n", - " epochs = np.arange(1, num_epochs * batches_per_epoch + 1) / batches_per_epoch\n", - " ls = \"-\"\n", - " else:\n", - " epochs = np.arange(1, num_epochs + 1)\n", - " ls = \"--\"\n", - " c = \"C0\" if \"Ridge\" not in label else \"C1\"\n", - " ax.plot(\n", - " epochs,\n", - " cost_history,\n", - " label=f\"{label} ({time_taken:.2f} s)\",\n", - " linestyle=ls,\n", - " color=c,\n", - " )\n", - "ax.set_xlabel(\"Epoch\")\n", - "ax.set_ylabel(\"Cost\")\n", - "ax.legend()\n", - "# ax.set_ylim(bottom=0.0, top=0.505)\n", - "fig.tight_layout()" + "optimizer_list = [ols_sgd, ols_gd, ridge_sgd, ridge_gd]\n", + "labels = [\n", + " \"Stochastic GD\",\n", + " \"Gradient Descent\",\n", + " \"Ridge Stochastic GD\",\n", + " \"Ridge Gradient Descent\",\n", + "]\n", + "fig, ax = plotting.optimization_performance_evaluation(\n", + " optimizer_list, labels, X_tr, y_tr\n", + ")\n", + "fig.tight_layout()\n", + "fig.savefig(os.path.join(FIG_DIR, \"optimization_performance.pdf\"))" ] }, { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "16", "metadata": {}, "outputs": [], "source": [ - "X_size = 10_000\n", + "X_size = 1_000_000\n", "num_epochs = 1000\n", "batches_per_epoch = 100\n", "batch_size = 128\n", @@ -587,67 +497,52 @@ "optimizers_ols = [\n", " (opt, {**kwargs, **ols_kwargs})\n", " for opt, kwargs in zip(\n", - " [\n", - " optimizers.OLSStochasticGradientDescent,\n", - " optimizers.OLSMomentumSGD,\n", - " optimizers.OLSAdaGradSGD,\n", - " optimizers.OLSRMSPropSGD,\n", - " optimizers.OLSAdamSGD,\n", - " ],\n", + " optimizers.OLS_SGD_OPTIMIZERS,\n", " optimizer_kwargs,\n", " )\n", "]\n", "optimizers_ridge = [\n", " (opt, {**kwargs, **ridge_kwargs})\n", " for opt, kwargs in zip(\n", - " [\n", - " optimizers.RidgeStochasticGradientDescent,\n", - " optimizers.RidgeMomentumSGD,\n", - " optimizers.RidgeAdaGradSGD,\n", - " optimizers.RidgeRMSPropSGD,\n", - " optimizers.RidgeAdamSGD,\n", - " ],\n", + " optimizers.RIDGE_SGD_OPTIMIZERS,\n", " optimizer_kwargs,\n", " )\n", "]\n", "optimizers_lasso = [\n", " (opt, {**kwargs, **lasso_kwargs})\n", " for opt, kwargs in zip(\n", - " [\n", - " optimizers.LASSOStochasticGradientDescent,\n", - " optimizers.LASSOMomentumSGD,\n", - " optimizers.LASSOAdaGradSGD,\n", - " optimizers.LASSORMSPropSGD,\n", - " optimizers.LASSOAdamSGD,\n", - " ],\n", + " optimizers.LASSO_SGD_OPTIMIZERS,\n", " optimizer_kwargs,\n", " )\n", "]\n", "\n", - "fig, axs = plt.subplots(1, 3, figsize=np.array(plotting.get_figsize(0.5)) * 2)\n", - "plot_optimizers(\n", - " axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch (OLS)\"\n", + "labels = [\"SGD\", \"Mom. SGD\", \"AdaGrad\", \"RMSProp\", \"Adam\"]\n", + "\n", + "fig, axs = plt.subplots(1, 3, figsize=plotting.get_figsize(0.5), sharey=True)\n", + "plotting.plot_optimizers(\n", + " axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n", ")\n", - "plot_optimizers(\n", - " axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch (Ridge)\"\n", + "plotting.plot_optimizers(\n", + " axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n", ")\n", - "plot_optimizers(\n", - " axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch (LASSO)\"\n", + "plotting.plot_optimizers(\n", + " axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n", ")\n", "for ax in axs:\n", " ax.set_xlabel(\"Epoch\")\n", - "fig.tight_layout()" + "fig.tight_layout()\n", + "fig.savefig(os.path.join(FIG_DIR, \"stochastic_gradient_descent_convergence.pdf\"))" ] }, { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [], "source": [ "x = np.linspace(-1, 1, 300)\n", - "y = datamanip.noise_data(datamanip.runge_function(x), 0.1)\n", + "y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n", "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", "\n", "polynomial_degrees = np.arange(1, 50)\n", @@ -676,7 +571,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -726,7 +621,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -750,7 +645,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -786,13 +681,14 @@ "ax.set_ylabel(\"Mean Squared Error\")\n", "ax.set_yscale(\"log\")\n", "ax.legend()\n", - "fig.tight_layout()" + "fig.tight_layout()\n", + "fig.savefig(os.path.join(FIG_DIR, \"k_fold_bootstrapping_comparision.pdf\"))" ] }, { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -830,7 +726,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -856,16 +752,15 @@ "\n", "ax.set_xlabel(\"Polynomial Degree\")\n", "ax.set_ylabel(f\"MSE ({k_folds}-fold validation)\")\n", - "# ax.set_yscale(\"log\")\n", - "# ax.set_ylim(0, 2)\n", "ax.legend()\n", - "fig.tight_layout()" + "fig.tight_layout()\n", + "fig.savefig(os.path.join(FIG_DIR, \"kfold_mse_comparison_per_cost_function.pdf\"))" ] }, { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [] diff --git a/src/__init__.py b/src/pyoptim/__init__.py similarity index 100% rename from src/__init__.py rename to src/pyoptim/__init__.py diff --git a/src/datamanip.py b/src/pyoptim/datamanip.py similarity index 100% rename from src/datamanip.py rename to src/pyoptim/datamanip.py diff --git a/src/optimizers.py b/src/pyoptim/optimizers.py similarity index 93% rename from src/optimizers.py rename to src/pyoptim/optimizers.py index e91e32c..e1a6942 100644 --- a/src/optimizers.py +++ b/src/pyoptim/optimizers.py @@ -380,3 +380,42 @@ class RidgeAdamSGD(RidgeAdam, RidgeStochasticGradientDescent): class LASSOAdamSGD(LASSOAdam, LASSOStochasticGradientDescent): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + + +OLS_GD_OPTIMIZERS = [OLSGradientDescent, OLSMomentum, OLSAdaGrad, OLSRMSProp, OLSAdam] +RIDGE_GD_OPTIMIZERS = [ + RidgeGradientDescent, + RidgeMomentum, + RidgeAdaGrad, + RidgeRMSProp, + RidgeAdam, +] +LASSO_GD_OPTIMIZERS = [ + LASSOGradientDescent, + LASSOMomentum, + LASSOAdaGrad, + LASSORMSProp, + LASSOAdam, +] + +OLS_SGD_OPTIMIZERS = [ + OLSStochasticGradientDescent, + OLSMomentumSGD, + OLSAdaGradSGD, + OLSRMSPropSGD, + OLSAdamSGD, +] +RIDGE_SGD_OPTIMIZERS = [ + RidgeStochasticGradientDescent, + RidgeMomentumSGD, + RidgeAdaGradSGD, + RidgeRMSPropSGD, + RidgeAdamSGD, +] +LASSO_SGD_OPTIMIZERS = [ + LASSOStochasticGradientDescent, + LASSOMomentumSGD, + LASSOAdaGradSGD, + LASSORMSPropSGD, + LASSOAdamSGD, +] diff --git a/src/plotting.py b/src/pyoptim/plotting.py similarity index 60% rename from src/plotting.py rename to src/pyoptim/plotting.py index 1bc41d7..de5582d 100644 --- a/src/plotting.py +++ b/src/pyoptim/plotting.py @@ -1,8 +1,11 @@ import matplotlib.pyplot as plt from matplotlib.colors import SymLogNorm import numpy as np +import time -FIG_WIDTH = 6 +from pyoptim.optimizers import GradientDescent + +FIG_WIDTH = 7 def get_rc_params(): @@ -12,12 +15,12 @@ def get_rc_params(): # Setup fonts rcParams["text.usetex"] = True rcParams["font.family"] = "serif" - rcParams["font.size"] = 12 - rcParams["axes.labelsize"] = 12 - rcParams["axes.titlesize"] = 12 - rcParams["legend.fontsize"] = 10 - rcParams["xtick.labelsize"] = 10 - rcParams["ytick.labelsize"] = 10 + rcParams["font.size"] = 10 + rcParams["axes.labelsize"] = 10 + rcParams["axes.titlesize"] = 10 + rcParams["legend.fontsize"] = 8 + rcParams["xtick.labelsize"] = 8 + rcParams["ytick.labelsize"] = 8 # Figure size and resolution rcParams["figure.figsize"] = (FIG_WIDTH, 4) rcParams["figure.dpi"] = 300 @@ -131,3 +134,82 @@ def parameter_plot( ax.set_yticklabels(np.arange(1, beta_OLS.shape[0] + 1, step=5)) fig.tight_layout() return fig, ax + + +def scatter_dataset( + x_train: np.ndarray, x_test: np.ndarray, y_train: np.ndarray, y_test: np.ndarray +) -> tuple[plt.Figure, plt.Axes]: + fig, ax = plt.subplots(figsize=get_figsize(0.5)) + ax.scatter(x_train, y_train, label="Train Data", s=1) + ax.scatter(x_test, y_test, label="Test Data", s=1) + ax.set_xlabel("$x$") + ax.set_ylabel("$y$") + ax.legend() + fig.tight_layout() + return fig, ax + + +def get_cost_history( + optimizer: type[GradientDescent], X: np.ndarray, y: np.ndarray, **kwargs +) -> tuple[np.ndarray, np.ndarray]: + opt = optimizer(**kwargs) + opt.fit(X, y) + iterations = opt.get_epochs() + return iterations, opt.cost_history + + +def plot_optimizers( + ax: plt.Axes, + optimizer_list: list[tuple[type[GradientDescent], dict]], + X: np.ndarray, + y: np.ndarray, + ylabel="Cost", + labels: list | None = None, +) -> None: + for i, (opt_class, params) in enumerate(optimizer_list): + iter, history = get_cost_history(opt_class, X, y, **params) + if labels: + label = labels[i] + else: + label = f"{opt_class.__name__}" + ax.plot(iter, history, label=label) + ax.set_xlabel("Iteration") + ax.set_ylabel(ylabel) + ax.legend() + + +def optimization_performance_evaluation( + optimizers: list[GradientDescent], + labels: list[str], + X_tr: np.ndarray, + y_tr: np.ndarray, +): + fig, ax = plt.subplots(figsize=get_figsize(0.5)) + + for optimizer, label in zip(optimizers, labels): + start_time = time.time() + optimizer.fit(X_tr, y_tr) + time_taken = time.time() - start_time + cost_history = optimizer._cost_history + if "Stochastic" in label: + b_p_e = getattr(optimizer, "batches_per_epoch", 1) + epochs = ( + np.arange(1, optimizer.num_iterations + 1, dtype=np.float64) / b_p_e + ) + ls = "-" + else: + epochs = np.arange(1, optimizer.num_iterations + 1, dtype=np.float64) + ls = "--" + c = "C0" if "Ridge" not in label else "C1" + ax.plot( + epochs, + cost_history, + label=f"{label} ({time_taken:.2f} s)", + linestyle=ls, + color=c, + ) + ax.set_xlabel("Epoch") + ax.set_ylabel("Cost") + ax.legend() + fig.tight_layout() + return fig, ax