lars/FYSSTK-Project1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish creating and saving all the plots

Browse Source
This commit is contained in:
lars
2025-09-10 09:58:24 +02:00
parent 40f7862055
commit f79b4c01f6
19 changed files with 220 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
figures/bias_variance_tradeoff.pdf
BIN
View File
Binary file not shown.
figures/cost_function_comparison.pdf
BIN
View File
Binary file not shown.
figures/data_scatter.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 322 KiB

After

Width:  |  Height:  |  Size: 551 KiB

figures/gradient_descent_convergence.pdf
BIN
View File
Binary file not shown.
figures/k_fold_bootstrapping_comparision.pdf
BIN
View File
Binary file not shown.
figures/kfold_mse_comparison_per_cost_function.pdf
BIN
View File
Binary file not shown.
figures/ols_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ols_parameter_plot.pdf
BIN
View File
Binary file not shown.
figures/optimization_performance.pdf
BIN
View File
Binary file not shown.
figures/optimizer_comparison.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2_lambda.pdf
BIN
View File
Binary file not shown.
figures/ridge_parameter_plot.pdf
BIN
View File
Binary file not shown.
figures/stochastic_gradient_descent_convergence.pdf
BIN
View File
Binary file not shown.
src/main.ipynb
+92 -197
View File
@@ -18,15 +18,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"import optimizers\n", "import pyoptim.optimizers as optimizers\n",
"import datamanip\n", "import pyoptim.datamanip as datamanip\n",
"import plotting\n", "import pyoptim.plotting as plotting\n",
"\n", "\n",
"import numpy as np\n", "import numpy as np\n",
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
"from sklearn.model_selection import train_test_split\n", "from sklearn.model_selection import train_test_split\n",
"import os\n", "import os"
"import time"
] ]
}, },
{ {
@@ -37,7 +36,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"FIG_DIR = os.path.abspath(\n", "FIG_DIR = os.path.abspath(\n",
" os.path.join(os.path.dirname(plotting.__file__), \"..\", \"figures\")\n", " os.path.join(os.path.dirname(plotting.__file__), \"../..\", \"figures\")\n",
")\n", ")\n",
"print(FIG_DIR)" "print(FIG_DIR)"
] ]
@@ -53,12 +52,7 @@
"y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n", "y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"\n", "\n",
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", "fig, ax = plotting.scatter_dataset(x_train, x_test, y_train, y_test)\n",
"ax.scatter(x_train, y_train, label=\"Train Data\", s=1)\n",
"ax.scatter(x_test, y_test, label=\"Test Data\", s=1)\n",
"ax.set_xlabel(\"$x$\")\n",
"ax.set_ylabel(\"$y$\")\n",
"ax.legend()\n",
"fig.savefig(os.path.join(FIG_DIR, \"data_scatter.png\"), dpi=300)" "fig.savefig(os.path.join(FIG_DIR, \"data_scatter.png\"), dpi=300)"
] ]
}, },
@@ -273,29 +267,30 @@
"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5))\n", "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5))\n",
"num_iters = 1_000\n", "num_iters = 1_000\n",
"\n", "\n",
"for learning_rate in np.logspace(-4, 0, 5):\n", "learning_rates = np.logspace(-4, 0, 5)\n",
" OLSGrad_Desc = optimizers.OLSGradientDescent(\n", "ols_optimizers = [\n",
" learning_rate=learning_rate, num_iterations=num_iters\n", " (\n",
" optimizers.OLSGradientDescent,\n",
" {\"learning_rate\": learning_rate, \"num_iterations\": num_iters},\n",
" )\n", " )\n",
" OLSGrad_Desc.fit(X_tr, y_tr)\n", " for learning_rate in learning_rates\n",
" cost_history = OLSGrad_Desc.cost_history\n", "]\n",
" RidgeGrad_Desc = optimizers.RidgeGradientDescent(\n", "ridge_optimizers = [\n",
" learning_rate=learning_rate, num_iterations=num_iters, lam=0.1\n", " (\n",
" optimizers.RidgeGradientDescent,\n",
" {\"learning_rate\": learning_rate, \"num_iterations\": num_iters, \"lam\": 0.1},\n",
" )\n", " )\n",
" RidgeGrad_Desc.fit(X_tr, y_tr)\n", " for learning_rate in learning_rates\n",
" cost_history_ridge = RidgeGrad_Desc.cost_history\n", "]\n",
" ax1.plot(cost_history, label=f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\")\n", "labels = [\n",
" ax2.plot(\n", " f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\"\n",
" cost_history_ridge, label=f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\"\n", " for learning_rate in learning_rates\n",
" )\n", "]\n",
"ax1.set_xlabel(\"Iteration\")\n", "\n",
"ax1.set_ylabel(\"OLS Cost\")\n", "plotting.plot_optimizers(ax1, ols_optimizers, X_tr, y_tr, \"OLS Cost\", labels=labels)\n",
"ax1.legend()\n", "plotting.plot_optimizers(ax2, ridge_optimizers, X_tr, y_tr, \"Ridge Cost\", labels=labels)\n",
"ax1.set_ylim(bottom=0.465, top=0.505)\n", "ax1.set_ylim(bottom=0.465, top=0.505)\n",
"ax2.set_ylim(bottom=0.4775, top=0.505)\n", "ax2.set_ylim(bottom=0.4775, top=0.505)\n",
"ax2.set_xlabel(\"Iteration\")\n",
"ax2.set_ylabel(\"Ridge Cost\")\n",
"ax2.legend()\n",
"fig.tight_layout()\n", "fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"gradient_descent_convergence.pdf\"))" "fig.savefig(os.path.join(FIG_DIR, \"gradient_descent_convergence.pdf\"))"
] ]
@@ -306,20 +301,6 @@
"id": "13", "id": "13",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [
"def get_cost_history(optimizer, X, y, **kwargs):\n",
" opt = optimizer(**kwargs)\n",
" opt.fit(X, y)\n",
" iterations = opt.get_epochs()\n",
" return iterations, opt.cost_history"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "14",
"metadata": {},
"outputs": [],
"source": [ "source": [
"fig, (ax1, ax2) = plt.subplots(2, 1, figsize=plotting.get_figsize(0.8))\n", "fig, (ax1, ax2) = plt.subplots(2, 1, figsize=plotting.get_figsize(0.8))\n",
"\n", "\n",
@@ -333,87 +314,36 @@
"learning_rate_ridge = 0.01\n", "learning_rate_ridge = 0.01\n",
"lam = 0.1\n", "lam = 0.1\n",
"\n", "\n",
"general_kwargs = {\n",
" \"num_iterations\": num_iters,\n",
"}\n",
"optimizer_kwargs = [\n",
" {},\n",
" {\"delta\": 0.9},\n",
" {},\n",
" {\"gamma\": 0.995},\n",
" {\"beta1\": 0.9, \"beta2\": 0.999},\n",
"]\n",
"ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n",
"ridge_kwargs = {\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"lam\": lam,\n",
" **general_kwargs,\n",
"}\n",
"optimizers_ols = [\n", "optimizers_ols = [\n",
" (\n", " (opt, {**ols_kwargs, **opt_kwargs})\n",
" optimizers.OLSGradientDescent,\n", " for opt, opt_kwargs in zip(optimizers.OLS_GD_OPTIMIZERS, optimizer_kwargs)\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
" ),\n",
" (\n",
" optimizers.OLSMomentum,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"delta\": 0.9},\n",
" ),\n",
" (\n",
" optimizers.OLSAdaGrad,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
" ),\n",
" (\n",
" optimizers.OLSRMSProp,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"gamma\": 0.9},\n",
" ),\n",
" (\n",
" optimizers.OLSAdam,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ols,\n",
" \"beta1\": 0.9,\n",
" \"beta2\": 0.999,\n",
" },\n",
" ),\n",
"]\n", "]\n",
"\n",
"optimizers_ridge = [\n", "optimizers_ridge = [\n",
" (\n", " (opt, {**ridge_kwargs, **opt_kwargs})\n",
" optimizers.RidgeGradientDescent,\n", " for opt, opt_kwargs in zip(optimizers.RIDGE_GD_OPTIMIZERS, optimizer_kwargs)\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
" ),\n",
" (\n",
" optimizers.RidgeMomentum,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"delta\": 0.9,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
" (\n",
" optimizers.RidgeAdaGrad,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
" ),\n",
" (\n",
" optimizers.RidgeRMSProp,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"gamma\": 0.9,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
" (\n",
" optimizers.RidgeAdam,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"beta1\": 0.9,\n",
" \"beta2\": 0.999,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
"]\n", "]\n",
"\n", "\n",
"\n", "\n",
"def plot_optimizers(ax, optimizer_list, X, y, ylabel=\"Cost\"):\n", "plotting.plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n",
" for opt_class, params in optimizer_list:\n", "plotting.plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n",
" iter, history = get_cost_history(opt_class, X, y, **params)\n",
" ax.plot(iter, history, label=f\"{opt_class.__name__}\")\n",
" ax.set_xlabel(\"Iteration\")\n",
" ax.set_ylabel(ylabel)\n",
" ax.legend()\n",
"\n", "\n",
"\n",
"plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n",
"ax1.set_ylim(bottom=0.465, top=0.505)\n", "ax1.set_ylim(bottom=0.465, top=0.505)\n",
"\n",
"plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n",
"ax2.set_ylim(bottom=0.4775, top=0.505)\n", "ax2.set_ylim(bottom=0.4775, top=0.505)\n",
"fig.tight_layout()\n", "fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))" "fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))"
@@ -422,7 +352,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "15", "id": "14",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -464,7 +394,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "16", "id": "15",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -478,6 +408,7 @@
" learning_rate=0.1,\n", " learning_rate=0.1,\n",
" num_iterations=num_epochs * batches_per_epoch,\n", " num_iterations=num_epochs * batches_per_epoch,\n",
" batch_size=batch_size,\n", " batch_size=batch_size,\n",
" batches_per_epoch=batches_per_epoch,\n",
")\n", ")\n",
"ridge_gd = optimizers.RidgeGradientDescent(\n", "ridge_gd = optimizers.RidgeGradientDescent(\n",
" learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n", " learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n",
@@ -487,6 +418,7 @@
" lam=0.1,\n", " lam=0.1,\n",
" num_iterations=num_epochs * batches_per_epoch,\n", " num_iterations=num_epochs * batches_per_epoch,\n",
" batch_size=batch_size,\n", " batch_size=batch_size,\n",
" batches_per_epoch=batches_per_epoch,\n",
")\n", ")\n",
"\n", "\n",
"x = np.linspace(-1, 1, X_size)\n", "x = np.linspace(-1, 1, X_size)\n",
@@ -498,50 +430,28 @@
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n", "X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n", "y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
"\n", "\n",
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", "optimizer_list = [ols_sgd, ols_gd, ridge_sgd, ridge_gd]\n",
"\n", "labels = [\n",
"for optimizer, label in zip(\n", " \"Stochastic GD\",\n",
" [ols_sgd, ols_gd, ridge_sgd, ridge_gd],\n", " \"Gradient Descent\",\n",
" [\n", " \"Ridge Stochastic GD\",\n",
" \"Stochastic GD\",\n", " \"Ridge Gradient Descent\",\n",
" \"Gradient Descent\",\n", "]\n",
" \"Ridge Stochastic GD\",\n", "fig, ax = plotting.optimization_performance_evaluation(\n",
" \"Ridge Gradient Descent\",\n", " optimizer_list, labels, X_tr, y_tr\n",
" ],\n", ")\n",
"):\n", "fig.tight_layout()\n",
" start_time = time.time()\n", "fig.savefig(os.path.join(FIG_DIR, \"optimization_performance.pdf\"))"
" optimizer.fit(X_tr, y_tr)\n",
" time_taken = time.time() - start_time\n",
" cost_history = optimizer._cost_history\n",
" if \"Stochastic\" in label:\n",
" epochs = np.arange(1, num_epochs * batches_per_epoch + 1) / batches_per_epoch\n",
" ls = \"-\"\n",
" else:\n",
" epochs = np.arange(1, num_epochs + 1)\n",
" ls = \"--\"\n",
" c = \"C0\" if \"Ridge\" not in label else \"C1\"\n",
" ax.plot(\n",
" epochs,\n",
" cost_history,\n",
" label=f\"{label} ({time_taken:.2f} s)\",\n",
" linestyle=ls,\n",
" color=c,\n",
" )\n",
"ax.set_xlabel(\"Epoch\")\n",
"ax.set_ylabel(\"Cost\")\n",
"ax.legend()\n",
"# ax.set_ylim(bottom=0.0, top=0.505)\n",
"fig.tight_layout()"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "17", "id": "16",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"X_size = 10_000\n", "X_size = 1_000_000\n",
"num_epochs = 1000\n", "num_epochs = 1000\n",
"batches_per_epoch = 100\n", "batches_per_epoch = 100\n",
"batch_size = 128\n", "batch_size = 128\n",
@@ -587,67 +497,52 @@
"optimizers_ols = [\n", "optimizers_ols = [\n",
" (opt, {**kwargs, **ols_kwargs})\n", " (opt, {**kwargs, **ols_kwargs})\n",
" for opt, kwargs in zip(\n", " for opt, kwargs in zip(\n",
" [\n", " optimizers.OLS_SGD_OPTIMIZERS,\n",
" optimizers.OLSStochasticGradientDescent,\n",
" optimizers.OLSMomentumSGD,\n",
" optimizers.OLSAdaGradSGD,\n",
" optimizers.OLSRMSPropSGD,\n",
" optimizers.OLSAdamSGD,\n",
" ],\n",
" optimizer_kwargs,\n", " optimizer_kwargs,\n",
" )\n", " )\n",
"]\n", "]\n",
"optimizers_ridge = [\n", "optimizers_ridge = [\n",
" (opt, {**kwargs, **ridge_kwargs})\n", " (opt, {**kwargs, **ridge_kwargs})\n",
" for opt, kwargs in zip(\n", " for opt, kwargs in zip(\n",
" [\n", " optimizers.RIDGE_SGD_OPTIMIZERS,\n",
" optimizers.RidgeStochasticGradientDescent,\n",
" optimizers.RidgeMomentumSGD,\n",
" optimizers.RidgeAdaGradSGD,\n",
" optimizers.RidgeRMSPropSGD,\n",
" optimizers.RidgeAdamSGD,\n",
" ],\n",
" optimizer_kwargs,\n", " optimizer_kwargs,\n",
" )\n", " )\n",
"]\n", "]\n",
"optimizers_lasso = [\n", "optimizers_lasso = [\n",
" (opt, {**kwargs, **lasso_kwargs})\n", " (opt, {**kwargs, **lasso_kwargs})\n",
" for opt, kwargs in zip(\n", " for opt, kwargs in zip(\n",
" [\n", " optimizers.LASSO_SGD_OPTIMIZERS,\n",
" optimizers.LASSOStochasticGradientDescent,\n",
" optimizers.LASSOMomentumSGD,\n",
" optimizers.LASSOAdaGradSGD,\n",
" optimizers.LASSORMSPropSGD,\n",
" optimizers.LASSOAdamSGD,\n",
" ],\n",
" optimizer_kwargs,\n", " optimizer_kwargs,\n",
" )\n", " )\n",
"]\n", "]\n",
"\n", "\n",
"fig, axs = plt.subplots(1, 3, figsize=np.array(plotting.get_figsize(0.5)) * 2)\n", "labels = [\"SGD\", \"Mom. SGD\", \"AdaGrad\", \"RMSProp\", \"Adam\"]\n",
"plot_optimizers(\n", "\n",
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch (OLS)\"\n", "fig, axs = plt.subplots(1, 3, figsize=plotting.get_figsize(0.5), sharey=True)\n",
"plotting.plot_optimizers(\n",
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
")\n", ")\n",
"plot_optimizers(\n", "plotting.plot_optimizers(\n",
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch (Ridge)\"\n", " axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
")\n", ")\n",
"plot_optimizers(\n", "plotting.plot_optimizers(\n",
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch (LASSO)\"\n", " axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
")\n", ")\n",
"for ax in axs:\n", "for ax in axs:\n",
" ax.set_xlabel(\"Epoch\")\n", " ax.set_xlabel(\"Epoch\")\n",
"fig.tight_layout()" "fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"stochastic_gradient_descent_convergence.pdf\"))"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "18", "id": "17",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"x = np.linspace(-1, 1, 300)\n", "x = np.linspace(-1, 1, 300)\n",
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1)\n", "y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"\n", "\n",
"polynomial_degrees = np.arange(1, 50)\n", "polynomial_degrees = np.arange(1, 50)\n",
@@ -676,7 +571,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "19", "id": "18",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -726,7 +621,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "20", "id": "19",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -750,7 +645,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "21", "id": "20",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -786,13 +681,14 @@
"ax.set_ylabel(\"Mean Squared Error\")\n", "ax.set_ylabel(\"Mean Squared Error\")\n",
"ax.set_yscale(\"log\")\n", "ax.set_yscale(\"log\")\n",
"ax.legend()\n", "ax.legend()\n",
"fig.tight_layout()" "fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"k_fold_bootstrapping_comparision.pdf\"))"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "22", "id": "21",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -830,7 +726,7 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "23", "id": "22",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -856,16 +752,15 @@
"\n", "\n",
"ax.set_xlabel(\"Polynomial Degree\")\n", "ax.set_xlabel(\"Polynomial Degree\")\n",
"ax.set_ylabel(f\"MSE ({k_folds}-fold validation)\")\n", "ax.set_ylabel(f\"MSE ({k_folds}-fold validation)\")\n",
"# ax.set_yscale(\"log\")\n",
"# ax.set_ylim(0, 2)\n",
"ax.legend()\n", "ax.legend()\n",
"fig.tight_layout()" "fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"kfold_mse_comparison_per_cost_function.pdf\"))"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"id": "24", "id": "23",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [] "source": []
src/__init__.py → src/pyoptim/__init__.py
View File
src/datamanip.py → src/pyoptim/datamanip.py
View File
src/optimizers.py → src/pyoptim/optimizers.py
+39
View File
@@ -380,3 +380,42 @@ class RidgeAdamSGD(RidgeAdam, RidgeStochasticGradientDescent):
class LASSOAdamSGD(LASSOAdam, LASSOStochasticGradientDescent): class LASSOAdamSGD(LASSOAdam, LASSOStochasticGradientDescent):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
OLS_GD_OPTIMIZERS = [OLSGradientDescent, OLSMomentum, OLSAdaGrad, OLSRMSProp, OLSAdam]
RIDGE_GD_OPTIMIZERS = [
RidgeGradientDescent,
RidgeMomentum,
RidgeAdaGrad,
RidgeRMSProp,
RidgeAdam,
]
LASSO_GD_OPTIMIZERS = [
LASSOGradientDescent,
LASSOMomentum,
LASSOAdaGrad,
LASSORMSProp,
LASSOAdam,
]
OLS_SGD_OPTIMIZERS = [
OLSStochasticGradientDescent,
OLSMomentumSGD,
OLSAdaGradSGD,
OLSRMSPropSGD,
OLSAdamSGD,
]
RIDGE_SGD_OPTIMIZERS = [
RidgeStochasticGradientDescent,
RidgeMomentumSGD,
RidgeAdaGradSGD,
RidgeRMSPropSGD,
RidgeAdamSGD,
]
LASSO_SGD_OPTIMIZERS = [
LASSOStochasticGradientDescent,
LASSOMomentumSGD,
LASSOAdaGradSGD,
LASSORMSPropSGD,
LASSOAdamSGD,
]
src/plotting.py → src/pyoptim/plotting.py
+89 -7
View File
@@ -1,8 +1,11 @@
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from matplotlib.colors import SymLogNorm from matplotlib.colors import SymLogNorm
import numpy as np import numpy as np
import time
FIG_WIDTH = 6 from pyoptim.optimizers import GradientDescent
FIG_WIDTH = 7
def get_rc_params(): def get_rc_params():
@@ -12,12 +15,12 @@ def get_rc_params():
# Setup fonts # Setup fonts
rcParams["text.usetex"] = True rcParams["text.usetex"] = True
rcParams["font.family"] = "serif" rcParams["font.family"] = "serif"
rcParams["font.size"] = 12 rcParams["font.size"] = 10
rcParams["axes.labelsize"] = 12 rcParams["axes.labelsize"] = 10
rcParams["axes.titlesize"] = 12 rcParams["axes.titlesize"] = 10
rcParams["legend.fontsize"] = 10 rcParams["legend.fontsize"] = 8
rcParams["xtick.labelsize"] = 10 rcParams["xtick.labelsize"] = 8
rcParams["ytick.labelsize"] = 10 rcParams["ytick.labelsize"] = 8
# Figure size and resolution # Figure size and resolution
rcParams["figure.figsize"] = (FIG_WIDTH, 4) rcParams["figure.figsize"] = (FIG_WIDTH, 4)
rcParams["figure.dpi"] = 300 rcParams["figure.dpi"] = 300
@@ -131,3 +134,82 @@ def parameter_plot(
ax.set_yticklabels(np.arange(1, beta_OLS.shape[0] + 1, step=5)) ax.set_yticklabels(np.arange(1, beta_OLS.shape[0] + 1, step=5))
fig.tight_layout() fig.tight_layout()
return fig, ax return fig, ax
def scatter_dataset(
x_train: np.ndarray, x_test: np.ndarray, y_train: np.ndarray, y_test: np.ndarray
) -> tuple[plt.Figure, plt.Axes]:
fig, ax = plt.subplots(figsize=get_figsize(0.5))
ax.scatter(x_train, y_train, label="Train Data", s=1)
ax.scatter(x_test, y_test, label="Test Data", s=1)
ax.set_xlabel("$x$")
ax.set_ylabel("$y$")
ax.legend()
fig.tight_layout()
return fig, ax
def get_cost_history(
optimizer: type[GradientDescent], X: np.ndarray, y: np.ndarray, **kwargs
) -> tuple[np.ndarray, np.ndarray]:
opt = optimizer(**kwargs)
opt.fit(X, y)
iterations = opt.get_epochs()
return iterations, opt.cost_history
def plot_optimizers(
ax: plt.Axes,
optimizer_list: list[tuple[type[GradientDescent], dict]],
X: np.ndarray,
y: np.ndarray,
ylabel="Cost",
labels: list | None = None,
) -> None:
for i, (opt_class, params) in enumerate(optimizer_list):
iter, history = get_cost_history(opt_class, X, y, **params)
if labels:
label = labels[i]
else:
label = f"{opt_class.__name__}"
ax.plot(iter, history, label=label)
ax.set_xlabel("Iteration")
ax.set_ylabel(ylabel)
ax.legend()
def optimization_performance_evaluation(
optimizers: list[GradientDescent],
labels: list[str],
X_tr: np.ndarray,
y_tr: np.ndarray,
):
fig, ax = plt.subplots(figsize=get_figsize(0.5))
for optimizer, label in zip(optimizers, labels):
start_time = time.time()
optimizer.fit(X_tr, y_tr)
time_taken = time.time() - start_time
cost_history = optimizer._cost_history
if "Stochastic" in label:
b_p_e = getattr(optimizer, "batches_per_epoch", 1)
epochs = (
np.arange(1, optimizer.num_iterations + 1, dtype=np.float64) / b_p_e
)
ls = "-"
else:
epochs = np.arange(1, optimizer.num_iterations + 1, dtype=np.float64)
ls = "--"
c = "C0" if "Ridge" not in label else "C1"
ax.plot(
epochs,
cost_history,
label=f"{label} ({time_taken:.2f} s)",
linestyle=ls,
color=c,
)
ax.set_xlabel("Epoch")
ax.set_ylabel("Cost")
ax.legend()
fig.tight_layout()
return fig, ax