lars/FYSSTK-Project1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finish creating and saving all the plots

Browse Source
This commit is contained in:
Lars Bogner
2025-09-10 09:58:24 +02:00
parent 40f7862055
commit f79b4c01f6
19 changed files with 220 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
figures/bias_variance_tradeoff.pdf
BIN
View File
Binary file not shown.
figures/cost_function_comparison.pdf
BIN
View File
Binary file not shown.
figures/data_scatter.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 322 KiB

After

Width:  |  Height:  |  Size: 551 KiB

figures/gradient_descent_convergence.pdf
BIN
View File
Binary file not shown.
figures/k_fold_bootstrapping_comparision.pdf
BIN
View File
Binary file not shown.
figures/kfold_mse_comparison_per_cost_function.pdf
BIN
View File
Binary file not shown.
figures/ols_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ols_parameter_plot.pdf
BIN
View File
Binary file not shown.
figures/optimization_performance.pdf
BIN
View File
Binary file not shown.
figures/optimizer_comparison.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2_lambda.pdf
BIN
View File
Binary file not shown.
figures/ridge_parameter_plot.pdf
BIN
View File
Binary file not shown.
figures/stochastic_gradient_descent_convergence.pdf
BIN
View File
Binary file not shown.
src/main.ipynb
+92 -197
View File
@@ -18,15 +18,14 @@
"metadata": {},
"outputs": [],
"source": [
"import optimizers\n",
"import datamanip\n",
"import plotting\n",
"import pyoptim.optimizers as optimizers\n",
"import pyoptim.datamanip as datamanip\n",
"import pyoptim.plotting as plotting\n",
"\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from sklearn.model_selection import train_test_split\n",
"import os\n",
"import time"
"import os"
]
},
{
@@ -37,7 +36,7 @@
"outputs": [],
"source": [
"FIG_DIR = os.path.abspath(\n",
" os.path.join(os.path.dirname(plotting.__file__), \"..\", \"figures\")\n",
" os.path.join(os.path.dirname(plotting.__file__), \"../..\", \"figures\")\n",
")\n",
"print(FIG_DIR)"
]
@@ -53,12 +52,7 @@
"y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"\n",
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
"ax.scatter(x_train, y_train, label=\"Train Data\", s=1)\n",
"ax.scatter(x_test, y_test, label=\"Test Data\", s=1)\n",
"ax.set_xlabel(\"$x$\")\n",
"ax.set_ylabel(\"$y$\")\n",
"ax.legend()\n",
"fig, ax = plotting.scatter_dataset(x_train, x_test, y_train, y_test)\n",
"fig.savefig(os.path.join(FIG_DIR, \"data_scatter.png\"), dpi=300)"
]
},
@@ -273,29 +267,30 @@
"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5))\n",
"num_iters = 1_000\n",
"\n",
"for learning_rate in np.logspace(-4, 0, 5):\n",
" OLSGrad_Desc = optimizers.OLSGradientDescent(\n",
" learning_rate=learning_rate, num_iterations=num_iters\n",
"learning_rates = np.logspace(-4, 0, 5)\n",
"ols_optimizers = [\n",
" (\n",
" optimizers.OLSGradientDescent,\n",
" {\"learning_rate\": learning_rate, \"num_iterations\": num_iters},\n",
" )\n",
" OLSGrad_Desc.fit(X_tr, y_tr)\n",
" cost_history = OLSGrad_Desc.cost_history\n",
" RidgeGrad_Desc = optimizers.RidgeGradientDescent(\n",
" learning_rate=learning_rate, num_iterations=num_iters, lam=0.1\n",
" for learning_rate in learning_rates\n",
"]\n",
"ridge_optimizers = [\n",
" (\n",
" optimizers.RidgeGradientDescent,\n",
" {\"learning_rate\": learning_rate, \"num_iterations\": num_iters, \"lam\": 0.1},\n",
" )\n",
" RidgeGrad_Desc.fit(X_tr, y_tr)\n",
" cost_history_ridge = RidgeGrad_Desc.cost_history\n",
" ax1.plot(cost_history, label=f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\")\n",
" ax2.plot(\n",
" cost_history_ridge, label=f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\"\n",
" )\n",
"ax1.set_xlabel(\"Iteration\")\n",
"ax1.set_ylabel(\"OLS Cost\")\n",
"ax1.legend()\n",
" for learning_rate in learning_rates\n",
"]\n",
"labels = [\n",
" f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\"\n",
" for learning_rate in learning_rates\n",
"]\n",
"\n",
"plotting.plot_optimizers(ax1, ols_optimizers, X_tr, y_tr, \"OLS Cost\", labels=labels)\n",
"plotting.plot_optimizers(ax2, ridge_optimizers, X_tr, y_tr, \"Ridge Cost\", labels=labels)\n",
"ax1.set_ylim(bottom=0.465, top=0.505)\n",
"ax2.set_ylim(bottom=0.4775, top=0.505)\n",
"ax2.set_xlabel(\"Iteration\")\n",
"ax2.set_ylabel(\"Ridge Cost\")\n",
"ax2.legend()\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"gradient_descent_convergence.pdf\"))"
]
@@ -306,20 +301,6 @@
"id": "13",
"metadata": {},
"outputs": [],
"source": [
"def get_cost_history(optimizer, X, y, **kwargs):\n",
" opt = optimizer(**kwargs)\n",
" opt.fit(X, y)\n",
" iterations = opt.get_epochs()\n",
" return iterations, opt.cost_history"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "14",
"metadata": {},
"outputs": [],
"source": [
"fig, (ax1, ax2) = plt.subplots(2, 1, figsize=plotting.get_figsize(0.8))\n",
"\n",
@@ -333,87 +314,36 @@
"learning_rate_ridge = 0.01\n",
"lam = 0.1\n",
"\n",
"general_kwargs = {\n",
" \"num_iterations\": num_iters,\n",
"}\n",
"optimizer_kwargs = [\n",
" {},\n",
" {\"delta\": 0.9},\n",
" {},\n",
" {\"gamma\": 0.995},\n",
" {\"beta1\": 0.9, \"beta2\": 0.999},\n",
"]\n",
"ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n",
"ridge_kwargs = {\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"lam\": lam,\n",
" **general_kwargs,\n",
"}\n",
"optimizers_ols = [\n",
" (\n",
" optimizers.OLSGradientDescent,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
" ),\n",
" (\n",
" optimizers.OLSMomentum,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"delta\": 0.9},\n",
" ),\n",
" (\n",
" optimizers.OLSAdaGrad,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
" ),\n",
" (\n",
" optimizers.OLSRMSProp,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"gamma\": 0.9},\n",
" ),\n",
" (\n",
" optimizers.OLSAdam,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ols,\n",
" \"beta1\": 0.9,\n",
" \"beta2\": 0.999,\n",
" },\n",
" ),\n",
" (opt, {**ols_kwargs, **opt_kwargs})\n",
" for opt, opt_kwargs in zip(optimizers.OLS_GD_OPTIMIZERS, optimizer_kwargs)\n",
"]\n",
"\n",
"optimizers_ridge = [\n",
" (\n",
" optimizers.RidgeGradientDescent,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
" ),\n",
" (\n",
" optimizers.RidgeMomentum,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"delta\": 0.9,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
" (\n",
" optimizers.RidgeAdaGrad,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
" ),\n",
" (\n",
" optimizers.RidgeRMSProp,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"gamma\": 0.9,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
" (\n",
" optimizers.RidgeAdam,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"beta1\": 0.9,\n",
" \"beta2\": 0.999,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
" (opt, {**ridge_kwargs, **opt_kwargs})\n",
" for opt, opt_kwargs in zip(optimizers.RIDGE_GD_OPTIMIZERS, optimizer_kwargs)\n",
"]\n",
"\n",
"\n",
"def plot_optimizers(ax, optimizer_list, X, y, ylabel=\"Cost\"):\n",
" for opt_class, params in optimizer_list:\n",
" iter, history = get_cost_history(opt_class, X, y, **params)\n",
" ax.plot(iter, history, label=f\"{opt_class.__name__}\")\n",
" ax.set_xlabel(\"Iteration\")\n",
" ax.set_ylabel(ylabel)\n",
" ax.legend()\n",
"plotting.plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n",
"plotting.plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n",
"\n",
"\n",
"plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n",
"ax1.set_ylim(bottom=0.465, top=0.505)\n",
"\n",
"plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n",
"ax2.set_ylim(bottom=0.4775, top=0.505)\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))"
@@ -422,7 +352,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "15",
"id": "14",
"metadata": {},
"outputs": [],
"source": [
@@ -464,7 +394,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "16",
"id": "15",
"metadata": {},
"outputs": [],
"source": [
@@ -478,6 +408,7 @@
" learning_rate=0.1,\n",
" num_iterations=num_epochs * batches_per_epoch,\n",
" batch_size=batch_size,\n",
" batches_per_epoch=batches_per_epoch,\n",
")\n",
"ridge_gd = optimizers.RidgeGradientDescent(\n",
" learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n",
@@ -487,6 +418,7 @@
" lam=0.1,\n",
" num_iterations=num_epochs * batches_per_epoch,\n",
" batch_size=batch_size,\n",
" batches_per_epoch=batches_per_epoch,\n",
")\n",
"\n",
"x = np.linspace(-1, 1, X_size)\n",
@@ -498,50 +430,28 @@
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
"\n",
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
"\n",
"for optimizer, label in zip(\n",
" [ols_sgd, ols_gd, ridge_sgd, ridge_gd],\n",
" [\n",
" \"Stochastic GD\",\n",
" \"Gradient Descent\",\n",
" \"Ridge Stochastic GD\",\n",
" \"Ridge Gradient Descent\",\n",
" ],\n",
"):\n",
" start_time = time.time()\n",
" optimizer.fit(X_tr, y_tr)\n",
" time_taken = time.time() - start_time\n",
" cost_history = optimizer._cost_history\n",
" if \"Stochastic\" in label:\n",
" epochs = np.arange(1, num_epochs * batches_per_epoch + 1) / batches_per_epoch\n",
" ls = \"-\"\n",
" else:\n",
" epochs = np.arange(1, num_epochs + 1)\n",
" ls = \"--\"\n",
" c = \"C0\" if \"Ridge\" not in label else \"C1\"\n",
" ax.plot(\n",
" epochs,\n",
" cost_history,\n",
" label=f\"{label} ({time_taken:.2f} s)\",\n",
" linestyle=ls,\n",
" color=c,\n",
" )\n",
"ax.set_xlabel(\"Epoch\")\n",
"ax.set_ylabel(\"Cost\")\n",
"ax.legend()\n",
"# ax.set_ylim(bottom=0.0, top=0.505)\n",
"fig.tight_layout()"
"optimizer_list = [ols_sgd, ols_gd, ridge_sgd, ridge_gd]\n",
"labels = [\n",
" \"Stochastic GD\",\n",
" \"Gradient Descent\",\n",
" \"Ridge Stochastic GD\",\n",
" \"Ridge Gradient Descent\",\n",
"]\n",
"fig, ax = plotting.optimization_performance_evaluation(\n",
" optimizer_list, labels, X_tr, y_tr\n",
")\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"optimization_performance.pdf\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "17",
"id": "16",
"metadata": {},
"outputs": [],
"source": [
"X_size = 10_000\n",
"X_size = 1_000_000\n",
"num_epochs = 1000\n",
"batches_per_epoch = 100\n",
"batch_size = 128\n",
@@ -587,67 +497,52 @@
"optimizers_ols = [\n",
" (opt, {**kwargs, **ols_kwargs})\n",
" for opt, kwargs in zip(\n",
" [\n",
" optimizers.OLSStochasticGradientDescent,\n",
" optimizers.OLSMomentumSGD,\n",
" optimizers.OLSAdaGradSGD,\n",
" optimizers.OLSRMSPropSGD,\n",
" optimizers.OLSAdamSGD,\n",
" ],\n",
" optimizers.OLS_SGD_OPTIMIZERS,\n",
" optimizer_kwargs,\n",
" )\n",
"]\n",
"optimizers_ridge = [\n",
" (opt, {**kwargs, **ridge_kwargs})\n",
" for opt, kwargs in zip(\n",
" [\n",
" optimizers.RidgeStochasticGradientDescent,\n",
" optimizers.RidgeMomentumSGD,\n",
" optimizers.RidgeAdaGradSGD,\n",
" optimizers.RidgeRMSPropSGD,\n",
" optimizers.RidgeAdamSGD,\n",
" ],\n",
" optimizers.RIDGE_SGD_OPTIMIZERS,\n",
" optimizer_kwargs,\n",
" )\n",
"]\n",
"optimizers_lasso = [\n",
" (opt, {**kwargs, **lasso_kwargs})\n",
" for opt, kwargs in zip(\n",
" [\n",
" optimizers.LASSOStochasticGradientDescent,\n",
" optimizers.LASSOMomentumSGD,\n",
" optimizers.LASSOAdaGradSGD,\n",
" optimizers.LASSORMSPropSGD,\n",
" optimizers.LASSOAdamSGD,\n",
" ],\n",
" optimizers.LASSO_SGD_OPTIMIZERS,\n",
" optimizer_kwargs,\n",
" )\n",
"]\n",
"\n",
"fig, axs = plt.subplots(1, 3, figsize=np.array(plotting.get_figsize(0.5)) * 2)\n",
"plot_optimizers(\n",
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch (OLS)\"\n",
"labels = [\"SGD\", \"Mom. SGD\", \"AdaGrad\", \"RMSProp\", \"Adam\"]\n",
"\n",
"fig, axs = plt.subplots(1, 3, figsize=plotting.get_figsize(0.5), sharey=True)\n",
"plotting.plot_optimizers(\n",
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
")\n",
"plot_optimizers(\n",
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch (Ridge)\"\n",
"plotting.plot_optimizers(\n",
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
")\n",
"plot_optimizers(\n",
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch (LASSO)\"\n",
"plotting.plot_optimizers(\n",
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
")\n",
"for ax in axs:\n",
" ax.set_xlabel(\"Epoch\")\n",
"fig.tight_layout()"
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"stochastic_gradient_descent_convergence.pdf\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18",
"id": "17",
"metadata": {},
"outputs": [],
"source": [
"x = np.linspace(-1, 1, 300)\n",
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1)\n",
"y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"\n",
"polynomial_degrees = np.arange(1, 50)\n",
@@ -676,7 +571,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "19",
"id": "18",
"metadata": {},
"outputs": [],
"source": [
@@ -726,7 +621,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "20",
"id": "19",
"metadata": {},
"outputs": [],
"source": [
@@ -750,7 +645,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "21",
"id": "20",
"metadata": {},
"outputs": [],
"source": [
@@ -786,13 +681,14 @@
"ax.set_ylabel(\"Mean Squared Error\")\n",
"ax.set_yscale(\"log\")\n",
"ax.legend()\n",
"fig.tight_layout()"
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"k_fold_bootstrapping_comparision.pdf\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "22",
"id": "21",
"metadata": {},
"outputs": [],
"source": [
@@ -830,7 +726,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "23",
"id": "22",
"metadata": {},
"outputs": [],
"source": [
@@ -856,16 +752,15 @@
"\n",
"ax.set_xlabel(\"Polynomial Degree\")\n",
"ax.set_ylabel(f\"MSE ({k_folds}-fold validation)\")\n",
"# ax.set_yscale(\"log\")\n",
"# ax.set_ylim(0, 2)\n",
"ax.legend()\n",
"fig.tight_layout()"
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"kfold_mse_comparison_per_cost_function.pdf\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "24",
"id": "23",
"metadata": {},
"outputs": [],
"source": []
src/__init__.py → src/pyoptim/__init__.py
View File
src/datamanip.py → src/pyoptim/datamanip.py
View File
src/optimizers.py → src/pyoptim/optimizers.py
+39
View File
@@ -380,3 +380,42 @@ class RidgeAdamSGD(RidgeAdam, RidgeStochasticGradientDescent):
class LASSOAdamSGD(LASSOAdam, LASSOStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
OLS_GD_OPTIMIZERS = [OLSGradientDescent, OLSMomentum, OLSAdaGrad, OLSRMSProp, OLSAdam]
RIDGE_GD_OPTIMIZERS = [
RidgeGradientDescent,
RidgeMomentum,
RidgeAdaGrad,
RidgeRMSProp,
RidgeAdam,
]
LASSO_GD_OPTIMIZERS = [
LASSOGradientDescent,
LASSOMomentum,
LASSOAdaGrad,
LASSORMSProp,
LASSOAdam,
]
OLS_SGD_OPTIMIZERS = [
OLSStochasticGradientDescent,
OLSMomentumSGD,
OLSAdaGradSGD,
OLSRMSPropSGD,
OLSAdamSGD,
]
RIDGE_SGD_OPTIMIZERS = [
RidgeStochasticGradientDescent,
RidgeMomentumSGD,
RidgeAdaGradSGD,
RidgeRMSPropSGD,
RidgeAdamSGD,
]
LASSO_SGD_OPTIMIZERS = [
LASSOStochasticGradientDescent,
LASSOMomentumSGD,
LASSOAdaGradSGD,
LASSORMSPropSGD,
LASSOAdamSGD,
]
src/plotting.py → src/pyoptim/plotting.py
+89 -7
View File
@@ -1,8 +1,11 @@
import matplotlib.pyplot as plt
from matplotlib.colors import SymLogNorm
import numpy as np
import time
FIG_WIDTH = 6
from pyoptim.optimizers import GradientDescent
FIG_WIDTH = 7
def get_rc_params():
@@ -12,12 +15,12 @@ def get_rc_params():
# Setup fonts
rcParams["text.usetex"] = True
rcParams["font.family"] = "serif"
rcParams["font.size"] = 12
rcParams["axes.labelsize"] = 12
rcParams["axes.titlesize"] = 12
rcParams["legend.fontsize"] = 10
rcParams["xtick.labelsize"] = 10
rcParams["ytick.labelsize"] = 10
rcParams["font.size"] = 10
rcParams["axes.labelsize"] = 10
rcParams["axes.titlesize"] = 10
rcParams["legend.fontsize"] = 8
rcParams["xtick.labelsize"] = 8
rcParams["ytick.labelsize"] = 8
# Figure size and resolution
rcParams["figure.figsize"] = (FIG_WIDTH, 4)
rcParams["figure.dpi"] = 300
@@ -131,3 +134,82 @@ def parameter_plot(
ax.set_yticklabels(np.arange(1, beta_OLS.shape[0] + 1, step=5))
fig.tight_layout()
return fig, ax
def scatter_dataset(
x_train: np.ndarray, x_test: np.ndarray, y_train: np.ndarray, y_test: np.ndarray
) -> tuple[plt.Figure, plt.Axes]:
fig, ax = plt.subplots(figsize=get_figsize(0.5))
ax.scatter(x_train, y_train, label="Train Data", s=1)
ax.scatter(x_test, y_test, label="Test Data", s=1)
ax.set_xlabel("$x$")
ax.set_ylabel("$y$")
ax.legend()
fig.tight_layout()
return fig, ax
def get_cost_history(
optimizer: type[GradientDescent], X: np.ndarray, y: np.ndarray, **kwargs
) -> tuple[np.ndarray, np.ndarray]:
opt = optimizer(**kwargs)
opt.fit(X, y)
iterations = opt.get_epochs()
return iterations, opt.cost_history
def plot_optimizers(
ax: plt.Axes,
optimizer_list: list[tuple[type[GradientDescent], dict]],
X: np.ndarray,
y: np.ndarray,
ylabel="Cost",
labels: list | None = None,
) -> None:
for i, (opt_class, params) in enumerate(optimizer_list):
iter, history = get_cost_history(opt_class, X, y, **params)
if labels:
label = labels[i]
else:
label = f"{opt_class.__name__}"
ax.plot(iter, history, label=label)
ax.set_xlabel("Iteration")
ax.set_ylabel(ylabel)
ax.legend()
def optimization_performance_evaluation(
optimizers: list[GradientDescent],
labels: list[str],
X_tr: np.ndarray,
y_tr: np.ndarray,
):
fig, ax = plt.subplots(figsize=get_figsize(0.5))
for optimizer, label in zip(optimizers, labels):
start_time = time.time()
optimizer.fit(X_tr, y_tr)
time_taken = time.time() - start_time
cost_history = optimizer._cost_history
if "Stochastic" in label:
b_p_e = getattr(optimizer, "batches_per_epoch", 1)
epochs = (
np.arange(1, optimizer.num_iterations + 1, dtype=np.float64) / b_p_e
)
ls = "-"
else:
epochs = np.arange(1, optimizer.num_iterations + 1, dtype=np.float64)
ls = "--"
c = "C0" if "Ridge" not in label else "C1"
ax.plot(
epochs,
cost_history,
label=f"{label} ({time_taken:.2f} s)",
linestyle=ls,
color=c,
)
ax.set_xlabel("Epoch")
ax.set_ylabel("Cost")
ax.legend()
fig.tight_layout()
return fig, ax