Implement sgd and bootstrapping methods. Hyperparameter tuning for the above mentioned.
This commit is contained in:
+310
-16
@@ -25,7 +25,8 @@
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"import os"
|
||||
"import os\n",
|
||||
"import time"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -309,7 +310,8 @@
|
||||
"def get_cost_history(optimizer, X, y, **kwargs):\n",
|
||||
" opt = optimizer(**kwargs)\n",
|
||||
" opt.fit(X, y)\n",
|
||||
" return opt.cost_history"
|
||||
" iterations = opt.get_epochs()\n",
|
||||
" return iterations, opt.cost_history"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -332,33 +334,64 @@
|
||||
"lam = 0.1\n",
|
||||
"\n",
|
||||
"optimizers_ols = [\n",
|
||||
" (optimizers.OLSGradientDescent, {\"learning_rate\": learning_rate_ols}),\n",
|
||||
" (optimizers.OLSMomentum, {\"learning_rate\": learning_rate_ols, \"delta\": 0.9}),\n",
|
||||
" (optimizers.OLSAdaGrad, {\"learning_rate\": learning_rate_ols}),\n",
|
||||
" (optimizers.OLSRMSProp, {\"learning_rate\": learning_rate_ols, \"gamma\": 0.9}),\n",
|
||||
" (\n",
|
||||
" optimizers.OLSGradientDescent,\n",
|
||||
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" optimizers.OLSMomentum,\n",
|
||||
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"delta\": 0.9},\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" optimizers.OLSAdaGrad,\n",
|
||||
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" optimizers.OLSRMSProp,\n",
|
||||
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"gamma\": 0.9},\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" optimizers.OLSAdam,\n",
|
||||
" {\"learning_rate\": learning_rate_ols, \"beta1\": 0.9, \"beta2\": 0.999},\n",
|
||||
" {\n",
|
||||
" \"num_iterations\": num_iters,\n",
|
||||
" \"learning_rate\": learning_rate_ols,\n",
|
||||
" \"beta1\": 0.9,\n",
|
||||
" \"beta2\": 0.999,\n",
|
||||
" },\n",
|
||||
" ),\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"optimizers_ridge = [\n",
|
||||
" (\n",
|
||||
" optimizers.RidgeGradientDescent,\n",
|
||||
" {\"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
|
||||
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" optimizers.RidgeMomentum,\n",
|
||||
" {\"learning_rate\": learning_rate_ridge, \"delta\": 0.9, \"lam\": lam},\n",
|
||||
" {\n",
|
||||
" \"num_iterations\": num_iters,\n",
|
||||
" \"learning_rate\": learning_rate_ridge,\n",
|
||||
" \"delta\": 0.9,\n",
|
||||
" \"lam\": lam,\n",
|
||||
" },\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" optimizers.RidgeAdaGrad,\n",
|
||||
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
|
||||
" ),\n",
|
||||
" (optimizers.RidgeAdaGrad, {\"learning_rate\": learning_rate_ridge, \"lam\": lam}),\n",
|
||||
" (\n",
|
||||
" optimizers.RidgeRMSProp,\n",
|
||||
" {\"learning_rate\": learning_rate_ridge, \"gamma\": 0.9, \"lam\": lam},\n",
|
||||
" {\n",
|
||||
" \"num_iterations\": num_iters,\n",
|
||||
" \"learning_rate\": learning_rate_ridge,\n",
|
||||
" \"gamma\": 0.9,\n",
|
||||
" \"lam\": lam,\n",
|
||||
" },\n",
|
||||
" ),\n",
|
||||
" (\n",
|
||||
" optimizers.RidgeAdam,\n",
|
||||
" {\n",
|
||||
" \"num_iterations\": num_iters,\n",
|
||||
" \"learning_rate\": learning_rate_ridge,\n",
|
||||
" \"beta1\": 0.9,\n",
|
||||
" \"beta2\": 0.999,\n",
|
||||
@@ -368,19 +401,19 @@
|
||||
"]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def plot_optimizers(ax, optimizer_list, X, y, num_iters, ylabel=\"Cost\"):\n",
|
||||
"def plot_optimizers(ax, optimizer_list, X, y, ylabel=\"Cost\"):\n",
|
||||
" for opt_class, params in optimizer_list:\n",
|
||||
" history = get_cost_history(opt_class, X, y, num_iterations=num_iters, **params)\n",
|
||||
" ax.plot(history, label=f\"{opt_class.__name__}\")\n",
|
||||
" iter, history = get_cost_history(opt_class, X, y, **params)\n",
|
||||
" ax.plot(iter, history, label=f\"{opt_class.__name__}\")\n",
|
||||
" ax.set_xlabel(\"Iteration\")\n",
|
||||
" ax.set_ylabel(ylabel)\n",
|
||||
" ax.legend()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, num_iters, ylabel=\"Cost (OLS)\")\n",
|
||||
"plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n",
|
||||
"ax1.set_ylim(bottom=0.465, top=0.505)\n",
|
||||
"\n",
|
||||
"plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, num_iters, ylabel=\"Cost (Ridge)\")\n",
|
||||
"plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n",
|
||||
"ax2.set_ylim(bottom=0.4775, top=0.505)\n",
|
||||
"fig.tight_layout()\n",
|
||||
"fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))"
|
||||
@@ -434,6 +467,267 @@
|
||||
"id": "16",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X_size = 1_000_000\n",
|
||||
"num_epochs = 1000\n",
|
||||
"batches_per_epoch = 100\n",
|
||||
"batch_size = 512\n",
|
||||
"\n",
|
||||
"ols_gd = optimizers.OLSGradientDescent(learning_rate=0.1, num_iterations=num_epochs)\n",
|
||||
"ols_sgd = optimizers.OLSStochasticGradientDescent(\n",
|
||||
" learning_rate=0.1,\n",
|
||||
" num_iterations=num_epochs * batches_per_epoch,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
")\n",
|
||||
"ridge_gd = optimizers.RidgeGradientDescent(\n",
|
||||
" learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n",
|
||||
")\n",
|
||||
"ridge_sgd = optimizers.RidgeStochasticGradientDescent(\n",
|
||||
" learning_rate=0.01,\n",
|
||||
" lam=0.1,\n",
|
||||
" num_iterations=num_epochs * batches_per_epoch,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"x = np.linspace(-1, 1, X_size)\n",
|
||||
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n",
|
||||
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
||||
"\n",
|
||||
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
|
||||
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
|
||||
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
|
||||
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
|
||||
"\n",
|
||||
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
|
||||
"\n",
|
||||
"for optimizer, label in zip(\n",
|
||||
" [ols_sgd, ols_gd, ridge_sgd, ridge_gd],\n",
|
||||
" [\n",
|
||||
" \"Stochastic GD\",\n",
|
||||
" \"Gradient Descent\",\n",
|
||||
" \"Ridge Stochastic GD\",\n",
|
||||
" \"Ridge Gradient Descent\",\n",
|
||||
" ],\n",
|
||||
"):\n",
|
||||
" start_time = time.time()\n",
|
||||
" optimizer.fit(X_tr, y_tr)\n",
|
||||
" time_taken = time.time() - start_time\n",
|
||||
" cost_history = optimizer._cost_history\n",
|
||||
" if \"Stochastic\" in label:\n",
|
||||
" epochs = np.arange(1, num_epochs * batches_per_epoch + 1) / batches_per_epoch\n",
|
||||
" ls = \"-\"\n",
|
||||
" else:\n",
|
||||
" epochs = np.arange(1, num_epochs + 1)\n",
|
||||
" ls = \"--\"\n",
|
||||
" c = \"C0\" if \"Ridge\" not in label else \"C1\"\n",
|
||||
" ax.plot(\n",
|
||||
" epochs,\n",
|
||||
" cost_history,\n",
|
||||
" label=f\"{label} ({time_taken:.2f} s)\",\n",
|
||||
" linestyle=ls,\n",
|
||||
" color=c,\n",
|
||||
" )\n",
|
||||
"ax.set_xlabel(\"Epoch\")\n",
|
||||
"ax.set_ylabel(\"Cost\")\n",
|
||||
"ax.legend()\n",
|
||||
"# ax.set_ylim(bottom=0.0, top=0.505)\n",
|
||||
"fig.tight_layout()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "17",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X_size = 10_000\n",
|
||||
"num_epochs = 1000\n",
|
||||
"batches_per_epoch = 100\n",
|
||||
"batch_size = 128\n",
|
||||
"\n",
|
||||
"learning_rate_ols = 0.1\n",
|
||||
"learning_rate_ridge = 0.001\n",
|
||||
"learning_rate_lasso = 0.001\n",
|
||||
"lam_ridge = 0.1\n",
|
||||
"lam_lasso = 0.1\n",
|
||||
"\n",
|
||||
"x = np.linspace(-1, 1, X_size)\n",
|
||||
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n",
|
||||
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
||||
"\n",
|
||||
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
|
||||
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
|
||||
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
|
||||
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
|
||||
"\n",
|
||||
"general_kwargs = {\n",
|
||||
" \"num_iterations\": num_epochs * batches_per_epoch,\n",
|
||||
" \"batch_size\": batch_size,\n",
|
||||
" \"batches_per_epoch\": batches_per_epoch,\n",
|
||||
"}\n",
|
||||
"optimizer_kwargs = [\n",
|
||||
" {},\n",
|
||||
" {\"delta\": 0.9},\n",
|
||||
" {},\n",
|
||||
" {\"gamma\": 0.9},\n",
|
||||
" {\"beta1\": 0.9, \"beta2\": 0.999},\n",
|
||||
"]\n",
|
||||
"ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n",
|
||||
"ridge_kwargs = {\n",
|
||||
" \"learning_rate\": learning_rate_ridge,\n",
|
||||
" \"lam\": lam_ridge,\n",
|
||||
" **general_kwargs,\n",
|
||||
"}\n",
|
||||
"lasso_kwargs = {\n",
|
||||
" \"learning_rate\": learning_rate_lasso,\n",
|
||||
" \"lam\": lam_lasso,\n",
|
||||
" **general_kwargs,\n",
|
||||
"}\n",
|
||||
"optimizers_ols = [\n",
|
||||
" (opt, {**kwargs, **ols_kwargs})\n",
|
||||
" for opt, kwargs in zip(\n",
|
||||
" [\n",
|
||||
" optimizers.OLSStochasticGradientDescent,\n",
|
||||
" optimizers.OLSMomentumSGD,\n",
|
||||
" optimizers.OLSAdaGradSGD,\n",
|
||||
" optimizers.OLSRMSPropSGD,\n",
|
||||
" optimizers.OLSAdamSGD,\n",
|
||||
" ],\n",
|
||||
" optimizer_kwargs,\n",
|
||||
" )\n",
|
||||
"]\n",
|
||||
"optimizers_ridge = [\n",
|
||||
" (opt, {**kwargs, **ridge_kwargs})\n",
|
||||
" for opt, kwargs in zip(\n",
|
||||
" [\n",
|
||||
" optimizers.RidgeStochasticGradientDescent,\n",
|
||||
" optimizers.RidgeMomentumSGD,\n",
|
||||
" optimizers.RidgeAdaGradSGD,\n",
|
||||
" optimizers.RidgeRMSPropSGD,\n",
|
||||
" optimizers.RidgeAdamSGD,\n",
|
||||
" ],\n",
|
||||
" optimizer_kwargs,\n",
|
||||
" )\n",
|
||||
"]\n",
|
||||
"optimizers_lasso = [\n",
|
||||
" (opt, {**kwargs, **lasso_kwargs})\n",
|
||||
" for opt, kwargs in zip(\n",
|
||||
" [\n",
|
||||
" optimizers.LASSOStochasticGradientDescent,\n",
|
||||
" optimizers.LASSOMomentumSGD,\n",
|
||||
" optimizers.LASSOAdaGradSGD,\n",
|
||||
" optimizers.LASSORMSPropSGD,\n",
|
||||
" optimizers.LASSOAdamSGD,\n",
|
||||
" ],\n",
|
||||
" optimizer_kwargs,\n",
|
||||
" )\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"fig, axs = plt.subplots(1, 3, figsize=np.array(plotting.get_figsize(0.5)) * 2)\n",
|
||||
"plot_optimizers(\n",
|
||||
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch (OLS)\"\n",
|
||||
")\n",
|
||||
"plot_optimizers(\n",
|
||||
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch (Ridge)\"\n",
|
||||
")\n",
|
||||
"plot_optimizers(\n",
|
||||
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch (LASSO)\"\n",
|
||||
")\n",
|
||||
"for ax in axs:\n",
|
||||
" ax.set_xlabel(\"Epoch\")\n",
|
||||
"fig.tight_layout()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "18",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"x = np.linspace(-1, 1, 300)\n",
|
||||
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1)\n",
|
||||
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
||||
"\n",
|
||||
"polynomial_degrees = np.arange(1, 30)\n",
|
||||
"n_bootstraps = len(x_train)\n",
|
||||
"mses = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
|
||||
"biases = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
|
||||
"variances = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
|
||||
"\n",
|
||||
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
|
||||
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
|
||||
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
|
||||
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
|
||||
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
|
||||
"\n",
|
||||
" for b, (X_, y_) in enumerate(\n",
|
||||
" datamanip.bootstrap_resample(X_train_scaled, y_train_scaled, n_bootstraps)\n",
|
||||
" ):\n",
|
||||
" beta = optimizers.Ridge_parameters(X_, y_, lam=1e-10) # approx OLS but stable\n",
|
||||
" y_pred = X_test_scaled @ beta\n",
|
||||
" mse, _ = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
|
||||
" mses[i, b] = mse\n",
|
||||
" biases[i, b] = np.sqrt(np.mean((y_test_scaled - np.mean(y_pred)) ** 2))\n",
|
||||
" variances[i, b] = np.var(y_pred)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "19",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"FILL_BETWEEN = False\n",
|
||||
"fig, axs = plt.subplots(1, 2, figsize=plotting.get_figsize(0.4))\n",
|
||||
"axs[0].plot(polynomial_degrees, np.mean(mses, axis=1), label=\"MSE (Test)\", color=\"C0\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"axs[1].plot(polynomial_degrees, np.mean(biases, axis=1), label=\"Bias$^2$\", color=\"C1\")\n",
|
||||
"axs[1].plot(\n",
|
||||
" polynomial_degrees, np.mean(variances, axis=1), label=\"Variance\", color=\"C2\"\n",
|
||||
")\n",
|
||||
"if FILL_BETWEEN:\n",
|
||||
" axs[0].fill_between(\n",
|
||||
" polynomial_degrees,\n",
|
||||
" np.mean(mses, axis=1) - np.std(mses, axis=1),\n",
|
||||
" np.mean(mses, axis=1) + np.std(mses, axis=1),\n",
|
||||
" color=\"C0\",\n",
|
||||
" alpha=0.3,\n",
|
||||
" )\n",
|
||||
" axs[1].fill_between(\n",
|
||||
" polynomial_degrees,\n",
|
||||
" np.mean(biases, axis=1) - np.std(biases, axis=1),\n",
|
||||
" np.mean(biases, axis=1) + np.std(biases, axis=1),\n",
|
||||
" color=\"C1\",\n",
|
||||
" alpha=0.3,\n",
|
||||
" )\n",
|
||||
" axs[1].fill_between(\n",
|
||||
" polynomial_degrees,\n",
|
||||
" np.mean(variances, axis=1) - np.std(variances, axis=1),\n",
|
||||
" np.mean(variances, axis=1) + np.std(variances, axis=1),\n",
|
||||
" color=\"C2\",\n",
|
||||
" alpha=0.3,\n",
|
||||
" )\n",
|
||||
"axs[0].set_ylabel(\"Mean Squared Error\")\n",
|
||||
"axs[1].set_ylabel(\"Bias and Variance\")\n",
|
||||
"\n",
|
||||
"for ax in axs:\n",
|
||||
" ax.set_xlabel(\"Polynomial Degree\")\n",
|
||||
" ax.legend()\n",
|
||||
"\n",
|
||||
"fig.tight_layout()\n",
|
||||
"fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff.pdf\"))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "20",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user