878 lines
30 KiB
Plaintext
878 lines
30 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "0",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%load_ext autoreload\n",
|
|
"%autoreload 2"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "1",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pyoptim.optimizers as optimizers\n",
|
|
"import pyoptim.datamanip as datamanip\n",
|
|
"import pyoptim.plotting as plotting\n",
|
|
"\n",
|
|
"import numpy as np\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"from sklearn.model_selection import train_test_split\n",
|
|
"import os"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "2",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"FIG_DIR = os.path.abspath(\n",
|
|
" os.path.join(os.path.dirname(plotting.__file__), \"../..\", \"figures\")\n",
|
|
")\n",
|
|
"print(FIG_DIR)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "3",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"x = np.linspace(-1, 1, 100_000)\n",
|
|
"y = datamanip.noise_data(datamanip.runge_function(x), 1.0)\n",
|
|
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
|
"\n",
|
|
"fig, ax = plotting.scatter_dataset(x_train, x_test, y_train, y_test)\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"data_scatter.png\"), dpi=300)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "4",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"beta_OLS_list = []\n",
|
|
"mse_list = []\n",
|
|
"train_mse_list = []\n",
|
|
"r2_list = []\n",
|
|
"train_r2_list = []\n",
|
|
"\n",
|
|
"polynomial_degrees = np.arange(1, 21)\n",
|
|
"\n",
|
|
"for polynomial_degree in polynomial_degrees:\n",
|
|
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
|
|
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
|
|
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
|
|
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
" beta = optimizers.OLS_parameters(X_train_scaled, y_train_scaled)\n",
|
|
"\n",
|
|
" y_pred = X_test_scaled @ beta\n",
|
|
" mse, r2 = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
|
|
" y_train_pred = X_train_scaled @ beta\n",
|
|
" train_mse, train_r2 = datamanip.evaluate_model(y_train_scaled, y_train_pred)\n",
|
|
"\n",
|
|
" train_mse_list.append(train_mse)\n",
|
|
" train_r2_list.append(train_r2)\n",
|
|
" beta_OLS_list.append(beta)\n",
|
|
" mse_list.append(mse)\n",
|
|
" r2_list.append(r2)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "5",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plotting.mse_r2_plot(\n",
|
|
" polynomial_degrees, train_mse_list, mse_list, train_r2_list, r2_list\n",
|
|
")\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"ols_mse_r2.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "6",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plotting.parameter_plot(polynomial_degrees, beta_OLS_list)\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"ols_parameter_plot.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "7",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"beta_ridge_list = []\n",
|
|
"mse_ridge_list = []\n",
|
|
"train_mse_ridge_list = []\n",
|
|
"r2_ridge_list = []\n",
|
|
"train_r2_ridge_list = []\n",
|
|
"\n",
|
|
"lambda_values = [0.5]\n",
|
|
"\n",
|
|
"for polynomial_degree in polynomial_degrees:\n",
|
|
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
|
|
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
|
|
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
|
|
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
" for lambda_ in lambda_values:\n",
|
|
" beta = optimizers.Ridge_parameters(X_train_scaled, y_train_scaled, lambda_)\n",
|
|
"\n",
|
|
" y_pred = X_test_scaled @ beta\n",
|
|
" mse, r2 = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
|
|
" y_train_pred = X_train_scaled @ beta\n",
|
|
" train_mse, train_r2 = datamanip.evaluate_model(y_train_scaled, y_train_pred)\n",
|
|
"\n",
|
|
" train_mse_ridge_list.append(train_mse)\n",
|
|
" train_r2_ridge_list.append(train_r2)\n",
|
|
" beta_ridge_list.append(beta)\n",
|
|
" mse_ridge_list.append(mse)\n",
|
|
" r2_ridge_list.append(r2)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "8",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plotting.mse_r2_plot(\n",
|
|
" polynomial_degrees,\n",
|
|
" train_mse_ridge_list,\n",
|
|
" mse_ridge_list,\n",
|
|
" train_r2_ridge_list,\n",
|
|
" r2_ridge_list,\n",
|
|
")\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"ridge_mse_r2.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "9",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"beta_ridge_list = []\n",
|
|
"mse_ridge_list = []\n",
|
|
"train_mse_ridge_list = []\n",
|
|
"r2_ridge_list = []\n",
|
|
"train_r2_ridge_list = []\n",
|
|
"\n",
|
|
"lambda_values = np.logspace(-5, 3, 20)\n",
|
|
"polynomial_degrees = np.array([10])\n",
|
|
"\n",
|
|
"for polynomial_degree in polynomial_degrees:\n",
|
|
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
|
|
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
|
|
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
|
|
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
" for lambda_ in lambda_values:\n",
|
|
" beta = optimizers.Ridge_parameters(X_train_scaled, y_train_scaled, lambda_)\n",
|
|
"\n",
|
|
" y_pred = X_test_scaled @ beta\n",
|
|
" mse, r2 = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
|
|
" y_train_pred = X_train_scaled @ beta\n",
|
|
" train_mse, train_r2 = datamanip.evaluate_model(y_train_scaled, y_train_pred)\n",
|
|
"\n",
|
|
" train_mse_ridge_list.append(train_mse)\n",
|
|
" train_r2_ridge_list.append(train_r2)\n",
|
|
" beta_ridge_list.append(beta)\n",
|
|
" mse_ridge_list.append(mse)\n",
|
|
" r2_ridge_list.append(r2)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "10",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, axs = plotting.mse_r2_plot(\n",
|
|
" lambda_values,\n",
|
|
" train_mse_ridge_list,\n",
|
|
" mse_ridge_list,\n",
|
|
" train_r2_ridge_list,\n",
|
|
" r2_ridge_list,\n",
|
|
" labels={\"xlabel\": \"$\\\\lambda$\"},\n",
|
|
")\n",
|
|
"for ax in axs:\n",
|
|
" ax.set_xscale(\"log\")\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"ridge_mse_r2_lambda.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "11",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plotting.parameter_plot(\n",
|
|
" lambda_values, beta_ridge_list, labels={\"xlabel\": \"$\\\\lambda$\"}\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"# Set xtick labels to scientific notation\n",
|
|
"def format_number(num: float) -> str:\n",
|
|
" if num == 0:\n",
|
|
" return \"$0$\"\n",
|
|
" exponent = int(np.floor(np.log10(abs(num))))\n",
|
|
" coefficient = num / 10**exponent\n",
|
|
" return f\"${coefficient:.0f}\\\\cdot10^{{{exponent}}}$\"\n",
|
|
"\n",
|
|
"\n",
|
|
"ax.set_xticklabels(\n",
|
|
" [f\"${format_number(tick)}$\" for tick in lambda_values[::3]], rotation=45\n",
|
|
")\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"ridge_parameter_plot.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "12",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"lambda_values = np.logspace(-5, 3, 60)\n",
|
|
"polynomial_degrees = np.arange(1, 31, dtype=int)\n",
|
|
"\n",
|
|
"test_mse_ridge_list = np.zeros((len(polynomial_degrees), len(lambda_values)))\n",
|
|
"\n",
|
|
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
|
|
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
|
|
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
|
|
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
|
|
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
" for j, lambda_ in enumerate(lambda_values):\n",
|
|
" beta = optimizers.Ridge_parameters(X_train_scaled, y_train_scaled, lambda_)\n",
|
|
"\n",
|
|
" y_pred = X_test_scaled @ beta\n",
|
|
" mse, r2 = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
|
|
" test_mse_ridge_list[i, j] = mse"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "13",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
|
|
"c = ax.pcolormesh(\n",
|
|
" lambda_values,\n",
|
|
" polynomial_degrees,\n",
|
|
" test_mse_ridge_list,\n",
|
|
" shading=\"auto\",\n",
|
|
" cmap=\"inferno\",\n",
|
|
")\n",
|
|
"fig.colorbar(c, ax=ax, label=\"Test MSE\")\n",
|
|
"ax.set_xscale(\"log\")\n",
|
|
"ax.set_xlabel(\"$\\\\lambda$\")\n",
|
|
"ax.set_ylabel(\"Polynomial Degree\")\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"ridge_mse_heatmap.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "14",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
|
|
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
|
|
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
|
|
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5))\n",
|
|
"num_iters = 1_000\n",
|
|
"\n",
|
|
"learning_rates = np.logspace(-4, 0, 5)\n",
|
|
"ols_optimizers = [\n",
|
|
" (\n",
|
|
" optimizers.OLSGradientDescent,\n",
|
|
" {\"learning_rate\": learning_rate, \"num_iterations\": num_iters},\n",
|
|
" )\n",
|
|
" for learning_rate in learning_rates\n",
|
|
"]\n",
|
|
"ridge_optimizers = [\n",
|
|
" (\n",
|
|
" optimizers.RidgeGradientDescent,\n",
|
|
" {\"learning_rate\": learning_rate, \"num_iterations\": num_iters, \"lam\": 0.1},\n",
|
|
" )\n",
|
|
" for learning_rate in learning_rates\n",
|
|
"]\n",
|
|
"labels = [\n",
|
|
" f\"$\\\\eta = 10^{{{int(np.log10(learning_rate))}}}$\"\n",
|
|
" for learning_rate in learning_rates\n",
|
|
"]\n",
|
|
"\n",
|
|
"plotting.plot_optimizers(ax1, ols_optimizers, X_tr, y_tr, \"OLS Cost\", labels=labels)\n",
|
|
"plotting.plot_optimizers(ax2, ridge_optimizers, X_tr, y_tr, \"Ridge Cost\", labels=labels)\n",
|
|
"ax1.set_ylim(bottom=0.465, top=0.505)\n",
|
|
"ax2.set_ylim(bottom=0.4775, top=0.505)\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"gradient_descent_convergence.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "15",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, (ax1, ax2) = plt.subplots(2, 1, figsize=plotting.get_figsize(0.8))\n",
|
|
"\n",
|
|
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
|
|
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
|
|
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
|
|
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
"num_iters = 250\n",
|
|
"learning_rate_ols = 0.1\n",
|
|
"learning_rate_ridge = 0.01\n",
|
|
"lam = 0.1\n",
|
|
"\n",
|
|
"general_kwargs = {\n",
|
|
" \"num_iterations\": num_iters,\n",
|
|
"}\n",
|
|
"optimizer_kwargs = [\n",
|
|
" {},\n",
|
|
" {\"delta\": 0.9},\n",
|
|
" {},\n",
|
|
" {\"gamma\": 0.995},\n",
|
|
" {\"beta1\": 0.9, \"beta2\": 0.999},\n",
|
|
"]\n",
|
|
"ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n",
|
|
"ridge_kwargs = {\n",
|
|
" \"learning_rate\": learning_rate_ridge,\n",
|
|
" \"lam\": lam,\n",
|
|
" **general_kwargs,\n",
|
|
"}\n",
|
|
"optimizers_ols = [\n",
|
|
" (opt, {**ols_kwargs, **opt_kwargs})\n",
|
|
" for opt, opt_kwargs in zip(optimizers.OLS_GD_OPTIMIZERS, optimizer_kwargs)\n",
|
|
"]\n",
|
|
"optimizers_ridge = [\n",
|
|
" (opt, {**ridge_kwargs, **opt_kwargs})\n",
|
|
" for opt, opt_kwargs in zip(optimizers.RIDGE_GD_OPTIMIZERS, optimizer_kwargs)\n",
|
|
"]\n",
|
|
"\n",
|
|
"\n",
|
|
"plotting.plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n",
|
|
"plotting.plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n",
|
|
"\n",
|
|
"ax1.set_ylim(bottom=0.465, top=0.505)\n",
|
|
"ax2.set_ylim(bottom=0.4775, top=0.505)\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "16",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
|
|
"\n",
|
|
"ols_adam = optimizers.OLSAdam(learning_rate=0.01, beta1=0.9, beta2=0.999)\n",
|
|
"ridge_adam = optimizers.RidgeAdam(learning_rate=0.01, beta1=0.9, beta2=0.999, lam=0.1)\n",
|
|
"lasso_adam = optimizers.LASSOAdam(learning_rate=0.01, beta1=0.9, beta2=0.999, lam=0.1)\n",
|
|
"ols_gd = optimizers.OLSGradientDescent(learning_rate=0.1)\n",
|
|
"ridge_gd = optimizers.RidgeGradientDescent(learning_rate=0.01, lam=0.1)\n",
|
|
"lasso_gd = optimizers.LASSOGradientDescent(learning_rate=0.01, lam=0.1)\n",
|
|
"\n",
|
|
"for i, (optimizer, label) in enumerate(\n",
|
|
" zip(\n",
|
|
" [ols_adam, ridge_adam, lasso_adam, ols_gd, ridge_gd, lasso_gd],\n",
|
|
" [\"OLS Adam\", \"Ridge Adam\", \"LASSO Adam\", \"OLS GD\", \"Ridge GD\", \"LASSO GD\"],\n",
|
|
" )\n",
|
|
"):\n",
|
|
" optimizer.fit(X_tr, y_tr)\n",
|
|
" cost_history = optimizer.cost_history\n",
|
|
" ls = \"-\" if i < 3 else \"--\"\n",
|
|
" c = \"C\" + str(i % 3)\n",
|
|
" label = label.split()[0] if i < 3 else \"\"\n",
|
|
" ax.plot(cost_history, label=label, linestyle=ls, color=c)\n",
|
|
"ax.set_xlabel(\"Iteration\")\n",
|
|
"ax.set_ylabel(\"Cost\")\n",
|
|
"# Add linestyles manually to legend\n",
|
|
"handles, labels = ax.get_legend_handles_labels()\n",
|
|
"handles += [\n",
|
|
" plt.Line2D([0], [0], color=\"k\", linestyle=\"-\"),\n",
|
|
" plt.Line2D([0], [0], color=\"k\", linestyle=\"--\"),\n",
|
|
"]\n",
|
|
"labels += [\"Adam Opt.\", \"Grad. Desc.\"]\n",
|
|
"ax.legend(handles, labels)\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"cost_function_comparison.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "17",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"X_size = 1_000_000\n",
|
|
"num_epochs = 1000\n",
|
|
"batches_per_epoch = 100\n",
|
|
"batch_size = 512\n",
|
|
"\n",
|
|
"ols_gd = optimizers.OLSGradientDescent(learning_rate=0.1, num_iterations=num_epochs)\n",
|
|
"ols_sgd = optimizers.OLSStochasticGradientDescent(\n",
|
|
" learning_rate=0.1,\n",
|
|
" num_iterations=num_epochs * batches_per_epoch,\n",
|
|
" batch_size=batch_size,\n",
|
|
" batches_per_epoch=batches_per_epoch,\n",
|
|
")\n",
|
|
"ridge_gd = optimizers.RidgeGradientDescent(\n",
|
|
" learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n",
|
|
")\n",
|
|
"ridge_sgd = optimizers.RidgeStochasticGradientDescent(\n",
|
|
" learning_rate=0.01,\n",
|
|
" lam=0.1,\n",
|
|
" num_iterations=num_epochs * batches_per_epoch,\n",
|
|
" batch_size=batch_size,\n",
|
|
" batches_per_epoch=batches_per_epoch,\n",
|
|
")\n",
|
|
"\n",
|
|
"x = np.linspace(-1, 1, X_size)\n",
|
|
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n",
|
|
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
|
"\n",
|
|
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
|
|
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
|
|
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
|
|
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
"optimizer_list = [ols_sgd, ols_gd, ridge_sgd, ridge_gd]\n",
|
|
"labels = [\n",
|
|
" \"Stochastic GD\",\n",
|
|
" \"Gradient Descent\",\n",
|
|
" \"Ridge Stochastic GD\",\n",
|
|
" \"Ridge Gradient Descent\",\n",
|
|
"]\n",
|
|
"fig, ax = plotting.optimization_performance_evaluation(\n",
|
|
" optimizer_list, labels, X_tr, y_tr\n",
|
|
")\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"optimization_performance.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "18",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"X_size = 1_000_000\n",
|
|
"num_epochs = 1000\n",
|
|
"batches_per_epoch = 100\n",
|
|
"batch_size = 128\n",
|
|
"\n",
|
|
"learning_rate_ols = 0.1\n",
|
|
"learning_rate_ridge = 0.001\n",
|
|
"learning_rate_lasso = 0.001\n",
|
|
"lam_ridge = 0.1\n",
|
|
"lam_lasso = 0.1\n",
|
|
"\n",
|
|
"x = np.linspace(-1, 1, X_size)\n",
|
|
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n",
|
|
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
|
"\n",
|
|
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
|
|
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
|
|
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
|
|
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
"general_kwargs = {\n",
|
|
" \"num_iterations\": num_epochs * batches_per_epoch,\n",
|
|
" \"batch_size\": batch_size,\n",
|
|
" \"batches_per_epoch\": batches_per_epoch,\n",
|
|
"}\n",
|
|
"optimizer_kwargs = [\n",
|
|
" {},\n",
|
|
" {\"delta\": 0.9},\n",
|
|
" {},\n",
|
|
" {\"gamma\": 0.9},\n",
|
|
" {\"beta1\": 0.9, \"beta2\": 0.999},\n",
|
|
"]\n",
|
|
"ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n",
|
|
"ridge_kwargs = {\n",
|
|
" \"learning_rate\": learning_rate_ridge,\n",
|
|
" \"lam\": lam_ridge,\n",
|
|
" **general_kwargs,\n",
|
|
"}\n",
|
|
"lasso_kwargs = {\n",
|
|
" \"learning_rate\": learning_rate_lasso,\n",
|
|
" \"lam\": lam_lasso,\n",
|
|
" **general_kwargs,\n",
|
|
"}\n",
|
|
"optimizers_ols = [\n",
|
|
" (opt, {**kwargs, **ols_kwargs})\n",
|
|
" for opt, kwargs in zip(\n",
|
|
" optimizers.OLS_SGD_OPTIMIZERS,\n",
|
|
" optimizer_kwargs,\n",
|
|
" )\n",
|
|
"]\n",
|
|
"optimizers_ridge = [\n",
|
|
" (opt, {**kwargs, **ridge_kwargs})\n",
|
|
" for opt, kwargs in zip(\n",
|
|
" optimizers.RIDGE_SGD_OPTIMIZERS,\n",
|
|
" optimizer_kwargs,\n",
|
|
" )\n",
|
|
"]\n",
|
|
"optimizers_lasso = [\n",
|
|
" (opt, {**kwargs, **lasso_kwargs})\n",
|
|
" for opt, kwargs in zip(\n",
|
|
" optimizers.LASSO_SGD_OPTIMIZERS,\n",
|
|
" optimizer_kwargs,\n",
|
|
" )\n",
|
|
"]\n",
|
|
"\n",
|
|
"labels = [\"SGD\", \"Mom. SGD\", \"AdaGrad\", \"RMSProp\", \"Adam\"]\n",
|
|
"\n",
|
|
"fig, axs = plt.subplots(1, 3, figsize=plotting.get_figsize(0.5), sharey=True)\n",
|
|
"plotting.plot_optimizers(\n",
|
|
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch\", labels=labels\n",
|
|
")\n",
|
|
"plotting.plot_optimizers(\n",
|
|
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=None, labels=labels\n",
|
|
")\n",
|
|
"plotting.plot_optimizers(\n",
|
|
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=None, labels=labels\n",
|
|
")\n",
|
|
"for ax in axs:\n",
|
|
" ax.set_xlabel(\"Epoch\")\n",
|
|
"\n",
|
|
"axs[0].set_title(\"OLS\")\n",
|
|
"axs[1].set_title(\"Ridge\")\n",
|
|
"axs[2].set_title(\"LASSO\")\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"stochastic_gradient_descent_convergence.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "19",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"x = np.linspace(-1, 1, 300)\n",
|
|
"y = datamanip.runge_function(x)\n",
|
|
"x_train, x_test, y_train, y_test_noise_free = train_test_split(x, y, test_size=0.2)\n",
|
|
"y_train = datamanip.noise_data(y_train, 1.0)\n",
|
|
"y_test = datamanip.noise_data(y_test_noise_free, 1.0)\n",
|
|
"\n",
|
|
"polynomial_degrees = np.arange(1, 50)\n",
|
|
"n_bootstraps = len(x_train)\n",
|
|
"mses = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
|
|
"biases = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
|
|
"variances = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
|
|
"\n",
|
|
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
|
|
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
|
|
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
|
|
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
|
|
" y_train_scaled, y_test_scaled_nf = datamanip.scale_data(y_train, y_test_noise_free)\n",
|
|
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
|
|
"\n",
|
|
" for b, (X_, y_) in enumerate(\n",
|
|
" datamanip.bootstrap_resample(X_train_scaled, y_train_scaled, n_bootstraps)\n",
|
|
" ):\n",
|
|
" beta = optimizers.Ridge_parameters(X_, y_, lam=1e-10) # approx OLS but stable\n",
|
|
" y_pred = X_test_scaled @ beta\n",
|
|
" mse, _ = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
|
|
" mses[i, b] = mse\n",
|
|
" biases[i, b] = np.mean((y_test_scaled_nf - np.mean(y_pred)) ** 2)\n",
|
|
" variances[i, b] = np.var(y_pred)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "20",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
|
|
"mse_mean = np.mean(mses, axis=1)\n",
|
|
"bias_mean = np.mean(biases, axis=1)\n",
|
|
"var_mean = np.mean(variances, axis=1)\n",
|
|
"\n",
|
|
"ax.plot(polynomial_degrees, bias_mean, label=\"Bias$^2$\", color=\"C2\", ls=\"-\")\n",
|
|
"ax.fill_between(\n",
|
|
" polynomial_degrees,\n",
|
|
" np.zeros_like(bias_mean),\n",
|
|
" bias_mean,\n",
|
|
" color=\"C2\",\n",
|
|
" alpha=0.3,\n",
|
|
")\n",
|
|
"ax.plot(\n",
|
|
" polynomial_degrees,\n",
|
|
" var_mean + bias_mean,\n",
|
|
" label=\"Variance + Bias$^2$\",\n",
|
|
" color=\"C1\",\n",
|
|
" ls=\"-\",\n",
|
|
")\n",
|
|
"ax.fill_between(\n",
|
|
" polynomial_degrees,\n",
|
|
" bias_mean,\n",
|
|
" bias_mean + var_mean,\n",
|
|
" color=\"C1\",\n",
|
|
" alpha=0.3,\n",
|
|
")\n",
|
|
"ax.fill_between(\n",
|
|
" polynomial_degrees,\n",
|
|
" bias_mean + var_mean,\n",
|
|
" mse_mean,\n",
|
|
" color=\"C0\",\n",
|
|
" alpha=0.3,\n",
|
|
" label=\"Irreducible Error\",\n",
|
|
" hatch=\"//\",\n",
|
|
" edgecolor=\"white\",\n",
|
|
")\n",
|
|
"ax.plot(polynomial_degrees, mse_mean, label=\"MSE\", color=\"C0\", ls=\"-\")\n",
|
|
"\n",
|
|
"\n",
|
|
"ax.set_xlabel(\"Polynomial Degree\")\n",
|
|
"ax.set_ylabel(\"Error\")\n",
|
|
"ax.legend()\n",
|
|
"\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "21",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"k_folds = 5\n",
|
|
"k_fold_mses = np.zeros((len(polynomial_degrees), k_folds))\n",
|
|
"splits = datamanip.k_fold_split(x, datamanip.noise_data(y), k_folds)\n",
|
|
"\n",
|
|
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
|
|
" for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n",
|
|
" X_train = datamanip.polynomial_features(x_tr, polynomial_degree, False)\n",
|
|
" X_val = datamanip.polynomial_features(x_val, polynomial_degree, False)\n",
|
|
" X_train_scaled, X_val_scaled = datamanip.scale_data(X_train, X_val)\n",
|
|
" y_train_scaled, y_val_scaled = datamanip.scale_data(y_tr, y_val)\n",
|
|
"\n",
|
|
" beta = optimizers.Ridge_parameters(X_train_scaled, y_train_scaled, lam=1e-10)\n",
|
|
" y_pred = X_val_scaled @ beta\n",
|
|
" mse, _ = datamanip.evaluate_model(y_val_scaled, y_pred)\n",
|
|
" k_fold_mses[i, k] = mse"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "22",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"fig, (ax, ax2) = plt.subplots(1, 2, figsize=plotting.get_figsize(0.5), sharey=True)\n",
|
|
"mse_mean = np.mean(mses, axis=1)\n",
|
|
"bias_mean = np.mean(biases, axis=1)\n",
|
|
"var_mean = np.mean(variances, axis=1)\n",
|
|
"\n",
|
|
"ax.plot(polynomial_degrees, bias_mean, label=\"Bias$^2$\", color=\"C2\", ls=\"-\")\n",
|
|
"ax.fill_between(\n",
|
|
" polynomial_degrees,\n",
|
|
" np.zeros_like(bias_mean),\n",
|
|
" bias_mean,\n",
|
|
" color=\"C2\",\n",
|
|
" alpha=0.3,\n",
|
|
")\n",
|
|
"ax.plot(\n",
|
|
" polynomial_degrees,\n",
|
|
" var_mean + bias_mean,\n",
|
|
" label=\"Variance + Bias$^2$\",\n",
|
|
" color=\"C1\",\n",
|
|
" ls=\"-\",\n",
|
|
")\n",
|
|
"ax.fill_between(\n",
|
|
" polynomial_degrees,\n",
|
|
" bias_mean,\n",
|
|
" bias_mean + var_mean,\n",
|
|
" color=\"C1\",\n",
|
|
" alpha=0.3,\n",
|
|
")\n",
|
|
"ax.fill_between(\n",
|
|
" polynomial_degrees,\n",
|
|
" bias_mean + var_mean,\n",
|
|
" mse_mean,\n",
|
|
" color=\"C0\",\n",
|
|
" alpha=0.3,\n",
|
|
" label=\"Irreducible Error\",\n",
|
|
" hatch=\"//\",\n",
|
|
" edgecolor=\"white\",\n",
|
|
")\n",
|
|
"ax.plot(polynomial_degrees, mse_mean, label=\"MSE\", color=\"C0\", ls=\"-\")\n",
|
|
"\n",
|
|
"\n",
|
|
"ax.set_xlabel(\"Polynomial Degree\")\n",
|
|
"ax.set_ylabel(\"Error\")\n",
|
|
"ax.legend()\n",
|
|
"\n",
|
|
"fig.tight_layout()\n",
|
|
"\n",
|
|
"\n",
|
|
"ax2.plot(polynomial_degrees, mse_mean, label=\"Bootstrapping MSE\", color=\"C0\")\n",
|
|
"ax2.plot(\n",
|
|
" polynomial_degrees,\n",
|
|
" np.mean(k_fold_mses, axis=1),\n",
|
|
" label=f\"{k_folds}-Fold Crossvalidation MSE\",\n",
|
|
" color=\"C1\",\n",
|
|
")\n",
|
|
"\n",
|
|
"\n",
|
|
"ax2.set_xlabel(\"Polynomial Degree\")\n",
|
|
"# ax2.set_ylabel(\"Mean Squared Error\")\n",
|
|
"ax2.legend()\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff_combined_plot.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "23",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"polynomial_degrees = np.arange(1, 30)\n",
|
|
"k_folds = 5\n",
|
|
"\n",
|
|
"k_fold_mses_ols = np.zeros((len(polynomial_degrees), k_folds))\n",
|
|
"k_fold_mses_ridge = np.zeros_like(k_fold_mses_ols)\n",
|
|
"k_fold_mses_lasso = np.zeros_like(k_fold_mses_ols)\n",
|
|
"\n",
|
|
"splits = datamanip.k_fold_split(x, datamanip.noise_data(y), k_folds)\n",
|
|
"\n",
|
|
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
|
|
" for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n",
|
|
" X_train = datamanip.polynomial_features(x_tr, polynomial_degree, False)\n",
|
|
" X_val = datamanip.polynomial_features(x_val, polynomial_degree, False)\n",
|
|
" X_train_scaled, X_val_scaled = datamanip.scale_data(X_train, X_val)\n",
|
|
" y_train_scaled, y_val_scaled = datamanip.scale_data(y_tr, y_val)\n",
|
|
"\n",
|
|
" for optim, results_array in zip(\n",
|
|
" [\n",
|
|
" optimizers.OLSGradientDescent,\n",
|
|
" optimizers.RidgeGradientDescent,\n",
|
|
" optimizers.LASSOGradientDescent,\n",
|
|
" ],\n",
|
|
" [k_fold_mses_ols, k_fold_mses_ridge, k_fold_mses_lasso],\n",
|
|
" ):\n",
|
|
" Optimizer = optim(num_iterations=1000, learning_rate=0.1, lam=0.1)\n",
|
|
" theta = Optimizer.fit(X_train_scaled, y_train_scaled)\n",
|
|
" y_pred = X_val_scaled @ theta\n",
|
|
" mse, _ = datamanip.evaluate_model(y_val_scaled, y_pred)\n",
|
|
" results_array[i, k] = mse"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "24",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"FILL_BETWEEN = False\n",
|
|
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
|
|
"\n",
|
|
"for i, (mse_array, label) in enumerate(\n",
|
|
" zip(\n",
|
|
" [k_fold_mses_ols, k_fold_mses_ridge, k_fold_mses_lasso],\n",
|
|
" [\"OLS\", \"Ridge Regression\", \"Lasso Regression\"],\n",
|
|
" )\n",
|
|
"):\n",
|
|
" c = f\"C{i}\"\n",
|
|
" ax.plot(polynomial_degrees, np.mean(mse_array, axis=1), label=label, c=c)\n",
|
|
" if FILL_BETWEEN:\n",
|
|
" ax.fill_between(\n",
|
|
" polynomial_degrees,\n",
|
|
" np.mean(mse_array, axis=1) - np.std(mse_array, axis=1),\n",
|
|
" np.mean(mse_array, axis=1) + np.std(mse_array, axis=1),\n",
|
|
" color=c,\n",
|
|
" alpha=0.3,\n",
|
|
" )\n",
|
|
"\n",
|
|
"ax.set_xlabel(\"Polynomial Degree\")\n",
|
|
"ax.set_ylabel(f\"MSE ({k_folds}-fold validation)\")\n",
|
|
"ax.legend()\n",
|
|
"fig.tight_layout()\n",
|
|
"fig.savefig(os.path.join(FIG_DIR, \"kfold_mse_comparison_per_cost_function.pdf\"))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"id": "25",
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "project1",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.13.7"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 5
|
|
}
|