diff --git a/figures/bias_variance_tradeoff.pdf b/figures/bias_variance_tradeoff.pdf new file mode 100644 index 0000000..d91af47 Binary files /dev/null and b/figures/bias_variance_tradeoff.pdf differ diff --git a/figures/cost_function_comparison.pdf b/figures/cost_function_comparison.pdf index db9eea5..e30c1e2 100644 Binary files a/figures/cost_function_comparison.pdf and b/figures/cost_function_comparison.pdf differ diff --git a/figures/data_scatter.png b/figures/data_scatter.png index 8162739..925a5db 100644 Binary files a/figures/data_scatter.png and b/figures/data_scatter.png differ diff --git a/figures/gradient_descent_convergence.pdf b/figures/gradient_descent_convergence.pdf index 6fa93b5..90091cd 100644 Binary files a/figures/gradient_descent_convergence.pdf and b/figures/gradient_descent_convergence.pdf differ diff --git a/figures/ols_mse_r2.pdf b/figures/ols_mse_r2.pdf index 6df088f..755ec8d 100644 Binary files a/figures/ols_mse_r2.pdf and b/figures/ols_mse_r2.pdf differ diff --git a/figures/ols_parameter_plot.pdf b/figures/ols_parameter_plot.pdf index 96f13f6..b99f023 100644 Binary files a/figures/ols_parameter_plot.pdf and b/figures/ols_parameter_plot.pdf differ diff --git a/figures/optimizer_comparison.pdf b/figures/optimizer_comparison.pdf index c93b799..669a8fc 100644 Binary files a/figures/optimizer_comparison.pdf and b/figures/optimizer_comparison.pdf differ diff --git a/figures/ridge_mse_r2.pdf b/figures/ridge_mse_r2.pdf index 942e495..f97d6b7 100644 Binary files a/figures/ridge_mse_r2.pdf and b/figures/ridge_mse_r2.pdf differ diff --git a/figures/ridge_mse_r2_lambda.pdf b/figures/ridge_mse_r2_lambda.pdf index b116957..7a0fce9 100644 Binary files a/figures/ridge_mse_r2_lambda.pdf and b/figures/ridge_mse_r2_lambda.pdf differ diff --git a/figures/ridge_parameter_plot.pdf b/figures/ridge_parameter_plot.pdf index 0a36a8b..965c650 100644 Binary files a/figures/ridge_parameter_plot.pdf and b/figures/ridge_parameter_plot.pdf differ diff --git a/src/datamanip.py b/src/datamanip.py index 1ff3c65..db1df91 100644 --- a/src/datamanip.py +++ b/src/datamanip.py @@ -1,6 +1,7 @@ import numpy as np from sklearn.preprocessing import StandardScaler # type: ignore from sklearn.metrics import mean_squared_error, r2_score # type: ignore +from sklearn.utils import resample # type: ignore def polynomial_features(x: np.ndarray, p: int, intercept: bool = True) -> np.ndarray: @@ -80,3 +81,21 @@ def evaluate_model(y_true: np.ndarray, y_pred: np.ndarray) -> tuple[float, float A tuple containing the MSE and R^2 score. """ return mean_squared_error(y_true, y_pred), r2_score(y_true, y_pred) + + +def bootstrap_resample( + X: np.ndarray, y: np.ndarray, n_resamples: int +) -> list[tuple[np.ndarray, np.ndarray]]: + """Generates bootstrap resamples of the dataset. + Args: + X: The input data matrix of shape (n_samples, n_features). + y: The target vector of shape (n_samples,). + n_resamples: The number of bootstrap resamples to generate. + Returns: + A list of tuples, each containing a resampled (X_resample, y_resample). + """ + resamples = [] + for _ in range(n_resamples): + X_resample, y_resample = resample(X, y) + resamples.append((X_resample, y_resample)) + return resamples diff --git a/src/main.ipynb b/src/main.ipynb index 55562da..aabefe0 100644 --- a/src/main.ipynb +++ b/src/main.ipynb @@ -25,7 +25,8 @@ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "from sklearn.model_selection import train_test_split\n", - "import os" + "import os\n", + "import time" ] }, { @@ -309,7 +310,8 @@ "def get_cost_history(optimizer, X, y, **kwargs):\n", " opt = optimizer(**kwargs)\n", " opt.fit(X, y)\n", - " return opt.cost_history" + " iterations = opt.get_epochs()\n", + " return iterations, opt.cost_history" ] }, { @@ -332,33 +334,64 @@ "lam = 0.1\n", "\n", "optimizers_ols = [\n", - " (optimizers.OLSGradientDescent, {\"learning_rate\": learning_rate_ols}),\n", - " (optimizers.OLSMomentum, {\"learning_rate\": learning_rate_ols, \"delta\": 0.9}),\n", - " (optimizers.OLSAdaGrad, {\"learning_rate\": learning_rate_ols}),\n", - " (optimizers.OLSRMSProp, {\"learning_rate\": learning_rate_ols, \"gamma\": 0.9}),\n", + " (\n", + " optimizers.OLSGradientDescent,\n", + " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n", + " ),\n", + " (\n", + " optimizers.OLSMomentum,\n", + " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"delta\": 0.9},\n", + " ),\n", + " (\n", + " optimizers.OLSAdaGrad,\n", + " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n", + " ),\n", + " (\n", + " optimizers.OLSRMSProp,\n", + " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"gamma\": 0.9},\n", + " ),\n", " (\n", " optimizers.OLSAdam,\n", - " {\"learning_rate\": learning_rate_ols, \"beta1\": 0.9, \"beta2\": 0.999},\n", + " {\n", + " \"num_iterations\": num_iters,\n", + " \"learning_rate\": learning_rate_ols,\n", + " \"beta1\": 0.9,\n", + " \"beta2\": 0.999,\n", + " },\n", " ),\n", "]\n", "\n", "optimizers_ridge = [\n", " (\n", " optimizers.RidgeGradientDescent,\n", - " {\"learning_rate\": learning_rate_ridge, \"lam\": lam},\n", + " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n", " ),\n", " (\n", " optimizers.RidgeMomentum,\n", - " {\"learning_rate\": learning_rate_ridge, \"delta\": 0.9, \"lam\": lam},\n", + " {\n", + " \"num_iterations\": num_iters,\n", + " \"learning_rate\": learning_rate_ridge,\n", + " \"delta\": 0.9,\n", + " \"lam\": lam,\n", + " },\n", + " ),\n", + " (\n", + " optimizers.RidgeAdaGrad,\n", + " {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n", " ),\n", - " (optimizers.RidgeAdaGrad, {\"learning_rate\": learning_rate_ridge, \"lam\": lam}),\n", " (\n", " optimizers.RidgeRMSProp,\n", - " {\"learning_rate\": learning_rate_ridge, \"gamma\": 0.9, \"lam\": lam},\n", + " {\n", + " \"num_iterations\": num_iters,\n", + " \"learning_rate\": learning_rate_ridge,\n", + " \"gamma\": 0.9,\n", + " \"lam\": lam,\n", + " },\n", " ),\n", " (\n", " optimizers.RidgeAdam,\n", " {\n", + " \"num_iterations\": num_iters,\n", " \"learning_rate\": learning_rate_ridge,\n", " \"beta1\": 0.9,\n", " \"beta2\": 0.999,\n", @@ -368,19 +401,19 @@ "]\n", "\n", "\n", - "def plot_optimizers(ax, optimizer_list, X, y, num_iters, ylabel=\"Cost\"):\n", + "def plot_optimizers(ax, optimizer_list, X, y, ylabel=\"Cost\"):\n", " for opt_class, params in optimizer_list:\n", - " history = get_cost_history(opt_class, X, y, num_iterations=num_iters, **params)\n", - " ax.plot(history, label=f\"{opt_class.__name__}\")\n", + " iter, history = get_cost_history(opt_class, X, y, **params)\n", + " ax.plot(iter, history, label=f\"{opt_class.__name__}\")\n", " ax.set_xlabel(\"Iteration\")\n", " ax.set_ylabel(ylabel)\n", " ax.legend()\n", "\n", "\n", - "plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, num_iters, ylabel=\"Cost (OLS)\")\n", + "plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n", "ax1.set_ylim(bottom=0.465, top=0.505)\n", "\n", - "plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, num_iters, ylabel=\"Cost (Ridge)\")\n", + "plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n", "ax2.set_ylim(bottom=0.4775, top=0.505)\n", "fig.tight_layout()\n", "fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))" @@ -434,6 +467,267 @@ "id": "16", "metadata": {}, "outputs": [], + "source": [ + "X_size = 1_000_000\n", + "num_epochs = 1000\n", + "batches_per_epoch = 100\n", + "batch_size = 512\n", + "\n", + "ols_gd = optimizers.OLSGradientDescent(learning_rate=0.1, num_iterations=num_epochs)\n", + "ols_sgd = optimizers.OLSStochasticGradientDescent(\n", + " learning_rate=0.1,\n", + " num_iterations=num_epochs * batches_per_epoch,\n", + " batch_size=batch_size,\n", + ")\n", + "ridge_gd = optimizers.RidgeGradientDescent(\n", + " learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n", + ")\n", + "ridge_sgd = optimizers.RidgeStochasticGradientDescent(\n", + " learning_rate=0.01,\n", + " lam=0.1,\n", + " num_iterations=num_epochs * batches_per_epoch,\n", + " batch_size=batch_size,\n", + ")\n", + "\n", + "x = np.linspace(-1, 1, X_size)\n", + "y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "X_train = datamanip.polynomial_features(x_train, 10, False)\n", + "X_test = datamanip.polynomial_features(x_test, 10, False)\n", + "X_tr, X_te = datamanip.scale_data(X_train, X_test)\n", + "y_tr, y_te = datamanip.scale_data(y_train, y_test)\n", + "\n", + "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", + "\n", + "for optimizer, label in zip(\n", + " [ols_sgd, ols_gd, ridge_sgd, ridge_gd],\n", + " [\n", + " \"Stochastic GD\",\n", + " \"Gradient Descent\",\n", + " \"Ridge Stochastic GD\",\n", + " \"Ridge Gradient Descent\",\n", + " ],\n", + "):\n", + " start_time = time.time()\n", + " optimizer.fit(X_tr, y_tr)\n", + " time_taken = time.time() - start_time\n", + " cost_history = optimizer._cost_history\n", + " if \"Stochastic\" in label:\n", + " epochs = np.arange(1, num_epochs * batches_per_epoch + 1) / batches_per_epoch\n", + " ls = \"-\"\n", + " else:\n", + " epochs = np.arange(1, num_epochs + 1)\n", + " ls = \"--\"\n", + " c = \"C0\" if \"Ridge\" not in label else \"C1\"\n", + " ax.plot(\n", + " epochs,\n", + " cost_history,\n", + " label=f\"{label} ({time_taken:.2f} s)\",\n", + " linestyle=ls,\n", + " color=c,\n", + " )\n", + "ax.set_xlabel(\"Epoch\")\n", + "ax.set_ylabel(\"Cost\")\n", + "ax.legend()\n", + "# ax.set_ylim(bottom=0.0, top=0.505)\n", + "fig.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "X_size = 10_000\n", + "num_epochs = 1000\n", + "batches_per_epoch = 100\n", + "batch_size = 128\n", + "\n", + "learning_rate_ols = 0.1\n", + "learning_rate_ridge = 0.001\n", + "learning_rate_lasso = 0.001\n", + "lam_ridge = 0.1\n", + "lam_lasso = 0.1\n", + "\n", + "x = np.linspace(-1, 1, X_size)\n", + "y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "X_train = datamanip.polynomial_features(x_train, 10, False)\n", + "X_test = datamanip.polynomial_features(x_test, 10, False)\n", + "X_tr, X_te = datamanip.scale_data(X_train, X_test)\n", + "y_tr, y_te = datamanip.scale_data(y_train, y_test)\n", + "\n", + "general_kwargs = {\n", + " \"num_iterations\": num_epochs * batches_per_epoch,\n", + " \"batch_size\": batch_size,\n", + " \"batches_per_epoch\": batches_per_epoch,\n", + "}\n", + "optimizer_kwargs = [\n", + " {},\n", + " {\"delta\": 0.9},\n", + " {},\n", + " {\"gamma\": 0.9},\n", + " {\"beta1\": 0.9, \"beta2\": 0.999},\n", + "]\n", + "ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n", + "ridge_kwargs = {\n", + " \"learning_rate\": learning_rate_ridge,\n", + " \"lam\": lam_ridge,\n", + " **general_kwargs,\n", + "}\n", + "lasso_kwargs = {\n", + " \"learning_rate\": learning_rate_lasso,\n", + " \"lam\": lam_lasso,\n", + " **general_kwargs,\n", + "}\n", + "optimizers_ols = [\n", + " (opt, {**kwargs, **ols_kwargs})\n", + " for opt, kwargs in zip(\n", + " [\n", + " optimizers.OLSStochasticGradientDescent,\n", + " optimizers.OLSMomentumSGD,\n", + " optimizers.OLSAdaGradSGD,\n", + " optimizers.OLSRMSPropSGD,\n", + " optimizers.OLSAdamSGD,\n", + " ],\n", + " optimizer_kwargs,\n", + " )\n", + "]\n", + "optimizers_ridge = [\n", + " (opt, {**kwargs, **ridge_kwargs})\n", + " for opt, kwargs in zip(\n", + " [\n", + " optimizers.RidgeStochasticGradientDescent,\n", + " optimizers.RidgeMomentumSGD,\n", + " optimizers.RidgeAdaGradSGD,\n", + " optimizers.RidgeRMSPropSGD,\n", + " optimizers.RidgeAdamSGD,\n", + " ],\n", + " optimizer_kwargs,\n", + " )\n", + "]\n", + "optimizers_lasso = [\n", + " (opt, {**kwargs, **lasso_kwargs})\n", + " for opt, kwargs in zip(\n", + " [\n", + " optimizers.LASSOStochasticGradientDescent,\n", + " optimizers.LASSOMomentumSGD,\n", + " optimizers.LASSOAdaGradSGD,\n", + " optimizers.LASSORMSPropSGD,\n", + " optimizers.LASSOAdamSGD,\n", + " ],\n", + " optimizer_kwargs,\n", + " )\n", + "]\n", + "\n", + "fig, axs = plt.subplots(1, 3, figsize=np.array(plotting.get_figsize(0.5)) * 2)\n", + "plot_optimizers(\n", + " axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch (OLS)\"\n", + ")\n", + "plot_optimizers(\n", + " axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch (Ridge)\"\n", + ")\n", + "plot_optimizers(\n", + " axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch (LASSO)\"\n", + ")\n", + "for ax in axs:\n", + " ax.set_xlabel(\"Epoch\")\n", + "fig.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "x = np.linspace(-1, 1, 300)\n", + "y = datamanip.noise_data(datamanip.runge_function(x), 0.1)\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "polynomial_degrees = np.arange(1, 30)\n", + "n_bootstraps = len(x_train)\n", + "mses = np.zeros((len(polynomial_degrees), n_bootstraps))\n", + "biases = np.zeros((len(polynomial_degrees), n_bootstraps))\n", + "variances = np.zeros((len(polynomial_degrees), n_bootstraps))\n", + "\n", + "for i, polynomial_degree in enumerate(polynomial_degrees):\n", + " X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n", + " X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n", + " X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n", + " y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n", + "\n", + " for b, (X_, y_) in enumerate(\n", + " datamanip.bootstrap_resample(X_train_scaled, y_train_scaled, n_bootstraps)\n", + " ):\n", + " beta = optimizers.Ridge_parameters(X_, y_, lam=1e-10) # approx OLS but stable\n", + " y_pred = X_test_scaled @ beta\n", + " mse, _ = datamanip.evaluate_model(y_test_scaled, y_pred)\n", + " mses[i, b] = mse\n", + " biases[i, b] = np.sqrt(np.mean((y_test_scaled - np.mean(y_pred)) ** 2))\n", + " variances[i, b] = np.var(y_pred)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "FILL_BETWEEN = False\n", + "fig, axs = plt.subplots(1, 2, figsize=plotting.get_figsize(0.4))\n", + "axs[0].plot(polynomial_degrees, np.mean(mses, axis=1), label=\"MSE (Test)\", color=\"C0\")\n", + "\n", + "\n", + "axs[1].plot(polynomial_degrees, np.mean(biases, axis=1), label=\"Bias$^2$\", color=\"C1\")\n", + "axs[1].plot(\n", + " polynomial_degrees, np.mean(variances, axis=1), label=\"Variance\", color=\"C2\"\n", + ")\n", + "if FILL_BETWEEN:\n", + " axs[0].fill_between(\n", + " polynomial_degrees,\n", + " np.mean(mses, axis=1) - np.std(mses, axis=1),\n", + " np.mean(mses, axis=1) + np.std(mses, axis=1),\n", + " color=\"C0\",\n", + " alpha=0.3,\n", + " )\n", + " axs[1].fill_between(\n", + " polynomial_degrees,\n", + " np.mean(biases, axis=1) - np.std(biases, axis=1),\n", + " np.mean(biases, axis=1) + np.std(biases, axis=1),\n", + " color=\"C1\",\n", + " alpha=0.3,\n", + " )\n", + " axs[1].fill_between(\n", + " polynomial_degrees,\n", + " np.mean(variances, axis=1) - np.std(variances, axis=1),\n", + " np.mean(variances, axis=1) + np.std(variances, axis=1),\n", + " color=\"C2\",\n", + " alpha=0.3,\n", + " )\n", + "axs[0].set_ylabel(\"Mean Squared Error\")\n", + "axs[1].set_ylabel(\"Bias and Variance\")\n", + "\n", + "for ax in axs:\n", + " ax.set_xlabel(\"Polynomial Degree\")\n", + " ax.legend()\n", + "\n", + "fig.tight_layout()\n", + "fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff.pdf\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], "source": [] } ], diff --git a/src/optimizers.py b/src/optimizers.py index 8685473..dc7c9dc 100644 --- a/src/optimizers.py +++ b/src/optimizers.py @@ -35,10 +35,27 @@ class GradientDescent: learning_rate (float, optional): Learning rate used for step updates. Defaults to 0.1. num_iterations (int, optional): Number of iterations for gradient descent. Defaults to 1000. """ - self.cost_history = np.zeros(num_iterations) + self._cost_history = np.zeros(num_iterations) self.learning_rate = learning_rate self.num_iterations = num_iterations + @property + def cost_history(self) -> np.ndarray: + """Returns the cost history of the optimization process. + + Returns: + np.ndarray: Array of cost values for each iteration. + """ + return self._cost_history + + def get_epochs(self) -> np.ndarray: + """Returns an array of epoch numbers from 0 to num_iterations - 1. + + Returns: + np.ndarray: Array of epoch numbers. + """ + return np.arange(self.num_iterations) + def fit(self, X: np.ndarray, y: np.ndarray) -> np.ndarray: """Returns the optimal solution for an optimization problem using the gradient descent @@ -55,7 +72,7 @@ class GradientDescent: self._precomp() for t in range(self.num_iterations): self._comp_step() - self.cost_history[t] = self._compute_cost() + self._cost_history[t] = self._compute_cost() self._update_theta() return self.theta @@ -188,21 +205,24 @@ class OLSAdam(OLSGradientDescent): super().__init__( *args, learning_rate=learning_rate, num_iterations=num_iterations, **kwargs ) + self.beta1 = beta1 self.beta2 = beta2 def _precomp(self): self.m = np.zeros_like(self.theta) self.v = np.zeros_like(self.theta) + self.current_iteration = 0 return super()._precomp() def _update_theta(self): + self.current_iteration += 1 grad = self._compute_grad() self.m = self.beta1 * self.m + (1 - self.beta1) * grad self.v = self.beta2 * self.v + (1 - self.beta2) * np.square(grad) - m_hat = self.m / (1 - self.beta1 ** (self.num_iterations)) - v_hat = self.v / (1 - self.beta2 ** (self.num_iterations)) + m_hat = self.m / (1 - self.beta1 ** (self.current_iteration)) + v_hat = self.v / (1 - self.beta2 ** (self.current_iteration)) self.theta -= ( self.learning_rate * m_hat / (np.sqrt(v_hat) + 1e-10) @@ -244,3 +264,116 @@ class LASSORMSProp(OLSRMSProp, LASSOGradientDescent): class LASSOAdam(OLSAdam, LASSOGradientDescent): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + + +class OLSStochasticGradientDescent(OLSGradientDescent): + def __init__( + self, *args, batch_size: int = 100, batches_per_epoch: int = 1, **kwargs + ): + # print(self.__class__.__name__) # If you see this: debugging yaaaay, the programmer that wrote this line is stupid... + super().__init__(*args, **kwargs) + self.batch_size = batch_size + self.batches_per_epoch = batches_per_epoch + + def _precomp(self): + self.N = len(self.y) + self.indices = np.arange(self.N) + self.n = self.batch_size + + def _comp_step(self): + index = np.random.randint(0, self.N) + batch_indices = slice(index, index + self.batch_size) + if index + self.batch_size > self.N: + batch_indices = slice(index, self.N) + + X_batch = self.X[batch_indices] + y_batch = self.y[batch_indices] + self.err = X_batch @ self.theta - y_batch + self.XTX = X_batch.T @ X_batch + self.XTy = X_batch.T @ y_batch + + def get_epochs(self): + return np.arange(self.num_iterations // self.batches_per_epoch) + + @property + def cost_history(self) -> np.ndarray: + """Returns the cost history of the optimization process. + + Returns: + np.ndarray: Array of cost values for each epoch. + """ + return np.mean(self._cost_history.reshape(-1, self.batches_per_epoch), axis=1) + + +class RidgeStochasticGradientDescent( + OLSStochasticGradientDescent, RidgeGradientDescent +): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class LASSOStochasticGradientDescent( + OLSStochasticGradientDescent, LASSOGradientDescent +): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class OLSMomentumSGD(OLSMomentum, OLSStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class RidgeMomentumSGD(RidgeMomentum, RidgeStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class LASSOMomentumSGD(LASSOMomentum, LASSOStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class OLSAdaGradSGD(OLSAdaGrad, OLSStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class RidgeAdaGradSGD(RidgeAdaGrad, RidgeStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class LASSOAdaGradSGD(LASSOAdaGrad, LASSOStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class OLSRMSPropSGD(OLSRMSProp, OLSStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class RidgeRMSPropSGD(RidgeRMSProp, RidgeStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class LASSORMSPropSGD(LASSORMSProp, LASSOStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class OLSAdamSGD(OLSAdam, OLSStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class RidgeAdamSGD(RidgeAdam, RidgeStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class LASSOAdamSGD(LASSOAdam, LASSOStochasticGradientDescent): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs)