lars/FYSSTK-Project1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement sgd and bootstrapping methods. Hyperparameter tuning for the above mentioned.

Browse Source
This commit is contained in:
Lars Bogner
2025-09-09 17:28:08 +02:00
parent 89a8381a98
commit fd29ebfdad
13 changed files with 466 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
figures/bias_variance_tradeoff.pdf
BIN
View File
Binary file not shown.
figures/cost_function_comparison.pdf
BIN
View File
Binary file not shown.
figures/data_scatter.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 324 KiB

After

Width:  |  Height:  |  Size: 315 KiB

figures/gradient_descent_convergence.pdf
BIN
View File
Binary file not shown.
figures/ols_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ols_parameter_plot.pdf
BIN
View File
Binary file not shown.
figures/optimizer_comparison.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2.pdf
BIN
View File
Binary file not shown.
figures/ridge_mse_r2_lambda.pdf
BIN
View File
Binary file not shown.
figures/ridge_parameter_plot.pdf
BIN
View File
Binary file not shown.
src/datamanip.py
+19
View File
@@ -1,6 +1,7 @@
import numpy as np
from sklearn.preprocessing import StandardScaler # type: ignore
from sklearn.metrics import mean_squared_error, r2_score # type: ignore
from sklearn.utils import resample # type: ignore
def polynomial_features(x: np.ndarray, p: int, intercept: bool = True) -> np.ndarray:
@@ -80,3 +81,21 @@ def evaluate_model(y_true: np.ndarray, y_pred: np.ndarray) -> tuple[float, float
A tuple containing the MSE and R^2 score.
"""
return mean_squared_error(y_true, y_pred), r2_score(y_true, y_pred)
def bootstrap_resample(
X: np.ndarray, y: np.ndarray, n_resamples: int
) -> list[tuple[np.ndarray, np.ndarray]]:
"""Generates bootstrap resamples of the dataset.
Args:
X: The input data matrix of shape (n_samples, n_features).
y: The target vector of shape (n_samples,).
n_resamples: The number of bootstrap resamples to generate.
Returns:
A list of tuples, each containing a resampled (X_resample, y_resample).
"""
resamples = []
for _ in range(n_resamples):
X_resample, y_resample = resample(X, y)
resamples.append((X_resample, y_resample))
return resamples
src/main.ipynb
+310 -16
View File
@@ -25,7 +25,8 @@
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from sklearn.model_selection import train_test_split\n",
"import os"
"import os\n",
"import time"
]
},
{
@@ -309,7 +310,8 @@
"def get_cost_history(optimizer, X, y, **kwargs):\n",
" opt = optimizer(**kwargs)\n",
" opt.fit(X, y)\n",
" return opt.cost_history"
" iterations = opt.get_epochs()\n",
" return iterations, opt.cost_history"
]
},
{
@@ -332,33 +334,64 @@
"lam = 0.1\n",
"\n",
"optimizers_ols = [\n",
" (optimizers.OLSGradientDescent, {\"learning_rate\": learning_rate_ols}),\n",
" (optimizers.OLSMomentum, {\"learning_rate\": learning_rate_ols, \"delta\": 0.9}),\n",
" (optimizers.OLSAdaGrad, {\"learning_rate\": learning_rate_ols}),\n",
" (optimizers.OLSRMSProp, {\"learning_rate\": learning_rate_ols, \"gamma\": 0.9}),\n",
" (\n",
" optimizers.OLSGradientDescent,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
" ),\n",
" (\n",
" optimizers.OLSMomentum,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"delta\": 0.9},\n",
" ),\n",
" (\n",
" optimizers.OLSAdaGrad,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols},\n",
" ),\n",
" (\n",
" optimizers.OLSRMSProp,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ols, \"gamma\": 0.9},\n",
" ),\n",
" (\n",
" optimizers.OLSAdam,\n",
" {\"learning_rate\": learning_rate_ols, \"beta1\": 0.9, \"beta2\": 0.999},\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ols,\n",
" \"beta1\": 0.9,\n",
" \"beta2\": 0.999,\n",
" },\n",
" ),\n",
"]\n",
"\n",
"optimizers_ridge = [\n",
" (\n",
" optimizers.RidgeGradientDescent,\n",
" {\"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
" ),\n",
" (\n",
" optimizers.RidgeMomentum,\n",
" {\"learning_rate\": learning_rate_ridge, \"delta\": 0.9, \"lam\": lam},\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"delta\": 0.9,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
" (\n",
" optimizers.RidgeAdaGrad,\n",
" {\"num_iterations\": num_iters, \"learning_rate\": learning_rate_ridge, \"lam\": lam},\n",
" ),\n",
" (optimizers.RidgeAdaGrad, {\"learning_rate\": learning_rate_ridge, \"lam\": lam}),\n",
" (\n",
" optimizers.RidgeRMSProp,\n",
" {\"learning_rate\": learning_rate_ridge, \"gamma\": 0.9, \"lam\": lam},\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"gamma\": 0.9,\n",
" \"lam\": lam,\n",
" },\n",
" ),\n",
" (\n",
" optimizers.RidgeAdam,\n",
" {\n",
" \"num_iterations\": num_iters,\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"beta1\": 0.9,\n",
" \"beta2\": 0.999,\n",
@@ -368,19 +401,19 @@
"]\n",
"\n",
"\n",
"def plot_optimizers(ax, optimizer_list, X, y, num_iters, ylabel=\"Cost\"):\n",
"def plot_optimizers(ax, optimizer_list, X, y, ylabel=\"Cost\"):\n",
" for opt_class, params in optimizer_list:\n",
" history = get_cost_history(opt_class, X, y, num_iterations=num_iters, **params)\n",
" ax.plot(history, label=f\"{opt_class.__name__}\")\n",
" iter, history = get_cost_history(opt_class, X, y, **params)\n",
" ax.plot(iter, history, label=f\"{opt_class.__name__}\")\n",
" ax.set_xlabel(\"Iteration\")\n",
" ax.set_ylabel(ylabel)\n",
" ax.legend()\n",
"\n",
"\n",
"plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, num_iters, ylabel=\"Cost (OLS)\")\n",
"plot_optimizers(ax1, optimizers_ols, X_tr, y_tr, ylabel=\"Cost (OLS)\")\n",
"ax1.set_ylim(bottom=0.465, top=0.505)\n",
"\n",
"plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, num_iters, ylabel=\"Cost (Ridge)\")\n",
"plot_optimizers(ax2, optimizers_ridge, X_tr, y_tr, ylabel=\"Cost (Ridge)\")\n",
"ax2.set_ylim(bottom=0.4775, top=0.505)\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"optimizer_comparison.pdf\"))"
@@ -434,6 +467,267 @@
"id": "16",
"metadata": {},
"outputs": [],
"source": [
"X_size = 1_000_000\n",
"num_epochs = 1000\n",
"batches_per_epoch = 100\n",
"batch_size = 512\n",
"\n",
"ols_gd = optimizers.OLSGradientDescent(learning_rate=0.1, num_iterations=num_epochs)\n",
"ols_sgd = optimizers.OLSStochasticGradientDescent(\n",
" learning_rate=0.1,\n",
" num_iterations=num_epochs * batches_per_epoch,\n",
" batch_size=batch_size,\n",
")\n",
"ridge_gd = optimizers.RidgeGradientDescent(\n",
" learning_rate=0.01, lam=0.1, num_iterations=num_epochs\n",
")\n",
"ridge_sgd = optimizers.RidgeStochasticGradientDescent(\n",
" learning_rate=0.01,\n",
" lam=0.1,\n",
" num_iterations=num_epochs * batches_per_epoch,\n",
" batch_size=batch_size,\n",
")\n",
"\n",
"x = np.linspace(-1, 1, X_size)\n",
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"\n",
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
"\n",
"fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n",
"\n",
"for optimizer, label in zip(\n",
" [ols_sgd, ols_gd, ridge_sgd, ridge_gd],\n",
" [\n",
" \"Stochastic GD\",\n",
" \"Gradient Descent\",\n",
" \"Ridge Stochastic GD\",\n",
" \"Ridge Gradient Descent\",\n",
" ],\n",
"):\n",
" start_time = time.time()\n",
" optimizer.fit(X_tr, y_tr)\n",
" time_taken = time.time() - start_time\n",
" cost_history = optimizer._cost_history\n",
" if \"Stochastic\" in label:\n",
" epochs = np.arange(1, num_epochs * batches_per_epoch + 1) / batches_per_epoch\n",
" ls = \"-\"\n",
" else:\n",
" epochs = np.arange(1, num_epochs + 1)\n",
" ls = \"--\"\n",
" c = \"C0\" if \"Ridge\" not in label else \"C1\"\n",
" ax.plot(\n",
" epochs,\n",
" cost_history,\n",
" label=f\"{label} ({time_taken:.2f} s)\",\n",
" linestyle=ls,\n",
" color=c,\n",
" )\n",
"ax.set_xlabel(\"Epoch\")\n",
"ax.set_ylabel(\"Cost\")\n",
"ax.legend()\n",
"# ax.set_ylim(bottom=0.0, top=0.505)\n",
"fig.tight_layout()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "17",
"metadata": {},
"outputs": [],
"source": [
"X_size = 10_000\n",
"num_epochs = 1000\n",
"batches_per_epoch = 100\n",
"batch_size = 128\n",
"\n",
"learning_rate_ols = 0.1\n",
"learning_rate_ridge = 0.001\n",
"learning_rate_lasso = 0.001\n",
"lam_ridge = 0.1\n",
"lam_lasso = 0.1\n",
"\n",
"x = np.linspace(-1, 1, X_size)\n",
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1) # LOWER NOISE FOR SGD\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"\n",
"X_train = datamanip.polynomial_features(x_train, 10, False)\n",
"X_test = datamanip.polynomial_features(x_test, 10, False)\n",
"X_tr, X_te = datamanip.scale_data(X_train, X_test)\n",
"y_tr, y_te = datamanip.scale_data(y_train, y_test)\n",
"\n",
"general_kwargs = {\n",
" \"num_iterations\": num_epochs * batches_per_epoch,\n",
" \"batch_size\": batch_size,\n",
" \"batches_per_epoch\": batches_per_epoch,\n",
"}\n",
"optimizer_kwargs = [\n",
" {},\n",
" {\"delta\": 0.9},\n",
" {},\n",
" {\"gamma\": 0.9},\n",
" {\"beta1\": 0.9, \"beta2\": 0.999},\n",
"]\n",
"ols_kwargs = {\"learning_rate\": learning_rate_ols, **general_kwargs}\n",
"ridge_kwargs = {\n",
" \"learning_rate\": learning_rate_ridge,\n",
" \"lam\": lam_ridge,\n",
" **general_kwargs,\n",
"}\n",
"lasso_kwargs = {\n",
" \"learning_rate\": learning_rate_lasso,\n",
" \"lam\": lam_lasso,\n",
" **general_kwargs,\n",
"}\n",
"optimizers_ols = [\n",
" (opt, {**kwargs, **ols_kwargs})\n",
" for opt, kwargs in zip(\n",
" [\n",
" optimizers.OLSStochasticGradientDescent,\n",
" optimizers.OLSMomentumSGD,\n",
" optimizers.OLSAdaGradSGD,\n",
" optimizers.OLSRMSPropSGD,\n",
" optimizers.OLSAdamSGD,\n",
" ],\n",
" optimizer_kwargs,\n",
" )\n",
"]\n",
"optimizers_ridge = [\n",
" (opt, {**kwargs, **ridge_kwargs})\n",
" for opt, kwargs in zip(\n",
" [\n",
" optimizers.RidgeStochasticGradientDescent,\n",
" optimizers.RidgeMomentumSGD,\n",
" optimizers.RidgeAdaGradSGD,\n",
" optimizers.RidgeRMSPropSGD,\n",
" optimizers.RidgeAdamSGD,\n",
" ],\n",
" optimizer_kwargs,\n",
" )\n",
"]\n",
"optimizers_lasso = [\n",
" (opt, {**kwargs, **lasso_kwargs})\n",
" for opt, kwargs in zip(\n",
" [\n",
" optimizers.LASSOStochasticGradientDescent,\n",
" optimizers.LASSOMomentumSGD,\n",
" optimizers.LASSOAdaGradSGD,\n",
" optimizers.LASSORMSPropSGD,\n",
" optimizers.LASSOAdamSGD,\n",
" ],\n",
" optimizer_kwargs,\n",
" )\n",
"]\n",
"\n",
"fig, axs = plt.subplots(1, 3, figsize=np.array(plotting.get_figsize(0.5)) * 2)\n",
"plot_optimizers(\n",
" axs[0], optimizers_ols, X_tr, y_tr, ylabel=\"Average Cost per Epoch (OLS)\"\n",
")\n",
"plot_optimizers(\n",
" axs[1], optimizers_ridge, X_tr, y_tr, ylabel=\"Average Cost per Epoch (Ridge)\"\n",
")\n",
"plot_optimizers(\n",
" axs[2], optimizers_lasso, X_tr, y_tr, ylabel=\"Average Cost per Epoch (LASSO)\"\n",
")\n",
"for ax in axs:\n",
" ax.set_xlabel(\"Epoch\")\n",
"fig.tight_layout()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18",
"metadata": {},
"outputs": [],
"source": [
"x = np.linspace(-1, 1, 300)\n",
"y = datamanip.noise_data(datamanip.runge_function(x), 0.1)\n",
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"\n",
"polynomial_degrees = np.arange(1, 30)\n",
"n_bootstraps = len(x_train)\n",
"mses = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
"biases = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
"variances = np.zeros((len(polynomial_degrees), n_bootstraps))\n",
"\n",
"for i, polynomial_degree in enumerate(polynomial_degrees):\n",
" X_train = datamanip.polynomial_features(x_train, polynomial_degree, False)\n",
" X_test = datamanip.polynomial_features(x_test, polynomial_degree, False)\n",
" X_train_scaled, X_test_scaled = datamanip.scale_data(X_train, X_test)\n",
" y_train_scaled, y_test_scaled = datamanip.scale_data(y_train, y_test)\n",
"\n",
" for b, (X_, y_) in enumerate(\n",
" datamanip.bootstrap_resample(X_train_scaled, y_train_scaled, n_bootstraps)\n",
" ):\n",
" beta = optimizers.Ridge_parameters(X_, y_, lam=1e-10) # approx OLS but stable\n",
" y_pred = X_test_scaled @ beta\n",
" mse, _ = datamanip.evaluate_model(y_test_scaled, y_pred)\n",
" mses[i, b] = mse\n",
" biases[i, b] = np.sqrt(np.mean((y_test_scaled - np.mean(y_pred)) ** 2))\n",
" variances[i, b] = np.var(y_pred)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "19",
"metadata": {},
"outputs": [],
"source": [
"FILL_BETWEEN = False\n",
"fig, axs = plt.subplots(1, 2, figsize=plotting.get_figsize(0.4))\n",
"axs[0].plot(polynomial_degrees, np.mean(mses, axis=1), label=\"MSE (Test)\", color=\"C0\")\n",
"\n",
"\n",
"axs[1].plot(polynomial_degrees, np.mean(biases, axis=1), label=\"Bias$^2$\", color=\"C1\")\n",
"axs[1].plot(\n",
" polynomial_degrees, np.mean(variances, axis=1), label=\"Variance\", color=\"C2\"\n",
")\n",
"if FILL_BETWEEN:\n",
" axs[0].fill_between(\n",
" polynomial_degrees,\n",
" np.mean(mses, axis=1) - np.std(mses, axis=1),\n",
" np.mean(mses, axis=1) + np.std(mses, axis=1),\n",
" color=\"C0\",\n",
" alpha=0.3,\n",
" )\n",
" axs[1].fill_between(\n",
" polynomial_degrees,\n",
" np.mean(biases, axis=1) - np.std(biases, axis=1),\n",
" np.mean(biases, axis=1) + np.std(biases, axis=1),\n",
" color=\"C1\",\n",
" alpha=0.3,\n",
" )\n",
" axs[1].fill_between(\n",
" polynomial_degrees,\n",
" np.mean(variances, axis=1) - np.std(variances, axis=1),\n",
" np.mean(variances, axis=1) + np.std(variances, axis=1),\n",
" color=\"C2\",\n",
" alpha=0.3,\n",
" )\n",
"axs[0].set_ylabel(\"Mean Squared Error\")\n",
"axs[1].set_ylabel(\"Bias and Variance\")\n",
"\n",
"for ax in axs:\n",
" ax.set_xlabel(\"Polynomial Degree\")\n",
" ax.legend()\n",
"\n",
"fig.tight_layout()\n",
"fig.savefig(os.path.join(FIG_DIR, \"bias_variance_tradeoff.pdf\"))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "20",
"metadata": {},
"outputs": [],
"source": []
}
],
src/optimizers.py
+137 -4
View File
@@ -35,10 +35,27 @@ class GradientDescent:
learning_rate (float, optional): Learning rate used for step updates. Defaults to 0.1.
num_iterations (int, optional): Number of iterations for gradient descent. Defaults to 1000.
"""
self.cost_history = np.zeros(num_iterations)
self._cost_history = np.zeros(num_iterations)
self.learning_rate = learning_rate
self.num_iterations = num_iterations
@property
def cost_history(self) -> np.ndarray:
"""Returns the cost history of the optimization process.
Returns:
np.ndarray: Array of cost values for each iteration.
"""
return self._cost_history
def get_epochs(self) -> np.ndarray:
"""Returns an array of epoch numbers from 0 to num_iterations - 1.
Returns:
np.ndarray: Array of epoch numbers.
"""
return np.arange(self.num_iterations)
def fit(self, X: np.ndarray, y: np.ndarray) -> np.ndarray:
"""Returns the optimal solution for an optimization problem using the gradient descent
@@ -55,7 +72,7 @@ class GradientDescent:
self._precomp()
for t in range(self.num_iterations):
self._comp_step()
self.cost_history[t] = self._compute_cost()
self._cost_history[t] = self._compute_cost()
self._update_theta()
return self.theta
@@ -188,21 +205,24 @@ class OLSAdam(OLSGradientDescent):
super().__init__(
*args, learning_rate=learning_rate, num_iterations=num_iterations, **kwargs
)
self.beta1 = beta1
self.beta2 = beta2
def _precomp(self):
self.m = np.zeros_like(self.theta)
self.v = np.zeros_like(self.theta)
self.current_iteration = 0
return super()._precomp()
def _update_theta(self):
self.current_iteration += 1
grad = self._compute_grad()
self.m = self.beta1 * self.m + (1 - self.beta1) * grad
self.v = self.beta2 * self.v + (1 - self.beta2) * np.square(grad)
m_hat = self.m / (1 - self.beta1 ** (self.num_iterations))
v_hat = self.v / (1 - self.beta2 ** (self.num_iterations))
m_hat = self.m / (1 - self.beta1 ** (self.current_iteration))
v_hat = self.v / (1 - self.beta2 ** (self.current_iteration))
self.theta -= (
self.learning_rate * m_hat / (np.sqrt(v_hat) + 1e-10)
@@ -244,3 +264,116 @@ class LASSORMSProp(OLSRMSProp, LASSOGradientDescent):
class LASSOAdam(OLSAdam, LASSOGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class OLSStochasticGradientDescent(OLSGradientDescent):
def __init__(
self, *args, batch_size: int = 100, batches_per_epoch: int = 1, **kwargs
):
# print(self.__class__.__name__) # If you see this: debugging yaaaay, the programmer that wrote this line is stupid...
super().__init__(*args, **kwargs)
self.batch_size = batch_size
self.batches_per_epoch = batches_per_epoch
def _precomp(self):
self.N = len(self.y)
self.indices = np.arange(self.N)
self.n = self.batch_size
def _comp_step(self):
index = np.random.randint(0, self.N)
batch_indices = slice(index, index + self.batch_size)
if index + self.batch_size > self.N:
batch_indices = slice(index, self.N)
X_batch = self.X[batch_indices]
y_batch = self.y[batch_indices]
self.err = X_batch @ self.theta - y_batch
self.XTX = X_batch.T @ X_batch
self.XTy = X_batch.T @ y_batch
def get_epochs(self):
return np.arange(self.num_iterations // self.batches_per_epoch)
@property
def cost_history(self) -> np.ndarray:
"""Returns the cost history of the optimization process.
Returns:
np.ndarray: Array of cost values for each epoch.
"""
return np.mean(self._cost_history.reshape(-1, self.batches_per_epoch), axis=1)
class RidgeStochasticGradientDescent(
OLSStochasticGradientDescent, RidgeGradientDescent
):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class LASSOStochasticGradientDescent(
OLSStochasticGradientDescent, LASSOGradientDescent
):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class OLSMomentumSGD(OLSMomentum, OLSStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class RidgeMomentumSGD(RidgeMomentum, RidgeStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class LASSOMomentumSGD(LASSOMomentum, LASSOStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class OLSAdaGradSGD(OLSAdaGrad, OLSStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class RidgeAdaGradSGD(RidgeAdaGrad, RidgeStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class LASSOAdaGradSGD(LASSOAdaGrad, LASSOStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class OLSRMSPropSGD(OLSRMSProp, OLSStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class RidgeRMSPropSGD(RidgeRMSProp, RidgeStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class LASSORMSPropSGD(LASSORMSProp, LASSOStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class OLSAdamSGD(OLSAdam, OLSStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class RidgeAdamSGD(RidgeAdam, RidgeStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class LASSOAdamSGD(LASSOAdam, LASSOStochasticGradientDescent):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)