diff --git a/figures/bias_variance_tradeoff.pdf b/figures/bias_variance_tradeoff.pdf index d91af47..5ecdcd5 100644 Binary files a/figures/bias_variance_tradeoff.pdf and b/figures/bias_variance_tradeoff.pdf differ diff --git a/figures/cost_function_comparison.pdf b/figures/cost_function_comparison.pdf index e30c1e2..4a5dd90 100644 Binary files a/figures/cost_function_comparison.pdf and b/figures/cost_function_comparison.pdf differ diff --git a/figures/data_scatter.png b/figures/data_scatter.png index 925a5db..7b3b69e 100644 Binary files a/figures/data_scatter.png and b/figures/data_scatter.png differ diff --git a/figures/gradient_descent_convergence.pdf b/figures/gradient_descent_convergence.pdf index 90091cd..186f889 100644 Binary files a/figures/gradient_descent_convergence.pdf and b/figures/gradient_descent_convergence.pdf differ diff --git a/figures/ols_mse_r2.pdf b/figures/ols_mse_r2.pdf index 755ec8d..0f132ff 100644 Binary files a/figures/ols_mse_r2.pdf and b/figures/ols_mse_r2.pdf differ diff --git a/figures/ols_parameter_plot.pdf b/figures/ols_parameter_plot.pdf index b99f023..935f78a 100644 Binary files a/figures/ols_parameter_plot.pdf and b/figures/ols_parameter_plot.pdf differ diff --git a/figures/optimizer_comparison.pdf b/figures/optimizer_comparison.pdf index 669a8fc..39a54b0 100644 Binary files a/figures/optimizer_comparison.pdf and b/figures/optimizer_comparison.pdf differ diff --git a/figures/ridge_mse_r2.pdf b/figures/ridge_mse_r2.pdf index f97d6b7..51199ef 100644 Binary files a/figures/ridge_mse_r2.pdf and b/figures/ridge_mse_r2.pdf differ diff --git a/figures/ridge_mse_r2_lambda.pdf b/figures/ridge_mse_r2_lambda.pdf index 7a0fce9..609a41f 100644 Binary files a/figures/ridge_mse_r2_lambda.pdf and b/figures/ridge_mse_r2_lambda.pdf differ diff --git a/figures/ridge_parameter_plot.pdf b/figures/ridge_parameter_plot.pdf index 965c650..15bbf7f 100644 Binary files a/figures/ridge_parameter_plot.pdf and b/figures/ridge_parameter_plot.pdf differ diff --git a/src/datamanip.py b/src/datamanip.py index db1df91..f3d8873 100644 --- a/src/datamanip.py +++ b/src/datamanip.py @@ -2,6 +2,7 @@ import numpy as np from sklearn.preprocessing import StandardScaler # type: ignore from sklearn.metrics import mean_squared_error, r2_score # type: ignore from sklearn.utils import resample # type: ignore +from sklearn.model_selection import KFold # type: ignore def polynomial_features(x: np.ndarray, p: int, intercept: bool = True) -> np.ndarray: @@ -99,3 +100,23 @@ def bootstrap_resample( X_resample, y_resample = resample(X, y) resamples.append((X_resample, y_resample)) return resamples + + +def k_fold_split( + X: np.ndarray, y: np.ndarray, k: int +) -> list[tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]]: + """Splits the dataset into k folds for cross-validation. + Args: + X: The input data matrix of shape (n_samples, n_features). + y: The target vector of shape (n_samples,). + k: The number of folds. + Returns: + A list of tuples, each containing (X_train, y_train, X_val, y_val) for each fold. + """ + kf = KFold(n_splits=k, shuffle=True) + folds = [] + for train_index, val_index in kf.split(X): + X_train, X_val = X[train_index], X[val_index] + y_train, y_val = y[train_index], y[val_index] + folds.append((X_train, y_train, X_val, y_val)) + return folds diff --git a/src/main.ipynb b/src/main.ipynb index aabefe0..284b15c 100644 --- a/src/main.ipynb +++ b/src/main.ipynb @@ -650,7 +650,7 @@ "y = datamanip.noise_data(datamanip.runge_function(x), 0.1)\n", "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", "\n", - "polynomial_degrees = np.arange(1, 30)\n", + "polynomial_degrees = np.arange(1, 50)\n", "n_bootstraps = len(x_train)\n", "mses = np.zeros((len(polynomial_degrees), n_bootstraps))\n", "biases = np.zeros((len(polynomial_degrees), n_bootstraps))\n", @@ -680,7 +680,7 @@ "metadata": {}, "outputs": [], "source": [ - "FILL_BETWEEN = False\n", + "FILL_BETWEEN = True\n", "fig, axs = plt.subplots(1, 2, figsize=plotting.get_figsize(0.4))\n", "axs[0].plot(polynomial_degrees, np.mean(mses, axis=1), label=\"MSE (Test)\", color=\"C0\")\n", "\n", @@ -716,6 +716,7 @@ "\n", "for ax in axs:\n", " ax.set_xlabel(\"Polynomial Degree\")\n", + " ax.set_yscale(\"log\")\n", " ax.legend()\n", "\n", "fig.tight_layout()\n", @@ -728,6 +729,145 @@ "id": "20", "metadata": {}, "outputs": [], + "source": [ + "k_folds = 5\n", + "k_fold_mses = np.zeros((len(polynomial_degrees), k_folds))\n", + "splits = datamanip.k_fold_split(x, y, k_folds)\n", + "\n", + "for i, polynomial_degree in enumerate(polynomial_degrees):\n", + " for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n", + " X_train = datamanip.polynomial_features(x_tr, polynomial_degree, False)\n", + " X_val = datamanip.polynomial_features(x_val, polynomial_degree, False)\n", + " X_train_scaled, X_val_scaled = datamanip.scale_data(X_train, X_val)\n", + " y_train_scaled, y_val_scaled = datamanip.scale_data(y_tr, y_val)\n", + "\n", + " beta = optimizers.Ridge_parameters(X_train_scaled, y_train_scaled, lam=1e-10)\n", + " y_pred = X_val_scaled @ beta\n", + " mse, _ = datamanip.evaluate_model(y_val_scaled, y_pred)\n", + " k_fold_mses[i, k] = mse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "FILL_BETWEEN = False\n", + "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", + "\n", + "ax.plot(\n", + " polynomial_degrees, np.mean(mses, axis=1), label=\"Bootstrapping (Test)\", color=\"C0\"\n", + ")\n", + "ax.plot(\n", + " polynomial_degrees,\n", + " np.mean(k_fold_mses, axis=1),\n", + " label=f\"{k_folds}-Fold (Test)\",\n", + " color=\"C1\",\n", + ")\n", + "if FILL_BETWEEN:\n", + " ax.fill_between(\n", + " polynomial_degrees,\n", + " np.mean(mses, axis=1) - np.std(mses, axis=1),\n", + " np.mean(mses, axis=1) + np.std(mses, axis=1),\n", + " color=\"C0\",\n", + " alpha=0.3,\n", + " )\n", + " ax.fill_between(\n", + " polynomial_degrees,\n", + " np.mean(k_fold_mses, axis=1) - np.std(k_fold_mses, axis=1),\n", + " np.mean(k_fold_mses, axis=1) + np.std(k_fold_mses, axis=1),\n", + " color=\"C1\",\n", + " alpha=0.3,\n", + " )\n", + "\n", + "ax.set_xlabel(\"Polynomial Degree\")\n", + "ax.set_ylabel(\"Mean Squared Error\")\n", + "ax.set_yscale(\"log\")\n", + "ax.legend()\n", + "fig.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "polynomial_degrees = np.arange(1, 30)\n", + "k_folds = 5\n", + "\n", + "k_fold_mses_ols = np.zeros((len(polynomial_degrees), k_folds))\n", + "k_fold_mses_ridge = np.zeros_like(k_fold_mses_ols)\n", + "k_fold_mses_lasso = np.zeros_like(k_fold_mses_ols)\n", + "\n", + "splits = datamanip.k_fold_split(x, y, k_folds)\n", + "\n", + "for i, polynomial_degree in enumerate(polynomial_degrees):\n", + " for k, (x_tr, y_tr, x_val, y_val) in enumerate(splits):\n", + " X_train = datamanip.polynomial_features(x_tr, polynomial_degree, False)\n", + " X_val = datamanip.polynomial_features(x_val, polynomial_degree, False)\n", + " X_train_scaled, X_val_scaled = datamanip.scale_data(X_train, X_val)\n", + " y_train_scaled, y_val_scaled = datamanip.scale_data(y_tr, y_val)\n", + "\n", + " for optim, results_array in zip(\n", + " [\n", + " optimizers.OLSGradientDescent,\n", + " optimizers.RidgeGradientDescent,\n", + " optimizers.LASSOGradientDescent,\n", + " ],\n", + " [k_fold_mses_ols, k_fold_mses_ridge, k_fold_mses_lasso],\n", + " ):\n", + " Optimizer = optim(num_iterations=1000, learning_rate=0.1, lam=0.1)\n", + " theta = Optimizer.fit(X_train_scaled, y_train_scaled)\n", + " y_pred = X_val_scaled @ theta\n", + " mse, _ = datamanip.evaluate_model(y_val_scaled, y_pred)\n", + " results_array[i, k] = mse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "FILL_BETWEEN = False\n", + "fig, ax = plt.subplots(figsize=plotting.get_figsize(0.5))\n", + "\n", + "for i, (mse_array, label) in enumerate(\n", + " zip(\n", + " [k_fold_mses_ols, k_fold_mses_ridge, k_fold_mses_lasso],\n", + " [\"OLS\", \"Ridge Regression\", \"Lasso Regression\"],\n", + " )\n", + "):\n", + " c = f\"C{i}\"\n", + " ax.plot(polynomial_degrees, np.mean(mse_array, axis=1), label=label, c=c)\n", + " if FILL_BETWEEN:\n", + " ax.fill_between(\n", + " polynomial_degrees,\n", + " np.mean(mse_array, axis=1) - np.std(mse_array, axis=1),\n", + " np.mean(mse_array, axis=1) + np.std(mse_array, axis=1),\n", + " color=c,\n", + " alpha=0.3,\n", + " )\n", + "\n", + "ax.set_xlabel(\"Polynomial Degree\")\n", + "ax.set_ylabel(f\"MSE ({k_folds}-fold validation)\")\n", + "# ax.set_yscale(\"log\")\n", + "# ax.set_ylim(0, 2)\n", + "ax.legend()\n", + "fig.tight_layout()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], "source": [] } ], diff --git a/src/optimizers.py b/src/optimizers.py index dc7c9dc..e91e32c 100644 --- a/src/optimizers.py +++ b/src/optimizers.py @@ -279,6 +279,9 @@ class OLSStochasticGradientDescent(OLSGradientDescent): self.N = len(self.y) self.indices = np.arange(self.N) self.n = self.batch_size + np.random.shuffle(self.indices) + self.X = self.X[self.indices] + self.y = self.y[self.indices] def _comp_step(self): index = np.random.randint(0, self.N)