From e6f4a7369f1cf70377f31fe762b51242263270c0 Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Wed, 5 Nov 2025 08:43:18 +0100 Subject: [PATCH] Run ruff --- notebooks/00_breast-cancer-exploration.ipynb | 18 ++- notebooks/01_simple-tests.ipynb | 29 ++-- notebooks/10_regression-analysis.ipynb | 142 ++++++++++++++----- notebooks/11_regression-usecase.ipynb | 58 ++++++-- notebooks/20_classification-analysis.ipynb | 86 +++++++---- notebooks/21_logisitic-regression.ipynb | 20 ++- notebooks/plotting.py | 4 +- pyproject.toml | 1 + src/easynn/feedforward.py | 65 +++++---- src/easynn/schedulers.py | 8 +- uv.lock | 11 ++ 11 files changed, 314 insertions(+), 128 deletions(-) diff --git a/notebooks/00_breast-cancer-exploration.ipynb b/notebooks/00_breast-cancer-exploration.ipynb index 2a32473..59cc627 100644 --- a/notebooks/00_breast-cancer-exploration.ipynb +++ b/notebooks/00_breast-cancer-exploration.ipynb @@ -586,8 +586,10 @@ } ], "source": [ - "data = pd.DataFrame(load_breast_cancer().data, columns=load_breast_cancer().feature_names)\n", - "data['target'] = load_breast_cancer().target\n", + "data = pd.DataFrame(\n", + " load_breast_cancer().data, columns=load_breast_cancer().feature_names\n", + ")\n", + "data[\"target\"] = load_breast_cancer().target\n", "data.head()" ] }, @@ -619,7 +621,7 @@ } ], "source": [ - "ax = sns.countplot(x='target', data=data)\n", + "ax = sns.countplot(x=\"target\", data=data)\n", "ax.set_title(\"Type of Tumors (Malignant/Benign)\")" ] }, @@ -683,14 +685,16 @@ } ], "source": [ - "df = data.melt(id_vars='target', var_name='feature', value_name='value')\n", + "df = data.melt(id_vars=\"target\", var_name=\"feature\", value_name=\"value\")\n", "\n", "GROUP_SIZE = 10\n", "for start in range(0, len(load_breast_cancer().feature_names), GROUP_SIZE):\n", " end = start + GROUP_SIZE\n", - " subset = df[df['feature'].isin(load_breast_cancer().feature_names[start:end])]\n", - " sns.violinplot(x='feature', y='value', hue='target', data=subset, split=True, inner='quart')\n", - " plt.xticks(rotation=45, ha='right')\n", + " subset = df[df[\"feature\"].isin(load_breast_cancer().feature_names[start:end])]\n", + " sns.violinplot(\n", + " x=\"feature\", y=\"value\", hue=\"target\", data=subset, split=True, inner=\"quart\"\n", + " )\n", + " plt.xticks(rotation=45, ha=\"right\")\n", " plt.tight_layout()\n", " plt.savefig(f\"violin_plot_features_{start}.pdf\")\n", "\n", diff --git a/notebooks/01_simple-tests.ipynb b/notebooks/01_simple-tests.ipynb index 7de86c6..7cb13d3 100644 --- a/notebooks/01_simple-tests.ipynb +++ b/notebooks/01_simple-tests.ipynb @@ -7,7 +7,16 @@ "metadata": {}, "outputs": [], "source": [ - "from easynn.feedforward import Layer, ReLU, Regularization, Linear, FFNN, MSELoss, Softmax, CrossEntropyLoss\n", + "from easynn.feedforward import (\n", + " Layer,\n", + " ReLU,\n", + " Regularization,\n", + " Linear,\n", + " FFNN,\n", + " MSELoss,\n", + " Softmax,\n", + " CrossEntropyLoss,\n", + ")\n", "from easynn.schedulers import AdamScheduler\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", @@ -39,7 +48,7 @@ "X = StandardScaler().fit_transform(X)\n", "y = y.reshape(-1, 1)\n", "y = StandardScaler().fit_transform(y)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y)\n" + "X_train, X_test, y_train, y_test = train_test_split(X, y)" ] }, { @@ -49,7 +58,7 @@ "metadata": {}, "outputs": [], "source": [ - "layers = [Layer(5, 5, ReLU()), Layer(5, 25, ReLU()), Layer(25, 1, Linear())]\n", + "layers = [Layer(5, 5, ReLU()), Layer(5, 25, ReLU()), Layer(25, 1, Linear())]\n", "network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), MSELoss())\n", "network.fit(X_train, y_train)\n", "predictions = network.predict(X_test)" @@ -119,7 +128,7 @@ ], "source": [ "plt.scatter(y_test, predictions)\n", - "plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=2)\n", + "plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], \"k--\", lw=2)\n", "plt.xlabel(\"True Values\")\n", "plt.ylabel(\"Predictions\")\n", "plt.title(\"True vs Predicted Values\")" @@ -165,7 +174,7 @@ "X = StandardScaler().fit_transform(X)\n", "y = y.reshape(-1, 1)\n", "y = OneHotEncoder(sparse_output=False).fit_transform(y)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y)\n" + "X_train, X_test, y_train, y_test = train_test_split(X, y)" ] }, { @@ -175,11 +184,15 @@ "metadata": {}, "outputs": [], "source": [ - "layers = [Layer(5, 10, ReLU()), Layer(10, 5, ReLU()), Layer(5, 2, Softmax())]\n", - "network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), CrossEntropyLoss())\n", + "layers = [Layer(5, 10, ReLU()), Layer(10, 5, ReLU()), Layer(5, 2, Softmax())]\n", + "network = FFNN(\n", + " layers, AdamScheduler(learning_rate=0.001, epochs=1000), CrossEntropyLoss()\n", + ")\n", "network.fit(X_train, y_train)\n", "predictions = network.predict(X_test)\n", - "single_class_predictions = OneHotEncoder(sparse_output=False).fit_transform(np.argmax(predictions, axis=1).reshape(-1, 1))\n" + "single_class_predictions = OneHotEncoder(sparse_output=False).fit_transform(\n", + " np.argmax(predictions, axis=1).reshape(-1, 1)\n", + ")" ] }, { diff --git a/notebooks/10_regression-analysis.ipynb b/notebooks/10_regression-analysis.ipynb index 759785a..65db46e 100644 --- a/notebooks/10_regression-analysis.ipynb +++ b/notebooks/10_regression-analysis.ipynb @@ -17,11 +17,18 @@ "data = load_breast_cancer()\n", "\n", "\n", - "feature_names = [n for n in data.feature_names if 'radius' in n or 'area' in n]\n", + "feature_names = [n for n in data.feature_names if \"radius\" in n or \"area\" in n]\n", "\n", "X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n", - "y = data.data[:, [data.feature_names.tolist().index(n) for n in data.feature_names if n not in feature_names]]\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n" + "y = data.data[\n", + " :,\n", + " [\n", + " data.feature_names.tolist().index(n)\n", + " for n in data.feature_names\n", + " if n not in feature_names\n", + " ],\n", + "]\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)" ] }, { @@ -31,7 +38,15 @@ "metadata": {}, "outputs": [], "source": [ - "from easynn.feedforward import Layer, FFNN, ReLU, LeakyReLU, Linear, MSELoss, Regularization\n", + "from easynn.feedforward import (\n", + " Layer,\n", + " FFNN,\n", + " ReLU,\n", + " LeakyReLU,\n", + " Linear,\n", + " MSELoss,\n", + " Regularization,\n", + ")\n", "from easynn.schedulers import AdamScheduler" ] }, @@ -44,14 +59,17 @@ "source": [ "feature_dim = X.shape[1]\n", "target_dim = y.shape[1]\n", - "def get_regression_model(n_hidden_layers: int, n_neurons: int, activation: type=ReLU) -> list[Layer]:\n", + "\n", + "\n", + "def get_regression_model(\n", + " n_hidden_layers: int, n_neurons: int, activation: type = ReLU\n", + ") -> list[Layer]:\n", " layers = []\n", " layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n", " for _ in range(n_hidden_layers - 1):\n", " layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n", " layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n", - " return layers\n", - "\n" + " return layers" ] }, { @@ -63,7 +81,11 @@ "source": [ "# First network test\n", "\n", - "model = FFNN(get_regression_model(n_hidden_layers=3, n_neurons=32, activation=LeakyReLU), AdamScheduler(epochs=10000, learning_rate=1e-2), MSELoss())\n", + "model = FFNN(\n", + " get_regression_model(n_hidden_layers=3, n_neurons=32, activation=LeakyReLU),\n", + " AdamScheduler(epochs=10000, learning_rate=1e-2),\n", + " MSELoss(),\n", + ")\n", "model.fit(X_train, y_train)" ] }, @@ -100,10 +122,14 @@ "import matplotlib.pyplot as plt\n", "\n", "y_pred = model.predict(X_test)\n", - "mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n", + "mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n", "\n", "sns.barplot(x=np.arange(len(mse)), y=mse)\n", - "plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n", + "plt.xticks(\n", + " ticks=np.arange(len(mse)),\n", + " labels=[n for n in data.feature_names if n not in feature_names],\n", + " rotation=90,\n", + ")\n", "plt.yscale(\"log\")\n", "plt.ylabel(\"Mean Squared Error\")" ] @@ -131,15 +157,23 @@ "n_neurons_list = [4, 8, 16, 32, 64, 128]\n", "mse_results = np.zeros((len(n_layers_list), len(n_neurons_list)))\n", "for idx, (n_layers, n_neurons) in enumerate(\n", - " tqdm(product(n_layers_list, n_neurons_list),\n", - " total=len(n_layers_list) * len(n_neurons_list))\n", + " tqdm(\n", + " product(n_layers_list, n_neurons_list),\n", + " total=len(n_layers_list) * len(n_neurons_list),\n", + " )\n", "):\n", " i, j = divmod(idx, len(n_neurons_list))\n", - " model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=15000, learning_rate=1e-3), MSELoss())\n", + " model = FFNN(\n", + " get_regression_model(\n", + " n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU\n", + " ),\n", + " AdamScheduler(epochs=15000, learning_rate=1e-3),\n", + " MSELoss(),\n", + " )\n", " model.fit(X_train, y_train)\n", " y_pred = model.predict(X_test)\n", " mse = mean_squared_error(y_test, y_pred)\n", - " mse_results[i, j] = mse\n" + " mse_results[i, j] = mse" ] }, { @@ -160,10 +194,16 @@ } ], "source": [ - "sns.heatmap(mse_results, xticklabels=n_neurons_list, yticklabels=n_layers_list, annot=True, fmt=\".2f\")\n", + "sns.heatmap(\n", + " mse_results,\n", + " xticklabels=n_neurons_list,\n", + " yticklabels=n_layers_list,\n", + " annot=True,\n", + " fmt=\".2f\",\n", + ")\n", "plt.xlabel(\"Number of Neurons\")\n", "plt.ylabel(\"Number of Layers\")\n", - "plt.savefig(\"nodenumber_tuning_regression.pdf\")\n" + "plt.savefig(\"nodenumber_tuning_regression.pdf\")" ] }, { @@ -177,10 +217,16 @@ "parameters = [(1, 32), (1, 64), (2, 32), (2, 64), (3, 64), (3, 128)]\n", "mse_results_slow_scan = np.zeros((len(parameters), target_dim))\n", "for k, (n_layers, n_neurons) in enumerate(parameters):\n", - " model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=20000, learning_rate=5e-4), MSELoss())\n", + " model = FFNN(\n", + " get_regression_model(\n", + " n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU\n", + " ),\n", + " AdamScheduler(epochs=20000, learning_rate=5e-4),\n", + " MSELoss(),\n", + " )\n", " model.fit(X_train, y_train)\n", " y_pred = model.predict(X_test)\n", - " mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n", + " mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n", " mse_results_slow_scan[k, :] = mse" ] }, @@ -202,12 +248,16 @@ } ], "source": [ - "sns.heatmap(np.log10(mse_results_slow_scan), xticklabels=[n for n in data.feature_names if n not in feature_names], yticklabels=[f\"L{l}_N{n}\" for l, n in parameters],)\n", - "plt.xticks(rotation=45, ha='right')\n", + "sns.heatmap(\n", + " np.log10(mse_results_slow_scan),\n", + " xticklabels=[n for n in data.feature_names if n not in feature_names],\n", + " yticklabels=[f\"L{l}_N{n}\" for l, n in parameters],\n", + ")\n", + "plt.xticks(rotation=45, ha=\"right\")\n", "plt.xlabel(\"Target Features\")\n", "plt.ylabel(\"Model (Layers_Neurons)\")\n", "plt.tight_layout()\n", - "plt.savefig(\"regression_detailed_hyperparameter_scan.pdf\")\n" + "plt.savefig(\"regression_detailed_hyperparameter_scan.pdf\")" ] }, { @@ -241,7 +291,7 @@ " x=log_mse.ravel(),\n", " y=np.repeat(np.arange(len(parameters)), log_mse.shape[1]),\n", " bins=[np.linspace(-5, 2, 15), len(parameters)],\n", - " cmap=cmap\n", + " cmap=cmap,\n", ")\n", "\n", "# Label axes\n", @@ -266,7 +316,7 @@ "outputs": [], "source": [ "n_hidden_layers = 2\n", - "n_neurons = 32\n" + "n_neurons = 32" ] }, { @@ -276,14 +326,15 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "# regularization test\n", - "layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n", + "layers = get_regression_model(\n", + " n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU\n", + ")\n", "regularization_strengths = [0.0, 1e-5, 1e-4, 1e-3, 1e-2]\n", "mse_results_reg = np.zeros(len(regularization_strengths))\n", "for i, reg_strength in enumerate(regularization_strengths):\n", " for layer in layers:\n", - " layer.regularization = Regularization(reg_strength, 'l2')\n", + " layer.regularization = Regularization(reg_strength, \"l2\")\n", " model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n", " model.fit(X_train, y_train)\n", " y_pred = model.predict(X_test)\n", @@ -335,9 +386,11 @@ "outputs": [], "source": [ "# Check final model training\n", - "layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n", + "layers = get_regression_model(\n", + " n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU\n", + ")\n", "for layer in layers:\n", - " layer.regularization = Regularization(reg_strength, 'l2')\n", + " layer.regularization = Regularization(reg_strength, \"l2\")\n", "model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n", "model.fit(X_train, y_train)" ] @@ -361,7 +414,7 @@ ], "source": [ "sns.lineplot(x=np.arange(20000), y=model.scheduler.loss_history)\n", - "plt.yscale(\"log\")\n" + "plt.yscale(\"log\")" ] }, { @@ -383,10 +436,14 @@ ], "source": [ "y_pred = model.predict(X_test)\n", - "mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n", + "mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n", "\n", "sns.barplot(x=np.arange(len(mse)), y=mse)\n", - "plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n", + "plt.xticks(\n", + " ticks=np.arange(len(mse)),\n", + " labels=[n for n in data.feature_names if n not in feature_names],\n", + " rotation=90,\n", + ")\n", "plt.yscale(\"log\")\n", "plt.ylabel(\"Mean Squared Error\")\n", "plt.tight_layout()\n", @@ -425,16 +482,23 @@ "source": [ "beta = optimizers.OLS_parameters(X_train, y_train)\n", "y_ols_pred = X_test @ beta\n", - "mse_ols = mean_squared_error(y_test, y_ols_pred, multioutput='raw_values')\n", + "mse_ols = mean_squared_error(y_test, y_ols_pred, multioutput=\"raw_values\")\n", "\n", - "mse_df = pd.DataFrame({\n", - " \"Feature\": [n for n in data.feature_names if n not in feature_names],\n", - " \"Neural Network\": mse,\n", - " \"OLS\": mse_ols\n", - "})\n", + "mse_df = pd.DataFrame(\n", + " {\n", + " \"Feature\": [n for n in data.feature_names if n not in feature_names],\n", + " \"Neural Network\": mse,\n", + " \"OLS\": mse_ols,\n", + " }\n", + ")\n", "\n", - "sns.barplot(x=\"Feature\", y=\"value\", hue=\"Model\", data=pd.melt(mse_df, id_vars=[\"Feature\"]).rename(columns={\"variable\": \"Model\"}))\n", - "plt.xticks(rotation=45, ha='right')\n", + "sns.barplot(\n", + " x=\"Feature\",\n", + " y=\"value\",\n", + " hue=\"Model\",\n", + " data=pd.melt(mse_df, id_vars=[\"Feature\"]).rename(columns={\"variable\": \"Model\"}),\n", + ")\n", + "plt.xticks(rotation=45, ha=\"right\")\n", "plt.yscale(\"log\")\n", "plt.ylabel(\"Mean Squared Error\")\n", "plt.xlabel(\"Target Feature\")\n", diff --git a/notebooks/11_regression-usecase.ipynb b/notebooks/11_regression-usecase.ipynb index c1b6cf1..d562e98 100644 --- a/notebooks/11_regression-usecase.ipynb +++ b/notebooks/11_regression-usecase.ipynb @@ -24,27 +24,50 @@ "data = load_breast_cancer()\n", "\n", "\n", - "feature_names = [n for n in data.feature_names if 'radius' in n or 'area' in n]\n", + "feature_names = [n for n in data.feature_names if \"radius\" in n or \"area\" in n]\n", "\n", "X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n", "X = StandardScaler().fit_transform(X)\n", - "y = data.data[:, [data.feature_names.tolist().index(n) for n in data.feature_names if n not in feature_names]]\n", + "y = data.data[\n", + " :,\n", + " [\n", + " data.feature_names.tolist().index(n)\n", + " for n in data.feature_names\n", + " if n not in feature_names\n", + " ],\n", + "]\n", "y = StandardScaler().fit_transform(y)\n", "feature_dim = X.shape[1]\n", "target_dim = y.shape[1]\n", "\n", - "def get_regression_model(n_hidden_layers: int, n_neurons: int, activation: type=LeakyReLU, regularization_strength: float = 1e-3) -> list[Layer]:\n", + "\n", + "def get_regression_model(\n", + " n_hidden_layers: int,\n", + " n_neurons: int,\n", + " activation: type = LeakyReLU,\n", + " regularization_strength: float = 1e-3,\n", + ") -> list[Layer]:\n", " layers = []\n", " layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n", " for _ in range(n_hidden_layers - 1):\n", " layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n", " layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n", " for layer in layers:\n", - " layer.regularization = Regularization(regularization_strength, 'l2')\n", + " layer.regularization = Regularization(regularization_strength, \"l2\")\n", " return layers\n", "\n", + "\n", "def get_model():\n", - " return FFNN(get_regression_model(n_hidden_layers=2, n_neurons=32, activation=LeakyReLU, regularization_strength=1e-3), AdamScheduler(epochs=10000, learning_rate=1e-2), MSELoss())" + " return FFNN(\n", + " get_regression_model(\n", + " n_hidden_layers=2,\n", + " n_neurons=32,\n", + " activation=LeakyReLU,\n", + " regularization_strength=1e-3,\n", + " ),\n", + " AdamScheduler(epochs=10000, learning_rate=1e-2),\n", + " MSELoss(),\n", + " )" ] }, { @@ -57,8 +80,12 @@ "crossvalidation_groups = 5\n", "group_length = X.shape[0] // crossvalidation_groups\n", "predictions = np.zeros_like(y)\n", - "for g in range(crossvalidation_groups + 1): # One final smaller group to catch the rest\n", - " indices = [i for i in range(X.shape[0]) if i < g*group_length or i > (g+1)*group_length]\n", + "for g in range(crossvalidation_groups + 1): # One final smaller group to catch the rest\n", + " indices = [\n", + " i\n", + " for i in range(X.shape[0])\n", + " if i < g * group_length or i > (g + 1) * group_length\n", + " ]\n", " prediction_indices = [i for i in range(X.shape[0]) if i not in indices]\n", " X_train = X[indices]\n", " X_pred = X[prediction_indices]\n", @@ -66,8 +93,7 @@ "\n", " model = get_model()\n", " model.fit(X_train, y_train)\n", - " predictions[prediction_indices] = model.predict(X_pred)\n", - "\n" + " predictions[prediction_indices] = model.predict(X_pred)" ] }, { @@ -122,15 +148,21 @@ "outputs": [], "source": [ "combined_data = np.zeros_like(data.data)\n", - "combined_data[:,:6] = X\n", + "combined_data[:, :6] = X\n", "combined_data[:, 6:] = predictions\n", "\n", - "combined_df = pd.DataFrame(combined_data, columns=[*feature_names, *[n for n in data.feature_names if n not in feature_names]])\n", - "combined_df['target'] = data.target\n", + "combined_df = pd.DataFrame(\n", + " combined_data,\n", + " columns=[\n", + " *feature_names,\n", + " *[n for n in data.feature_names if n not in feature_names],\n", + " ],\n", + ")\n", + "combined_df[\"target\"] = data.target\n", "combined_df.to_csv(\"breast_cancer_regression_results.csv\", index=False)\n", "\n", "original_df = pd.DataFrame(data.data, columns=data.feature_names)\n", - "original_df['target'] = data.target\n", + "original_df[\"target\"] = data.target\n", "original_df.to_csv(\"breast_cancer_original_data.csv\", index=False)" ] }, diff --git a/notebooks/20_classification-analysis.ipynb b/notebooks/20_classification-analysis.ipynb index 12cc117..59b138b 100644 --- a/notebooks/20_classification-analysis.ipynb +++ b/notebooks/20_classification-analysis.ipynb @@ -7,8 +7,21 @@ "metadata": {}, "outputs": [], "source": [ - "from easynn.feedforward import FFNN, Layer, Regularization, LeakyReLU, Softmax, CrossEntropyLoss\n", - "from easynn.schedulers import GradientDescentScheduler, AdamScheduler, RMSPropScheduler, AdaGradScheduler, MomentumScheduler\n" + "from easynn.feedforward import (\n", + " FFNN,\n", + " Layer,\n", + " Regularization,\n", + " LeakyReLU,\n", + " Softmax,\n", + " CrossEntropyLoss,\n", + ")\n", + "from easynn.schedulers import (\n", + " GradientDescentScheduler,\n", + " AdamScheduler,\n", + " RMSPropScheduler,\n", + " AdaGradScheduler,\n", + " MomentumScheduler,\n", + ")" ] }, { @@ -18,7 +31,12 @@ "metadata": {}, "outputs": [], "source": [ - "def get_classification_model(n_hidden_layers: int, n_neurons: int, activation: type=LeakyReLU, reg_strength: float=0.0) -> list[Layer]:\n", + "def get_classification_model(\n", + " n_hidden_layers: int,\n", + " n_neurons: int,\n", + " activation: type = LeakyReLU,\n", + " reg_strength: float = 0.0,\n", + ") -> list[Layer]:\n", " target_dim = 2\n", " feature_dim = 30\n", " layers = []\n", @@ -2337,21 +2355,32 @@ "source": [ "from sklearn.metrics import accuracy_score, roc_auc_score\n", "\n", - "def test_classification_model(n_hidden_layers: int, n_neurons: int, reg_strength: float, n_iterations: int=50) -> tuple[float, float, FFNN]:\n", + "\n", + "def test_classification_model(\n", + " n_hidden_layers: int, n_neurons: int, reg_strength: float, n_iterations: int = 50\n", + ") -> tuple[float, float, FFNN]:\n", " assert n_iterations > 0, \"n_iterations must be greater than 0\"\n", " accs = []\n", " aucs = []\n", " for _ in range(n_iterations): # Run n_iterations trials\n", " X_train, X_test, y_train, y_test = train_test_split(X, y_encoded, test_size=0.2)\n", - " layers = get_classification_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, reg_strength=reg_strength)\n", - " model = FFNN(layers, AdamScheduler(epochs=500, learning_rate=0.05), loss_fn=CrossEntropyLoss())\n", + " layers = get_classification_model(\n", + " n_hidden_layers=n_hidden_layers,\n", + " n_neurons=n_neurons,\n", + " reg_strength=reg_strength,\n", + " )\n", + " model = FFNN(\n", + " layers,\n", + " AdamScheduler(epochs=500, learning_rate=0.05),\n", + " loss_fn=CrossEntropyLoss(),\n", + " )\n", " model.fit(X_train, y_train)\n", " y_pred = model.predict(X_test)\n", " acc = accuracy_score(y_test.argmax(axis=1), y_pred.argmax(axis=1))\n", " auc = roc_auc_score(y_test, y_pred)\n", " accs.append(acc)\n", " aucs.append(auc)\n", - " return np.mean(accs), np.mean(aucs), model\n" + " return np.mean(accs), np.mean(aucs), model" ] }, { @@ -2389,14 +2418,18 @@ "):\n", " ...\n", "\n", - " acc, auc, model = test_classification_model(n_hidden_layers, n_neurons, reg_strength, 2)\n", - " results.append({\n", - " \"n_hidden_layers\": n_hidden_layers,\n", - " \"n_neurons\": n_neurons,\n", - " \"reg_strength\": reg_strength,\n", - " \"accuracy\": acc,\n", - " \"auc\": auc,\n", - " })\n", + " acc, auc, model = test_classification_model(\n", + " n_hidden_layers, n_neurons, reg_strength, 2\n", + " )\n", + " results.append(\n", + " {\n", + " \"n_hidden_layers\": n_hidden_layers,\n", + " \"n_neurons\": n_neurons,\n", + " \"reg_strength\": reg_strength,\n", + " \"accuracy\": acc,\n", + " \"auc\": auc,\n", + " }\n", + " )\n", "\n", " # You can use AUC as the main criterion, or combine them\n", " score = auc + 0.1 * acc\n", @@ -2449,21 +2482,16 @@ " if i == j:\n", " # Diagonal: 1D line plot of AUC vs parameter\n", " sns.lineplot(\n", - " data=results_df,\n", - " x=xparam,\n", - " y=\"auc\",\n", - " marker=\"o\",\n", - " ax=ax,\n", - " color=\"C0\"\n", + " data=results_df, x=xparam, y=\"auc\", marker=\"o\", ax=ax, color=\"C0\"\n", " )\n", " ax.set_xlabel(xparam)\n", " ax.set_ylabel(\"AUC\")\n", " if i == 2:\n", - " ax.set_xscale('log')\n", + " ax.set_xscale(\"log\")\n", " else:\n", " # Off-diagonal: 2D heatmap colored by AUC\n", " hm = results_df.pivot_table(index=yparam, columns=xparam, values=\"auc\")\n", - " sc = ax.imshow(hm, origin='lower', aspect='auto', cmap=cmap)\n", + " sc = ax.imshow(hm, origin=\"lower\", aspect=\"auto\", cmap=cmap)\n", " ax.set_xticks(np.arange(len(hm.columns)))\n", " ax.set_yticks(np.arange(len(hm.index)))\n", " ax.set_xticklabels(hm.columns)\n", @@ -2478,7 +2506,7 @@ "fig.suptitle(\"Pairwise Hyperparameter Relationships (AUC)\", fontsize=18)\n", "plt.tight_layout(rect=[0, 0, 0.9, 0.97])\n", "plt.savefig(\"classification_hyperparameter_scan.pdf\")\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -2500,8 +2528,10 @@ "n_neurons = 64\n", "reg_strength = 1e-6\n", "\n", - "acc, auc, best_model = test_classification_model(n_hidden_layers, n_neurons, reg_strength)\n", - "print(f\"Best Model - Accuracy: {acc:.4f}, AUC: {auc:.4f}\")\n" + "acc, auc, best_model = test_classification_model(\n", + " n_hidden_layers, n_neurons, reg_strength\n", + ")\n", + "print(f\"Best Model - Accuracy: {acc:.4f}, AUC: {auc:.4f}\")" ] }, { @@ -2545,7 +2575,7 @@ } ], "source": [ - "sns.heatmap(cm, annot=True, fmt='d')\n", + "sns.heatmap(cm, annot=True, fmt=\"d\")\n", "plt.xlabel(\"Predicted Label\")\n", "plt.ylabel(\"True Label\")\n", "plt.tight_layout()\n", @@ -2572,7 +2602,7 @@ ], "source": [ "sns.lineplot(x=fpr, y=tpr)\n", - "plt.plot([0, 1], [0, 1], linestyle='--', color='gray')\n", + "plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\")\n", "plt.xlabel(\"False Positive Rate\")\n", "plt.ylabel(\"True Positive Rate\")\n", "plt.tight_layout()\n", diff --git a/notebooks/21_logisitic-regression.ipynb b/notebooks/21_logisitic-regression.ipynb index e56a549..41675d0 100644 --- a/notebooks/21_logisitic-regression.ipynb +++ b/notebooks/21_logisitic-regression.ipynb @@ -7,7 +7,14 @@ "metadata": {}, "outputs": [], "source": [ - "from easynn.feedforward import FFNN, Layer, Regularization, LeakyReLU, Softmax, CrossEntropyLoss\n", + "from easynn.feedforward import (\n", + " FFNN,\n", + " Layer,\n", + " Regularization,\n", + " LeakyReLU,\n", + " Softmax,\n", + " CrossEntropyLoss,\n", + ")\n", "from easynn.schedulers import AdamScheduler\n", "\n", "import pandas as pd\n", @@ -38,9 +45,12 @@ "metadata": {}, "outputs": [], "source": [ - "logistic_model = FFNN([\n", - " Layer(30, 2, activation_function=Softmax()),\n", - "], loss_fn=CrossEntropyLoss(), scheduler=AdamScheduler(learning_rate=5e-4, epochs=25000)\n", + "logistic_model = FFNN(\n", + " [\n", + " Layer(30, 2, activation_function=Softmax()),\n", + " ],\n", + " loss_fn=CrossEntropyLoss(),\n", + " scheduler=AdamScheduler(learning_rate=5e-4, epochs=25000),\n", ")" ] }, @@ -155,7 +165,7 @@ "\n", "cm = confusion_matrix(y_test.argmax(axis=1), y_pred.argmax(axis=1))\n", "\n", - "sns.heatmap(cm, annot=True, fmt='d')\n", + "sns.heatmap(cm, annot=True, fmt=\"d\")\n", "plt.xlabel(\"Predicted Label\")\n", "plt.ylabel(\"True Label\")\n", "plt.tight_layout()\n", diff --git a/notebooks/plotting.py b/notebooks/plotting.py index c359210..fb4424f 100644 --- a/notebooks/plotting.py +++ b/notebooks/plotting.py @@ -1,5 +1,6 @@ import matplotlib.pyplot as plt + def get_rc_params(): colors = ["FF220C", "70D6FF", "8AAA79", "666370", "1C1F33"] rcParams = plt.rcParams @@ -29,4 +30,5 @@ def get_rc_params(): rcParams["ytick.right"] = True return rcParams -plt.rcParams.update(get_rc_params()) \ No newline at end of file + +plt.rcParams.update(get_rc_params()) diff --git a/pyproject.toml b/pyproject.toml index 2ce09eb..13865c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "pytest>=8.4.2", "ruff>=0.14.1", "scikit-learn>=1.7.2", + "scikit-learn-stubs>=0.0.3", "seaborn>=0.13.2", "tqdm>=4.67.1", ] diff --git a/src/easynn/feedforward.py b/src/easynn/feedforward.py index 7be1905..2fd4ee4 100644 --- a/src/easynn/feedforward.py +++ b/src/easynn/feedforward.py @@ -37,13 +37,15 @@ class LeakyReLU(ActivationFunction): def backward(self, values: np.ndarray) -> np.ndarray: return np.where(values >= 0, 1.0, self.leak) + class Linear(ActivationFunction): def forward(self, values: np.ndarray) -> np.ndarray: return values - + def backward(self, values: np.ndarray) -> np.ndarray: return np.ones_like(values) + class Softmax(ActivationFunction): def forward(self, values: np.ndarray) -> np.ndarray: exp_values = np.exp(values - np.max(values, axis=1, keepdims=True)) @@ -52,7 +54,8 @@ class Softmax(ActivationFunction): def backward(self, values: np.ndarray) -> np.ndarray: s = self.forward(values) return s * (1 - s) - + + # === Loss Functions === @@ -76,7 +79,7 @@ class CrossEntropyLoss(LossFunction): def __init__(self) -> None: super().__init__() self._override_activation_loss: bool = False # Used if activation and loss are combined for efficiency; Set by FFNN if needed; - + def set_override_activation_loss(self, override: bool = True) -> None: self._override_activation_loss = override @@ -84,32 +87,34 @@ class CrossEntropyLoss(LossFunction): eps = 1e-9 y_pred = np.clip(y_pred, eps, 1 - eps) return -float(np.mean(np.sum(y_true * np.log(y_pred), axis=1))) - def backward(self, y_pred: np.ndarray, y_true: np.ndarray) -> np.ndarray: if not self._override_activation_loss: eps = 1e-9 y_pred = np.clip(y_pred, eps, 1 - eps) - return - (y_true / y_pred) / y_true.shape[0] + return -(y_true / y_pred) / y_true.shape[0] else: return (y_pred - y_true) / y_true.shape[0] + # === Regularization Functions === class Regularization: - def __init__(self, reg_lambda: float, mode: Literal['l1', 'l2'] = 'l2') -> None: + def __init__(self, reg_lambda: float, mode: Literal["l1", "l2"] = "l2") -> None: self.reg_lambda = reg_lambda - if mode not in ['l1', 'l2']: + if mode not in ["l1", "l2"]: raise ValueError("mode must be 'l1' or 'l2'") self.mode = mode def compute_penalty(self, weights: np.ndarray, biases: np.ndarray) -> float: - if self.mode == 'l2': - return self.reg_lambda * (np.sum(weights ** 2) + np.sum(biases ** 2)) + if self.mode == "l2": + return self.reg_lambda * (np.sum(weights**2) + np.sum(biases**2)) else: # l1 return self.reg_lambda * (np.sum(np.abs(weights)) + np.sum(np.abs(biases))) - def compute_gradient(self, weights: np.ndarray, biases: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - if self.mode == 'l2': + def compute_gradient( + self, weights: np.ndarray, biases: np.ndarray + ) -> Tuple[np.ndarray, np.ndarray]: + if self.mode == "l2": return 2 * self.reg_lambda * weights, 2 * self.reg_lambda * biases else: # l1 return self.reg_lambda * np.sign(weights), self.reg_lambda * np.sign(biases) @@ -120,21 +125,25 @@ class Regularization: class Layer: def __init__( - self, input_dim: int, num_nodes: int, activation_function: ActivationFunction, regularization: Optional[Regularization] = None + self, + input_dim: int, + num_nodes: int, + activation_function: ActivationFunction, + regularization: Optional[Regularization] = None, ) -> None: self.input_dim, self.num_nodes = input_dim, num_nodes # Initialize weights and biases identically to PyTorch's default initialization - stdv = 1. / np.sqrt(input_dim * num_nodes) + stdv = 1.0 / np.sqrt(input_dim * num_nodes) self.weights = np.random.rand(input_dim, num_nodes) * 2 * stdv - stdv self.biases = np.random.rand(1, num_nodes) * 2 * stdv - stdv - + self.activation_function = activation_function self.last_input: Optional[np.ndarray] = None self.last_z: Optional[np.ndarray] = None # Pre-activation values self.regularization = regularization - self._override_activation_loss: bool = False # Used if activation and loss are combined for efficiency; Set by FFNN if needed; + self._override_activation_loss: bool = False # Used if activation and loss are combined for efficiency; Set by FFNN if needed; def set_override_activation_loss(self, override: bool = True) -> None: self._override_activation_loss = override @@ -161,14 +170,16 @@ class Layer: activation_grad = np.ones_like(self.last_z) else: activation_grad = self.activation_function.backward(self.last_z) - + local_grad = upstream_grad * activation_grad # Gradients for weights and biases bias_grad = np.sum(local_grad, axis=0, keepdims=True) / batch_size weights_grad = (self.last_input.T @ local_grad) / batch_size if self.regularization is not None: - reg_weights_grad, reg_biases_grad = self.regularization.compute_gradient(self.weights, self.biases) + reg_weights_grad, reg_biases_grad = self.regularization.compute_gradient( + self.weights, self.biases + ) weights_grad += reg_weights_grad bias_grad += reg_biases_grad @@ -176,7 +187,7 @@ class Layer: previous_grad = local_grad @ self.weights.T return previous_grad, (bias_grad, weights_grad) - + def compute_regularization_penalty(self) -> float: if self.regularization is not None: return self.regularization.compute_penalty(self.weights, self.biases) @@ -198,7 +209,6 @@ class FFNN: self.loss_fn.set_override_activation_loss(True) if isinstance(self.layers[-1].activation_function, Softmax): self.layers[-1].set_override_activation_loss(True) - def predict(self, X: np.ndarray) -> np.ndarray: if X.ndim == 1: @@ -221,11 +231,13 @@ class FFNN: # Compute loss and gradient loss_value = self.loss_fn.forward(y_pred, y) - logger.debug(f"Iteration {iteration + 1}, Pre-Regularization Loss: {loss_value}") + logger.debug( + f"Iteration {iteration + 1}, Pre-Regularization Loss: {loss_value}" + ) # Add regularization penalties for layer in self.layers: loss_value += layer.compute_regularization_penalty() - + loss_grad = self.loss_fn.backward(y_pred, y) # Backward pass @@ -251,10 +263,11 @@ class FFNN: # === Example usage === if __name__ == "__main__": - from easynn.schedulers import GradientDescentScheduler, AdamScheduler + from easynn.schedulers import AdamScheduler from sklearn.preprocessing import StandardScaler + x = np.linspace(0, 1, 100) - X = np.array([x, x + np.random.randn(len(x)), x + np.random.randn(len(x))*2]).T + X = np.array([x, x + np.random.randn(len(x)), x + np.random.randn(len(x)) * 2]).T X_scaler = StandardScaler().fit(X) X_s = X_scaler.transform(X) y = np.array([x + x**2 + x**3]).T @@ -262,7 +275,11 @@ if __name__ == "__main__": y_s = y_scaler.transform(y) print(X_s) print(y_s) - layers = [Layer(3, 5, ReLU(), Regularization(0.01, 'l2')), Layer(5, 25, ReLU(), Regularization(0.01, 'l2')), Layer(25, 1, Linear())] + layers = [ + Layer(3, 5, ReLU(), Regularization(0.01, "l2")), + Layer(5, 25, ReLU(), Regularization(0.01, "l2")), + Layer(25, 1, Linear()), + ] network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), MSELoss()) network.fit(X_s, y_s) print(np.abs(np.mean(network.predict(X_s) - y_s))) diff --git a/src/easynn/schedulers.py b/src/easynn/schedulers.py index cfa0697..cd857e5 100644 --- a/src/easynn/schedulers.py +++ b/src/easynn/schedulers.py @@ -14,10 +14,10 @@ class Scheduler: @property def cont(self) -> bool: return False - + def record_loss(self, loss: float) -> None: self._loss_history.append(loss) - + @property def loss_history(self) -> np.ndarray: return np.array(self._loss_history) @@ -71,7 +71,9 @@ class AdvancedScheduler(BasicScheduler): ) for i, (new_w_grad, new_b_grad) in enumerate(gradients) ] - logger.debug(f"Updates computed. Total update sum: {sum(np.sum(u) for pair in updates for u in pair)}") + logger.debug( + f"Updates computed. Total update sum: {sum(np.sum(u) for pair in updates for u in pair)}" + ) self._post_update(gradients, updates) return updates diff --git a/uv.lock b/uv.lock index 6cc48c7..7039099 100644 --- a/uv.lock +++ b/uv.lock @@ -187,6 +187,7 @@ dependencies = [ { name = "pytest" }, { name = "ruff" }, { name = "scikit-learn" }, + { name = "scikit-learn-stubs" }, { name = "seaborn" }, { name = "tqdm" }, ] @@ -202,6 +203,7 @@ requires-dist = [ { name = "pytest", specifier = ">=8.4.2" }, { name = "ruff", specifier = ">=0.14.1" }, { name = "scikit-learn", specifier = ">=1.7.2" }, + { name = "scikit-learn-stubs", specifier = ">=0.0.3" }, { name = "seaborn", specifier = ">=0.13.2" }, { name = "tqdm", specifier = ">=4.67.1" }, ] @@ -976,6 +978,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/87/24f541b6d62b1794939ae6422f8023703bbf6900378b2b34e0b4384dfefd/scikit_learn-1.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:bb24510ed3f9f61476181e4db51ce801e2ba37541def12dc9333b946fc7a9cf8", size = 8820007, upload-time = "2025-09-09T08:21:26.713Z" }, ] +[[package]] +name = "scikit-learn-stubs" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/12/35f54848f6160ea53164ef8919d210e16c5d4f78be32a7605254651d32b7/scikit_learn_stubs-0.0.3.tar.gz", hash = "sha256:6a41fd3e26aedb923298c3f68a0746d2eed757f24615173f900ce2b942d70814", size = 123044, upload-time = "2025-08-28T01:26:49.584Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/ee/8c483b9384f269a14f62fa9760d0021273d1786128273ee14962468d9114/scikit_learn_stubs-0.0.3-py3-none-any.whl", hash = "sha256:5d51257ab62c79d265fa7b05e69ad64b038ac7803bfb1d6490562d4da3109aae", size = 247033, upload-time = "2025-08-28T01:26:48.199Z" }, +] + [[package]] name = "scipy" version = "1.16.2"