lars/FYSSTK-Project2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run ruff

Browse Source
This commit is contained in:
Lars Bogner
2025-11-05 08:43:18 +01:00
parent 6466f07935
commit e6f4a7369f
11 changed files with 314 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/00_breast-cancer-exploration.ipynb
+11 -7
View File
@@ -586,8 +586,10 @@
} }
], ],
"source": [ "source": [
"data = pd.DataFrame(load_breast_cancer().data, columns=load_breast_cancer().feature_names)\n", "data = pd.DataFrame(\n",
"data['target'] = load_breast_cancer().target\n", " load_breast_cancer().data, columns=load_breast_cancer().feature_names\n",
")\n",
"data[\"target\"] = load_breast_cancer().target\n",
"data.head()" "data.head()"
] ]
}, },
@@ -619,7 +621,7 @@
} }
], ],
"source": [ "source": [
"ax = sns.countplot(x='target', data=data)\n", "ax = sns.countplot(x=\"target\", data=data)\n",
"ax.set_title(\"Type of Tumors (Malignant/Benign)\")" "ax.set_title(\"Type of Tumors (Malignant/Benign)\")"
] ]
}, },
@@ -683,14 +685,16 @@
} }
], ],
"source": [ "source": [
"df = data.melt(id_vars='target', var_name='feature', value_name='value')\n", "df = data.melt(id_vars=\"target\", var_name=\"feature\", value_name=\"value\")\n",
"\n", "\n",
"GROUP_SIZE = 10\n", "GROUP_SIZE = 10\n",
"for start in range(0, len(load_breast_cancer().feature_names), GROUP_SIZE):\n", "for start in range(0, len(load_breast_cancer().feature_names), GROUP_SIZE):\n",
" end = start + GROUP_SIZE\n", " end = start + GROUP_SIZE\n",
" subset = df[df['feature'].isin(load_breast_cancer().feature_names[start:end])]\n", " subset = df[df[\"feature\"].isin(load_breast_cancer().feature_names[start:end])]\n",
" sns.violinplot(x='feature', y='value', hue='target', data=subset, split=True, inner='quart')\n", " sns.violinplot(\n",
" plt.xticks(rotation=45, ha='right')\n", " x=\"feature\", y=\"value\", hue=\"target\", data=subset, split=True, inner=\"quart\"\n",
" )\n",
" plt.xticks(rotation=45, ha=\"right\")\n",
" plt.tight_layout()\n", " plt.tight_layout()\n",
" plt.savefig(f\"violin_plot_features_{start}.pdf\")\n", " plt.savefig(f\"violin_plot_features_{start}.pdf\")\n",
"\n", "\n",
notebooks/01_simple-tests.ipynb
+21 -8
View File
@@ -7,7 +7,16 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"from easynn.feedforward import Layer, ReLU, Regularization, Linear, FFNN, MSELoss, Softmax, CrossEntropyLoss\n", "from easynn.feedforward import (\n",
" Layer,\n",
" ReLU,\n",
" Regularization,\n",
" Linear,\n",
" FFNN,\n",
" MSELoss,\n",
" Softmax,\n",
" CrossEntropyLoss,\n",
")\n",
"from easynn.schedulers import AdamScheduler\n", "from easynn.schedulers import AdamScheduler\n",
"import numpy as np\n", "import numpy as np\n",
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
@@ -39,7 +48,7 @@
"X = StandardScaler().fit_transform(X)\n", "X = StandardScaler().fit_transform(X)\n",
"y = y.reshape(-1, 1)\n", "y = y.reshape(-1, 1)\n",
"y = StandardScaler().fit_transform(y)\n", "y = StandardScaler().fit_transform(y)\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y)\n" "X_train, X_test, y_train, y_test = train_test_split(X, y)"
] ]
}, },
{ {
@@ -49,7 +58,7 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"layers = [Layer(5, 5, ReLU()), Layer(5, 25, ReLU()), Layer(25, 1, Linear())]\n", "layers = [Layer(5, 5, ReLU()), Layer(5, 25, ReLU()), Layer(25, 1, Linear())]\n",
"network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), MSELoss())\n", "network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), MSELoss())\n",
"network.fit(X_train, y_train)\n", "network.fit(X_train, y_train)\n",
"predictions = network.predict(X_test)" "predictions = network.predict(X_test)"
@@ -119,7 +128,7 @@
], ],
"source": [ "source": [
"plt.scatter(y_test, predictions)\n", "plt.scatter(y_test, predictions)\n",
"plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'k--', lw=2)\n", "plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], \"k--\", lw=2)\n",
"plt.xlabel(\"True Values\")\n", "plt.xlabel(\"True Values\")\n",
"plt.ylabel(\"Predictions\")\n", "plt.ylabel(\"Predictions\")\n",
"plt.title(\"True vs Predicted Values\")" "plt.title(\"True vs Predicted Values\")"
@@ -165,7 +174,7 @@
"X = StandardScaler().fit_transform(X)\n", "X = StandardScaler().fit_transform(X)\n",
"y = y.reshape(-1, 1)\n", "y = y.reshape(-1, 1)\n",
"y = OneHotEncoder(sparse_output=False).fit_transform(y)\n", "y = OneHotEncoder(sparse_output=False).fit_transform(y)\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y)\n" "X_train, X_test, y_train, y_test = train_test_split(X, y)"
] ]
}, },
{ {
@@ -175,11 +184,15 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"layers = [Layer(5, 10, ReLU()), Layer(10, 5, ReLU()), Layer(5, 2, Softmax())]\n", "layers = [Layer(5, 10, ReLU()), Layer(10, 5, ReLU()), Layer(5, 2, Softmax())]\n",
"network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), CrossEntropyLoss())\n", "network = FFNN(\n",
" layers, AdamScheduler(learning_rate=0.001, epochs=1000), CrossEntropyLoss()\n",
")\n",
"network.fit(X_train, y_train)\n", "network.fit(X_train, y_train)\n",
"predictions = network.predict(X_test)\n", "predictions = network.predict(X_test)\n",
"single_class_predictions = OneHotEncoder(sparse_output=False).fit_transform(np.argmax(predictions, axis=1).reshape(-1, 1))\n" "single_class_predictions = OneHotEncoder(sparse_output=False).fit_transform(\n",
" np.argmax(predictions, axis=1).reshape(-1, 1)\n",
")"
] ]
}, },
{ {
notebooks/10_regression-analysis.ipynb
+103 -39
View File
@@ -17,11 +17,18 @@
"data = load_breast_cancer()\n", "data = load_breast_cancer()\n",
"\n", "\n",
"\n", "\n",
"feature_names = [n for n in data.feature_names if 'radius' in n or 'area' in n]\n", "feature_names = [n for n in data.feature_names if \"radius\" in n or \"area\" in n]\n",
"\n", "\n",
"X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n", "X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n",
"y = data.data[:, [data.feature_names.tolist().index(n) for n in data.feature_names if n not in feature_names]]\n", "y = data.data[\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n" " :,\n",
" [\n",
" data.feature_names.tolist().index(n)\n",
" for n in data.feature_names\n",
" if n not in feature_names\n",
" ],\n",
"]\n",
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
] ]
}, },
{ {
@@ -31,7 +38,15 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"from easynn.feedforward import Layer, FFNN, ReLU, LeakyReLU, Linear, MSELoss, Regularization\n", "from easynn.feedforward import (\n",
" Layer,\n",
" FFNN,\n",
" ReLU,\n",
" LeakyReLU,\n",
" Linear,\n",
" MSELoss,\n",
" Regularization,\n",
")\n",
"from easynn.schedulers import AdamScheduler" "from easynn.schedulers import AdamScheduler"
] ]
}, },
@@ -44,14 +59,17 @@
"source": [ "source": [
"feature_dim = X.shape[1]\n", "feature_dim = X.shape[1]\n",
"target_dim = y.shape[1]\n", "target_dim = y.shape[1]\n",
"def get_regression_model(n_hidden_layers: int, n_neurons: int, activation: type=ReLU) -> list[Layer]:\n", "\n",
"\n",
"def get_regression_model(\n",
" n_hidden_layers: int, n_neurons: int, activation: type = ReLU\n",
") -> list[Layer]:\n",
" layers = []\n", " layers = []\n",
" layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n", " layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n",
" for _ in range(n_hidden_layers - 1):\n", " for _ in range(n_hidden_layers - 1):\n",
" layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n", " layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n",
" layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n", " layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n",
" return layers\n", " return layers"
"\n"
] ]
}, },
{ {
@@ -63,7 +81,11 @@
"source": [ "source": [
"# First network test\n", "# First network test\n",
"\n", "\n",
"model = FFNN(get_regression_model(n_hidden_layers=3, n_neurons=32, activation=LeakyReLU), AdamScheduler(epochs=10000, learning_rate=1e-2), MSELoss())\n", "model = FFNN(\n",
" get_regression_model(n_hidden_layers=3, n_neurons=32, activation=LeakyReLU),\n",
" AdamScheduler(epochs=10000, learning_rate=1e-2),\n",
" MSELoss(),\n",
")\n",
"model.fit(X_train, y_train)" "model.fit(X_train, y_train)"
] ]
}, },
@@ -100,10 +122,14 @@
"import matplotlib.pyplot as plt\n", "import matplotlib.pyplot as plt\n",
"\n", "\n",
"y_pred = model.predict(X_test)\n", "y_pred = model.predict(X_test)\n",
"mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n", "mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n",
"\n", "\n",
"sns.barplot(x=np.arange(len(mse)), y=mse)\n", "sns.barplot(x=np.arange(len(mse)), y=mse)\n",
"plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n", "plt.xticks(\n",
" ticks=np.arange(len(mse)),\n",
" labels=[n for n in data.feature_names if n not in feature_names],\n",
" rotation=90,\n",
")\n",
"plt.yscale(\"log\")\n", "plt.yscale(\"log\")\n",
"plt.ylabel(\"Mean Squared Error\")" "plt.ylabel(\"Mean Squared Error\")"
] ]
@@ -131,15 +157,23 @@
"n_neurons_list = [4, 8, 16, 32, 64, 128]\n", "n_neurons_list = [4, 8, 16, 32, 64, 128]\n",
"mse_results = np.zeros((len(n_layers_list), len(n_neurons_list)))\n", "mse_results = np.zeros((len(n_layers_list), len(n_neurons_list)))\n",
"for idx, (n_layers, n_neurons) in enumerate(\n", "for idx, (n_layers, n_neurons) in enumerate(\n",
" tqdm(product(n_layers_list, n_neurons_list),\n", " tqdm(\n",
" total=len(n_layers_list) * len(n_neurons_list))\n", " product(n_layers_list, n_neurons_list),\n",
" total=len(n_layers_list) * len(n_neurons_list),\n",
" )\n",
"):\n", "):\n",
" i, j = divmod(idx, len(n_neurons_list))\n", " i, j = divmod(idx, len(n_neurons_list))\n",
" model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=15000, learning_rate=1e-3), MSELoss())\n", " model = FFNN(\n",
" get_regression_model(\n",
" n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
" ),\n",
" AdamScheduler(epochs=15000, learning_rate=1e-3),\n",
" MSELoss(),\n",
" )\n",
" model.fit(X_train, y_train)\n", " model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n", " y_pred = model.predict(X_test)\n",
" mse = mean_squared_error(y_test, y_pred)\n", " mse = mean_squared_error(y_test, y_pred)\n",
" mse_results[i, j] = mse\n" " mse_results[i, j] = mse"
] ]
}, },
{ {
@@ -160,10 +194,16 @@
} }
], ],
"source": [ "source": [
"sns.heatmap(mse_results, xticklabels=n_neurons_list, yticklabels=n_layers_list, annot=True, fmt=\".2f\")\n", "sns.heatmap(\n",
" mse_results,\n",
" xticklabels=n_neurons_list,\n",
" yticklabels=n_layers_list,\n",
" annot=True,\n",
" fmt=\".2f\",\n",
")\n",
"plt.xlabel(\"Number of Neurons\")\n", "plt.xlabel(\"Number of Neurons\")\n",
"plt.ylabel(\"Number of Layers\")\n", "plt.ylabel(\"Number of Layers\")\n",
"plt.savefig(\"nodenumber_tuning_regression.pdf\")\n" "plt.savefig(\"nodenumber_tuning_regression.pdf\")"
] ]
}, },
{ {
@@ -177,10 +217,16 @@
"parameters = [(1, 32), (1, 64), (2, 32), (2, 64), (3, 64), (3, 128)]\n", "parameters = [(1, 32), (1, 64), (2, 32), (2, 64), (3, 64), (3, 128)]\n",
"mse_results_slow_scan = np.zeros((len(parameters), target_dim))\n", "mse_results_slow_scan = np.zeros((len(parameters), target_dim))\n",
"for k, (n_layers, n_neurons) in enumerate(parameters):\n", "for k, (n_layers, n_neurons) in enumerate(parameters):\n",
" model = FFNN(get_regression_model(n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU), AdamScheduler(epochs=20000, learning_rate=5e-4), MSELoss())\n", " model = FFNN(\n",
" get_regression_model(\n",
" n_hidden_layers=n_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
" ),\n",
" AdamScheduler(epochs=20000, learning_rate=5e-4),\n",
" MSELoss(),\n",
" )\n",
" model.fit(X_train, y_train)\n", " model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n", " y_pred = model.predict(X_test)\n",
" mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n", " mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n",
" mse_results_slow_scan[k, :] = mse" " mse_results_slow_scan[k, :] = mse"
] ]
}, },
@@ -202,12 +248,16 @@
} }
], ],
"source": [ "source": [
"sns.heatmap(np.log10(mse_results_slow_scan), xticklabels=[n for n in data.feature_names if n not in feature_names], yticklabels=[f\"L{l}_N{n}\" for l, n in parameters],)\n", "sns.heatmap(\n",
"plt.xticks(rotation=45, ha='right')\n", " np.log10(mse_results_slow_scan),\n",
" xticklabels=[n for n in data.feature_names if n not in feature_names],\n",
" yticklabels=[f\"L{l}_N{n}\" for l, n in parameters],\n",
")\n",
"plt.xticks(rotation=45, ha=\"right\")\n",
"plt.xlabel(\"Target Features\")\n", "plt.xlabel(\"Target Features\")\n",
"plt.ylabel(\"Model (Layers_Neurons)\")\n", "plt.ylabel(\"Model (Layers_Neurons)\")\n",
"plt.tight_layout()\n", "plt.tight_layout()\n",
"plt.savefig(\"regression_detailed_hyperparameter_scan.pdf\")\n" "plt.savefig(\"regression_detailed_hyperparameter_scan.pdf\")"
] ]
}, },
{ {
@@ -241,7 +291,7 @@
" x=log_mse.ravel(),\n", " x=log_mse.ravel(),\n",
" y=np.repeat(np.arange(len(parameters)), log_mse.shape[1]),\n", " y=np.repeat(np.arange(len(parameters)), log_mse.shape[1]),\n",
" bins=[np.linspace(-5, 2, 15), len(parameters)],\n", " bins=[np.linspace(-5, 2, 15), len(parameters)],\n",
" cmap=cmap\n", " cmap=cmap,\n",
")\n", ")\n",
"\n", "\n",
"# Label axes\n", "# Label axes\n",
@@ -266,7 +316,7 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"n_hidden_layers = 2\n", "n_hidden_layers = 2\n",
"n_neurons = 32\n" "n_neurons = 32"
] ]
}, },
{ {
@@ -276,14 +326,15 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"\n",
"# regularization test\n", "# regularization test\n",
"layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n", "layers = get_regression_model(\n",
" n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
")\n",
"regularization_strengths = [0.0, 1e-5, 1e-4, 1e-3, 1e-2]\n", "regularization_strengths = [0.0, 1e-5, 1e-4, 1e-3, 1e-2]\n",
"mse_results_reg = np.zeros(len(regularization_strengths))\n", "mse_results_reg = np.zeros(len(regularization_strengths))\n",
"for i, reg_strength in enumerate(regularization_strengths):\n", "for i, reg_strength in enumerate(regularization_strengths):\n",
" for layer in layers:\n", " for layer in layers:\n",
" layer.regularization = Regularization(reg_strength, 'l2')\n", " layer.regularization = Regularization(reg_strength, \"l2\")\n",
" model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n", " model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n",
" model.fit(X_train, y_train)\n", " model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n", " y_pred = model.predict(X_test)\n",
@@ -335,9 +386,11 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"# Check final model training\n", "# Check final model training\n",
"layers = get_regression_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU)\n", "layers = get_regression_model(\n",
" n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, activation=LeakyReLU\n",
")\n",
"for layer in layers:\n", "for layer in layers:\n",
" layer.regularization = Regularization(reg_strength, 'l2')\n", " layer.regularization = Regularization(reg_strength, \"l2\")\n",
"model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n", "model = FFNN(layers, AdamScheduler(epochs=20000, learning_rate=1e-4), MSELoss())\n",
"model.fit(X_train, y_train)" "model.fit(X_train, y_train)"
] ]
@@ -361,7 +414,7 @@
], ],
"source": [ "source": [
"sns.lineplot(x=np.arange(20000), y=model.scheduler.loss_history)\n", "sns.lineplot(x=np.arange(20000), y=model.scheduler.loss_history)\n",
"plt.yscale(\"log\")\n" "plt.yscale(\"log\")"
] ]
}, },
{ {
@@ -383,10 +436,14 @@
], ],
"source": [ "source": [
"y_pred = model.predict(X_test)\n", "y_pred = model.predict(X_test)\n",
"mse = mean_squared_error(y_test, y_pred, multioutput='raw_values')\n", "mse = mean_squared_error(y_test, y_pred, multioutput=\"raw_values\")\n",
"\n", "\n",
"sns.barplot(x=np.arange(len(mse)), y=mse)\n", "sns.barplot(x=np.arange(len(mse)), y=mse)\n",
"plt.xticks(ticks=np.arange(len(mse)), labels=[n for n in data.feature_names if n not in feature_names], rotation=90)\n", "plt.xticks(\n",
" ticks=np.arange(len(mse)),\n",
" labels=[n for n in data.feature_names if n not in feature_names],\n",
" rotation=90,\n",
")\n",
"plt.yscale(\"log\")\n", "plt.yscale(\"log\")\n",
"plt.ylabel(\"Mean Squared Error\")\n", "plt.ylabel(\"Mean Squared Error\")\n",
"plt.tight_layout()\n", "plt.tight_layout()\n",
@@ -425,16 +482,23 @@
"source": [ "source": [
"beta = optimizers.OLS_parameters(X_train, y_train)\n", "beta = optimizers.OLS_parameters(X_train, y_train)\n",
"y_ols_pred = X_test @ beta\n", "y_ols_pred = X_test @ beta\n",
"mse_ols = mean_squared_error(y_test, y_ols_pred, multioutput='raw_values')\n", "mse_ols = mean_squared_error(y_test, y_ols_pred, multioutput=\"raw_values\")\n",
"\n", "\n",
"mse_df = pd.DataFrame({\n", "mse_df = pd.DataFrame(\n",
" \"Feature\": [n for n in data.feature_names if n not in feature_names],\n", " {\n",
" \"Neural Network\": mse,\n", " \"Feature\": [n for n in data.feature_names if n not in feature_names],\n",
" \"OLS\": mse_ols\n", " \"Neural Network\": mse,\n",
"})\n", " \"OLS\": mse_ols,\n",
" }\n",
")\n",
"\n", "\n",
"sns.barplot(x=\"Feature\", y=\"value\", hue=\"Model\", data=pd.melt(mse_df, id_vars=[\"Feature\"]).rename(columns={\"variable\": \"Model\"}))\n", "sns.barplot(\n",
"plt.xticks(rotation=45, ha='right')\n", " x=\"Feature\",\n",
" y=\"value\",\n",
" hue=\"Model\",\n",
" data=pd.melt(mse_df, id_vars=[\"Feature\"]).rename(columns={\"variable\": \"Model\"}),\n",
")\n",
"plt.xticks(rotation=45, ha=\"right\")\n",
"plt.yscale(\"log\")\n", "plt.yscale(\"log\")\n",
"plt.ylabel(\"Mean Squared Error\")\n", "plt.ylabel(\"Mean Squared Error\")\n",
"plt.xlabel(\"Target Feature\")\n", "plt.xlabel(\"Target Feature\")\n",
notebooks/11_regression-usecase.ipynb
+45 -13
View File
@@ -24,27 +24,50 @@
"data = load_breast_cancer()\n", "data = load_breast_cancer()\n",
"\n", "\n",
"\n", "\n",
"feature_names = [n for n in data.feature_names if 'radius' in n or 'area' in n]\n", "feature_names = [n for n in data.feature_names if \"radius\" in n or \"area\" in n]\n",
"\n", "\n",
"X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n", "X = data.data[:, [data.feature_names.tolist().index(n) for n in feature_names]]\n",
"X = StandardScaler().fit_transform(X)\n", "X = StandardScaler().fit_transform(X)\n",
"y = data.data[:, [data.feature_names.tolist().index(n) for n in data.feature_names if n not in feature_names]]\n", "y = data.data[\n",
" :,\n",
" [\n",
" data.feature_names.tolist().index(n)\n",
" for n in data.feature_names\n",
" if n not in feature_names\n",
" ],\n",
"]\n",
"y = StandardScaler().fit_transform(y)\n", "y = StandardScaler().fit_transform(y)\n",
"feature_dim = X.shape[1]\n", "feature_dim = X.shape[1]\n",
"target_dim = y.shape[1]\n", "target_dim = y.shape[1]\n",
"\n", "\n",
"def get_regression_model(n_hidden_layers: int, n_neurons: int, activation: type=LeakyReLU, regularization_strength: float = 1e-3) -> list[Layer]:\n", "\n",
"def get_regression_model(\n",
" n_hidden_layers: int,\n",
" n_neurons: int,\n",
" activation: type = LeakyReLU,\n",
" regularization_strength: float = 1e-3,\n",
") -> list[Layer]:\n",
" layers = []\n", " layers = []\n",
" layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n", " layers.append(Layer(feature_dim, n_neurons, activation_function=activation()))\n",
" for _ in range(n_hidden_layers - 1):\n", " for _ in range(n_hidden_layers - 1):\n",
" layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n", " layers.append(Layer(n_neurons, n_neurons, activation_function=activation()))\n",
" layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n", " layers.append(Layer(n_neurons, target_dim, activation_function=Linear()))\n",
" for layer in layers:\n", " for layer in layers:\n",
" layer.regularization = Regularization(regularization_strength, 'l2')\n", " layer.regularization = Regularization(regularization_strength, \"l2\")\n",
" return layers\n", " return layers\n",
"\n", "\n",
"\n",
"def get_model():\n", "def get_model():\n",
" return FFNN(get_regression_model(n_hidden_layers=2, n_neurons=32, activation=LeakyReLU, regularization_strength=1e-3), AdamScheduler(epochs=10000, learning_rate=1e-2), MSELoss())" " return FFNN(\n",
" get_regression_model(\n",
" n_hidden_layers=2,\n",
" n_neurons=32,\n",
" activation=LeakyReLU,\n",
" regularization_strength=1e-3,\n",
" ),\n",
" AdamScheduler(epochs=10000, learning_rate=1e-2),\n",
" MSELoss(),\n",
" )"
] ]
}, },
{ {
@@ -57,8 +80,12 @@
"crossvalidation_groups = 5\n", "crossvalidation_groups = 5\n",
"group_length = X.shape[0] // crossvalidation_groups\n", "group_length = X.shape[0] // crossvalidation_groups\n",
"predictions = np.zeros_like(y)\n", "predictions = np.zeros_like(y)\n",
"for g in range(crossvalidation_groups + 1): # One final smaller group to catch the rest\n", "for g in range(crossvalidation_groups + 1): # One final smaller group to catch the rest\n",
" indices = [i for i in range(X.shape[0]) if i < g*group_length or i > (g+1)*group_length]\n", " indices = [\n",
" i\n",
" for i in range(X.shape[0])\n",
" if i < g * group_length or i > (g + 1) * group_length\n",
" ]\n",
" prediction_indices = [i for i in range(X.shape[0]) if i not in indices]\n", " prediction_indices = [i for i in range(X.shape[0]) if i not in indices]\n",
" X_train = X[indices]\n", " X_train = X[indices]\n",
" X_pred = X[prediction_indices]\n", " X_pred = X[prediction_indices]\n",
@@ -66,8 +93,7 @@
"\n", "\n",
" model = get_model()\n", " model = get_model()\n",
" model.fit(X_train, y_train)\n", " model.fit(X_train, y_train)\n",
" predictions[prediction_indices] = model.predict(X_pred)\n", " predictions[prediction_indices] = model.predict(X_pred)"
"\n"
] ]
}, },
{ {
@@ -122,15 +148,21 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"combined_data = np.zeros_like(data.data)\n", "combined_data = np.zeros_like(data.data)\n",
"combined_data[:,:6] = X\n", "combined_data[:, :6] = X\n",
"combined_data[:, 6:] = predictions\n", "combined_data[:, 6:] = predictions\n",
"\n", "\n",
"combined_df = pd.DataFrame(combined_data, columns=[*feature_names, *[n for n in data.feature_names if n not in feature_names]])\n", "combined_df = pd.DataFrame(\n",
"combined_df['target'] = data.target\n", " combined_data,\n",
" columns=[\n",
" *feature_names,\n",
" *[n for n in data.feature_names if n not in feature_names],\n",
" ],\n",
")\n",
"combined_df[\"target\"] = data.target\n",
"combined_df.to_csv(\"breast_cancer_regression_results.csv\", index=False)\n", "combined_df.to_csv(\"breast_cancer_regression_results.csv\", index=False)\n",
"\n", "\n",
"original_df = pd.DataFrame(data.data, columns=data.feature_names)\n", "original_df = pd.DataFrame(data.data, columns=data.feature_names)\n",
"original_df['target'] = data.target\n", "original_df[\"target\"] = data.target\n",
"original_df.to_csv(\"breast_cancer_original_data.csv\", index=False)" "original_df.to_csv(\"breast_cancer_original_data.csv\", index=False)"
] ]
}, },
notebooks/20_classification-analysis.ipynb
+58 -28
View File
@@ -7,8 +7,21 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"from easynn.feedforward import FFNN, Layer, Regularization, LeakyReLU, Softmax, CrossEntropyLoss\n", "from easynn.feedforward import (\n",
"from easynn.schedulers import GradientDescentScheduler, AdamScheduler, RMSPropScheduler, AdaGradScheduler, MomentumScheduler\n" " FFNN,\n",
" Layer,\n",
" Regularization,\n",
" LeakyReLU,\n",
" Softmax,\n",
" CrossEntropyLoss,\n",
")\n",
"from easynn.schedulers import (\n",
" GradientDescentScheduler,\n",
" AdamScheduler,\n",
" RMSPropScheduler,\n",
" AdaGradScheduler,\n",
" MomentumScheduler,\n",
")"
] ]
}, },
{ {
@@ -18,7 +31,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def get_classification_model(n_hidden_layers: int, n_neurons: int, activation: type=LeakyReLU, reg_strength: float=0.0) -> list[Layer]:\n", "def get_classification_model(\n",
" n_hidden_layers: int,\n",
" n_neurons: int,\n",
" activation: type = LeakyReLU,\n",
" reg_strength: float = 0.0,\n",
") -> list[Layer]:\n",
" target_dim = 2\n", " target_dim = 2\n",
" feature_dim = 30\n", " feature_dim = 30\n",
" layers = []\n", " layers = []\n",
@@ -2337,21 +2355,32 @@
"source": [ "source": [
"from sklearn.metrics import accuracy_score, roc_auc_score\n", "from sklearn.metrics import accuracy_score, roc_auc_score\n",
"\n", "\n",
"def test_classification_model(n_hidden_layers: int, n_neurons: int, reg_strength: float, n_iterations: int=50) -> tuple[float, float, FFNN]:\n", "\n",
"def test_classification_model(\n",
" n_hidden_layers: int, n_neurons: int, reg_strength: float, n_iterations: int = 50\n",
") -> tuple[float, float, FFNN]:\n",
" assert n_iterations > 0, \"n_iterations must be greater than 0\"\n", " assert n_iterations > 0, \"n_iterations must be greater than 0\"\n",
" accs = []\n", " accs = []\n",
" aucs = []\n", " aucs = []\n",
" for _ in range(n_iterations): # Run n_iterations trials\n", " for _ in range(n_iterations): # Run n_iterations trials\n",
" X_train, X_test, y_train, y_test = train_test_split(X, y_encoded, test_size=0.2)\n", " X_train, X_test, y_train, y_test = train_test_split(X, y_encoded, test_size=0.2)\n",
" layers = get_classification_model(n_hidden_layers=n_hidden_layers, n_neurons=n_neurons, reg_strength=reg_strength)\n", " layers = get_classification_model(\n",
" model = FFNN(layers, AdamScheduler(epochs=500, learning_rate=0.05), loss_fn=CrossEntropyLoss())\n", " n_hidden_layers=n_hidden_layers,\n",
" n_neurons=n_neurons,\n",
" reg_strength=reg_strength,\n",
" )\n",
" model = FFNN(\n",
" layers,\n",
" AdamScheduler(epochs=500, learning_rate=0.05),\n",
" loss_fn=CrossEntropyLoss(),\n",
" )\n",
" model.fit(X_train, y_train)\n", " model.fit(X_train, y_train)\n",
" y_pred = model.predict(X_test)\n", " y_pred = model.predict(X_test)\n",
" acc = accuracy_score(y_test.argmax(axis=1), y_pred.argmax(axis=1))\n", " acc = accuracy_score(y_test.argmax(axis=1), y_pred.argmax(axis=1))\n",
" auc = roc_auc_score(y_test, y_pred)\n", " auc = roc_auc_score(y_test, y_pred)\n",
" accs.append(acc)\n", " accs.append(acc)\n",
" aucs.append(auc)\n", " aucs.append(auc)\n",
" return np.mean(accs), np.mean(aucs), model\n" " return np.mean(accs), np.mean(aucs), model"
] ]
}, },
{ {
@@ -2389,14 +2418,18 @@
"):\n", "):\n",
" ...\n", " ...\n",
"\n", "\n",
" acc, auc, model = test_classification_model(n_hidden_layers, n_neurons, reg_strength, 2)\n", " acc, auc, model = test_classification_model(\n",
" results.append({\n", " n_hidden_layers, n_neurons, reg_strength, 2\n",
" \"n_hidden_layers\": n_hidden_layers,\n", " )\n",
" \"n_neurons\": n_neurons,\n", " results.append(\n",
" \"reg_strength\": reg_strength,\n", " {\n",
" \"accuracy\": acc,\n", " \"n_hidden_layers\": n_hidden_layers,\n",
" \"auc\": auc,\n", " \"n_neurons\": n_neurons,\n",
" })\n", " \"reg_strength\": reg_strength,\n",
" \"accuracy\": acc,\n",
" \"auc\": auc,\n",
" }\n",
" )\n",
"\n", "\n",
" # You can use AUC as the main criterion, or combine them\n", " # You can use AUC as the main criterion, or combine them\n",
" score = auc + 0.1 * acc\n", " score = auc + 0.1 * acc\n",
@@ -2449,21 +2482,16 @@
" if i == j:\n", " if i == j:\n",
" # Diagonal: 1D line plot of AUC vs parameter\n", " # Diagonal: 1D line plot of AUC vs parameter\n",
" sns.lineplot(\n", " sns.lineplot(\n",
" data=results_df,\n", " data=results_df, x=xparam, y=\"auc\", marker=\"o\", ax=ax, color=\"C0\"\n",
" x=xparam,\n",
" y=\"auc\",\n",
" marker=\"o\",\n",
" ax=ax,\n",
" color=\"C0\"\n",
" )\n", " )\n",
" ax.set_xlabel(xparam)\n", " ax.set_xlabel(xparam)\n",
" ax.set_ylabel(\"AUC\")\n", " ax.set_ylabel(\"AUC\")\n",
" if i == 2:\n", " if i == 2:\n",
" ax.set_xscale('log')\n", " ax.set_xscale(\"log\")\n",
" else:\n", " else:\n",
" # Off-diagonal: 2D heatmap colored by AUC\n", " # Off-diagonal: 2D heatmap colored by AUC\n",
" hm = results_df.pivot_table(index=yparam, columns=xparam, values=\"auc\")\n", " hm = results_df.pivot_table(index=yparam, columns=xparam, values=\"auc\")\n",
" sc = ax.imshow(hm, origin='lower', aspect='auto', cmap=cmap)\n", " sc = ax.imshow(hm, origin=\"lower\", aspect=\"auto\", cmap=cmap)\n",
" ax.set_xticks(np.arange(len(hm.columns)))\n", " ax.set_xticks(np.arange(len(hm.columns)))\n",
" ax.set_yticks(np.arange(len(hm.index)))\n", " ax.set_yticks(np.arange(len(hm.index)))\n",
" ax.set_xticklabels(hm.columns)\n", " ax.set_xticklabels(hm.columns)\n",
@@ -2478,7 +2506,7 @@
"fig.suptitle(\"Pairwise Hyperparameter Relationships (AUC)\", fontsize=18)\n", "fig.suptitle(\"Pairwise Hyperparameter Relationships (AUC)\", fontsize=18)\n",
"plt.tight_layout(rect=[0, 0, 0.9, 0.97])\n", "plt.tight_layout(rect=[0, 0, 0.9, 0.97])\n",
"plt.savefig(\"classification_hyperparameter_scan.pdf\")\n", "plt.savefig(\"classification_hyperparameter_scan.pdf\")\n",
"plt.show()\n" "plt.show()"
] ]
}, },
{ {
@@ -2500,8 +2528,10 @@
"n_neurons = 64\n", "n_neurons = 64\n",
"reg_strength = 1e-6\n", "reg_strength = 1e-6\n",
"\n", "\n",
"acc, auc, best_model = test_classification_model(n_hidden_layers, n_neurons, reg_strength)\n", "acc, auc, best_model = test_classification_model(\n",
"print(f\"Best Model - Accuracy: {acc:.4f}, AUC: {auc:.4f}\")\n" " n_hidden_layers, n_neurons, reg_strength\n",
")\n",
"print(f\"Best Model - Accuracy: {acc:.4f}, AUC: {auc:.4f}\")"
] ]
}, },
{ {
@@ -2545,7 +2575,7 @@
} }
], ],
"source": [ "source": [
"sns.heatmap(cm, annot=True, fmt='d')\n", "sns.heatmap(cm, annot=True, fmt=\"d\")\n",
"plt.xlabel(\"Predicted Label\")\n", "plt.xlabel(\"Predicted Label\")\n",
"plt.ylabel(\"True Label\")\n", "plt.ylabel(\"True Label\")\n",
"plt.tight_layout()\n", "plt.tight_layout()\n",
@@ -2572,7 +2602,7 @@
], ],
"source": [ "source": [
"sns.lineplot(x=fpr, y=tpr)\n", "sns.lineplot(x=fpr, y=tpr)\n",
"plt.plot([0, 1], [0, 1], linestyle='--', color='gray')\n", "plt.plot([0, 1], [0, 1], linestyle=\"--\", color=\"gray\")\n",
"plt.xlabel(\"False Positive Rate\")\n", "plt.xlabel(\"False Positive Rate\")\n",
"plt.ylabel(\"True Positive Rate\")\n", "plt.ylabel(\"True Positive Rate\")\n",
"plt.tight_layout()\n", "plt.tight_layout()\n",
notebooks/21_logisitic-regression.ipynb
+15 -5
View File
@@ -7,7 +7,14 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"from easynn.feedforward import FFNN, Layer, Regularization, LeakyReLU, Softmax, CrossEntropyLoss\n", "from easynn.feedforward import (\n",
" FFNN,\n",
" Layer,\n",
" Regularization,\n",
" LeakyReLU,\n",
" Softmax,\n",
" CrossEntropyLoss,\n",
")\n",
"from easynn.schedulers import AdamScheduler\n", "from easynn.schedulers import AdamScheduler\n",
"\n", "\n",
"import pandas as pd\n", "import pandas as pd\n",
@@ -38,9 +45,12 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"logistic_model = FFNN([\n", "logistic_model = FFNN(\n",
" Layer(30, 2, activation_function=Softmax()),\n", " [\n",
"], loss_fn=CrossEntropyLoss(), scheduler=AdamScheduler(learning_rate=5e-4, epochs=25000)\n", " Layer(30, 2, activation_function=Softmax()),\n",
" ],\n",
" loss_fn=CrossEntropyLoss(),\n",
" scheduler=AdamScheduler(learning_rate=5e-4, epochs=25000),\n",
")" ")"
] ]
}, },
@@ -155,7 +165,7 @@
"\n", "\n",
"cm = confusion_matrix(y_test.argmax(axis=1), y_pred.argmax(axis=1))\n", "cm = confusion_matrix(y_test.argmax(axis=1), y_pred.argmax(axis=1))\n",
"\n", "\n",
"sns.heatmap(cm, annot=True, fmt='d')\n", "sns.heatmap(cm, annot=True, fmt=\"d\")\n",
"plt.xlabel(\"Predicted Label\")\n", "plt.xlabel(\"Predicted Label\")\n",
"plt.ylabel(\"True Label\")\n", "plt.ylabel(\"True Label\")\n",
"plt.tight_layout()\n", "plt.tight_layout()\n",
notebooks/plotting.py
+3 -1
View File
@@ -1,5 +1,6 @@
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
def get_rc_params(): def get_rc_params():
colors = ["FF220C", "70D6FF", "8AAA79", "666370", "1C1F33"] colors = ["FF220C", "70D6FF", "8AAA79", "666370", "1C1F33"]
rcParams = plt.rcParams rcParams = plt.rcParams
@@ -29,4 +30,5 @@ def get_rc_params():
rcParams["ytick.right"] = True rcParams["ytick.right"] = True
return rcParams return rcParams
plt.rcParams.update(get_rc_params())
plt.rcParams.update(get_rc_params())
pyproject.toml
+1
View File
@@ -14,6 +14,7 @@ dependencies = [
"pytest>=8.4.2", "pytest>=8.4.2",
"ruff>=0.14.1", "ruff>=0.14.1",
"scikit-learn>=1.7.2", "scikit-learn>=1.7.2",
"scikit-learn-stubs>=0.0.3",
"seaborn>=0.13.2", "seaborn>=0.13.2",
"tqdm>=4.67.1", "tqdm>=4.67.1",
] ]
src/easynn/feedforward.py
+41 -24
View File
@@ -37,13 +37,15 @@ class LeakyReLU(ActivationFunction):
def backward(self, values: np.ndarray) -> np.ndarray: def backward(self, values: np.ndarray) -> np.ndarray:
return np.where(values >= 0, 1.0, self.leak) return np.where(values >= 0, 1.0, self.leak)
class Linear(ActivationFunction): class Linear(ActivationFunction):
def forward(self, values: np.ndarray) -> np.ndarray: def forward(self, values: np.ndarray) -> np.ndarray:
return values return values
def backward(self, values: np.ndarray) -> np.ndarray: def backward(self, values: np.ndarray) -> np.ndarray:
return np.ones_like(values) return np.ones_like(values)
class Softmax(ActivationFunction): class Softmax(ActivationFunction):
def forward(self, values: np.ndarray) -> np.ndarray: def forward(self, values: np.ndarray) -> np.ndarray:
exp_values = np.exp(values - np.max(values, axis=1, keepdims=True)) exp_values = np.exp(values - np.max(values, axis=1, keepdims=True))
@@ -52,7 +54,8 @@ class Softmax(ActivationFunction):
def backward(self, values: np.ndarray) -> np.ndarray: def backward(self, values: np.ndarray) -> np.ndarray:
s = self.forward(values) s = self.forward(values)
return s * (1 - s) return s * (1 - s)
# === Loss Functions === # === Loss Functions ===
@@ -76,7 +79,7 @@ class CrossEntropyLoss(LossFunction):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
self._override_activation_loss: bool = False # Used if activation and loss are combined for efficiency; Set by FFNN if needed; self._override_activation_loss: bool = False # Used if activation and loss are combined for efficiency; Set by FFNN if needed;
def set_override_activation_loss(self, override: bool = True) -> None: def set_override_activation_loss(self, override: bool = True) -> None:
self._override_activation_loss = override self._override_activation_loss = override
@@ -84,32 +87,34 @@ class CrossEntropyLoss(LossFunction):
eps = 1e-9 eps = 1e-9
y_pred = np.clip(y_pred, eps, 1 - eps) y_pred = np.clip(y_pred, eps, 1 - eps)
return -float(np.mean(np.sum(y_true * np.log(y_pred), axis=1))) return -float(np.mean(np.sum(y_true * np.log(y_pred), axis=1)))
def backward(self, y_pred: np.ndarray, y_true: np.ndarray) -> np.ndarray: def backward(self, y_pred: np.ndarray, y_true: np.ndarray) -> np.ndarray:
if not self._override_activation_loss: if not self._override_activation_loss:
eps = 1e-9 eps = 1e-9
y_pred = np.clip(y_pred, eps, 1 - eps) y_pred = np.clip(y_pred, eps, 1 - eps)
return - (y_true / y_pred) / y_true.shape[0] return -(y_true / y_pred) / y_true.shape[0]
else: else:
return (y_pred - y_true) / y_true.shape[0] return (y_pred - y_true) / y_true.shape[0]
# === Regularization Functions === # === Regularization Functions ===
class Regularization: class Regularization:
def __init__(self, reg_lambda: float, mode: Literal['l1', 'l2'] = 'l2') -> None: def __init__(self, reg_lambda: float, mode: Literal["l1", "l2"] = "l2") -> None:
self.reg_lambda = reg_lambda self.reg_lambda = reg_lambda
if mode not in ['l1', 'l2']: if mode not in ["l1", "l2"]:
raise ValueError("mode must be 'l1' or 'l2'") raise ValueError("mode must be 'l1' or 'l2'")
self.mode = mode self.mode = mode
def compute_penalty(self, weights: np.ndarray, biases: np.ndarray) -> float: def compute_penalty(self, weights: np.ndarray, biases: np.ndarray) -> float:
if self.mode == 'l2': if self.mode == "l2":
return self.reg_lambda * (np.sum(weights ** 2) + np.sum(biases ** 2)) return self.reg_lambda * (np.sum(weights**2) + np.sum(biases**2))
else: # l1 else: # l1
return self.reg_lambda * (np.sum(np.abs(weights)) + np.sum(np.abs(biases))) return self.reg_lambda * (np.sum(np.abs(weights)) + np.sum(np.abs(biases)))
def compute_gradient(self, weights: np.ndarray, biases: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: def compute_gradient(
if self.mode == 'l2': self, weights: np.ndarray, biases: np.ndarray
) -> Tuple[np.ndarray, np.ndarray]:
if self.mode == "l2":
return 2 * self.reg_lambda * weights, 2 * self.reg_lambda * biases return 2 * self.reg_lambda * weights, 2 * self.reg_lambda * biases
else: # l1 else: # l1
return self.reg_lambda * np.sign(weights), self.reg_lambda * np.sign(biases) return self.reg_lambda * np.sign(weights), self.reg_lambda * np.sign(biases)
@@ -120,21 +125,25 @@ class Regularization:
class Layer: class Layer:
def __init__( def __init__(
self, input_dim: int, num_nodes: int, activation_function: ActivationFunction, regularization: Optional[Regularization] = None self,
input_dim: int,
num_nodes: int,
activation_function: ActivationFunction,
regularization: Optional[Regularization] = None,
) -> None: ) -> None:
self.input_dim, self.num_nodes = input_dim, num_nodes self.input_dim, self.num_nodes = input_dim, num_nodes
# Initialize weights and biases identically to PyTorch's default initialization # Initialize weights and biases identically to PyTorch's default initialization
stdv = 1. / np.sqrt(input_dim * num_nodes) stdv = 1.0 / np.sqrt(input_dim * num_nodes)
self.weights = np.random.rand(input_dim, num_nodes) * 2 * stdv - stdv self.weights = np.random.rand(input_dim, num_nodes) * 2 * stdv - stdv
self.biases = np.random.rand(1, num_nodes) * 2 * stdv - stdv self.biases = np.random.rand(1, num_nodes) * 2 * stdv - stdv
self.activation_function = activation_function self.activation_function = activation_function
self.last_input: Optional[np.ndarray] = None self.last_input: Optional[np.ndarray] = None
self.last_z: Optional[np.ndarray] = None # Pre-activation values self.last_z: Optional[np.ndarray] = None # Pre-activation values
self.regularization = regularization self.regularization = regularization
self._override_activation_loss: bool = False # Used if activation and loss are combined for efficiency; Set by FFNN if needed; self._override_activation_loss: bool = False # Used if activation and loss are combined for efficiency; Set by FFNN if needed;
def set_override_activation_loss(self, override: bool = True) -> None: def set_override_activation_loss(self, override: bool = True) -> None:
self._override_activation_loss = override self._override_activation_loss = override
@@ -161,14 +170,16 @@ class Layer:
activation_grad = np.ones_like(self.last_z) activation_grad = np.ones_like(self.last_z)
else: else:
activation_grad = self.activation_function.backward(self.last_z) activation_grad = self.activation_function.backward(self.last_z)
local_grad = upstream_grad * activation_grad local_grad = upstream_grad * activation_grad
# Gradients for weights and biases # Gradients for weights and biases
bias_grad = np.sum(local_grad, axis=0, keepdims=True) / batch_size bias_grad = np.sum(local_grad, axis=0, keepdims=True) / batch_size
weights_grad = (self.last_input.T @ local_grad) / batch_size weights_grad = (self.last_input.T @ local_grad) / batch_size
if self.regularization is not None: if self.regularization is not None:
reg_weights_grad, reg_biases_grad = self.regularization.compute_gradient(self.weights, self.biases) reg_weights_grad, reg_biases_grad = self.regularization.compute_gradient(
self.weights, self.biases
)
weights_grad += reg_weights_grad weights_grad += reg_weights_grad
bias_grad += reg_biases_grad bias_grad += reg_biases_grad
@@ -176,7 +187,7 @@ class Layer:
previous_grad = local_grad @ self.weights.T previous_grad = local_grad @ self.weights.T
return previous_grad, (bias_grad, weights_grad) return previous_grad, (bias_grad, weights_grad)
def compute_regularization_penalty(self) -> float: def compute_regularization_penalty(self) -> float:
if self.regularization is not None: if self.regularization is not None:
return self.regularization.compute_penalty(self.weights, self.biases) return self.regularization.compute_penalty(self.weights, self.biases)
@@ -198,7 +209,6 @@ class FFNN:
self.loss_fn.set_override_activation_loss(True) self.loss_fn.set_override_activation_loss(True)
if isinstance(self.layers[-1].activation_function, Softmax): if isinstance(self.layers[-1].activation_function, Softmax):
self.layers[-1].set_override_activation_loss(True) self.layers[-1].set_override_activation_loss(True)
def predict(self, X: np.ndarray) -> np.ndarray: def predict(self, X: np.ndarray) -> np.ndarray:
if X.ndim == 1: if X.ndim == 1:
@@ -221,11 +231,13 @@ class FFNN:
# Compute loss and gradient # Compute loss and gradient
loss_value = self.loss_fn.forward(y_pred, y) loss_value = self.loss_fn.forward(y_pred, y)
logger.debug(f"Iteration {iteration + 1}, Pre-Regularization Loss: {loss_value}") logger.debug(
f"Iteration {iteration + 1}, Pre-Regularization Loss: {loss_value}"
)
# Add regularization penalties # Add regularization penalties
for layer in self.layers: for layer in self.layers:
loss_value += layer.compute_regularization_penalty() loss_value += layer.compute_regularization_penalty()
loss_grad = self.loss_fn.backward(y_pred, y) loss_grad = self.loss_fn.backward(y_pred, y)
# Backward pass # Backward pass
@@ -251,10 +263,11 @@ class FFNN:
# === Example usage === # === Example usage ===
if __name__ == "__main__": if __name__ == "__main__":
from easynn.schedulers import GradientDescentScheduler, AdamScheduler from easynn.schedulers import AdamScheduler
from sklearn.preprocessing import StandardScaler from sklearn.preprocessing import StandardScaler
x = np.linspace(0, 1, 100) x = np.linspace(0, 1, 100)
X = np.array([x, x + np.random.randn(len(x)), x + np.random.randn(len(x))*2]).T X = np.array([x, x + np.random.randn(len(x)), x + np.random.randn(len(x)) * 2]).T
X_scaler = StandardScaler().fit(X) X_scaler = StandardScaler().fit(X)
X_s = X_scaler.transform(X) X_s = X_scaler.transform(X)
y = np.array([x + x**2 + x**3]).T y = np.array([x + x**2 + x**3]).T
@@ -262,7 +275,11 @@ if __name__ == "__main__":
y_s = y_scaler.transform(y) y_s = y_scaler.transform(y)
print(X_s) print(X_s)
print(y_s) print(y_s)
layers = [Layer(3, 5, ReLU(), Regularization(0.01, 'l2')), Layer(5, 25, ReLU(), Regularization(0.01, 'l2')), Layer(25, 1, Linear())] layers = [
Layer(3, 5, ReLU(), Regularization(0.01, "l2")),
Layer(5, 25, ReLU(), Regularization(0.01, "l2")),
Layer(25, 1, Linear()),
]
network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), MSELoss()) network = FFNN(layers, AdamScheduler(learning_rate=0.001, epochs=1000), MSELoss())
network.fit(X_s, y_s) network.fit(X_s, y_s)
print(np.abs(np.mean(network.predict(X_s) - y_s))) print(np.abs(np.mean(network.predict(X_s) - y_s)))
src/easynn/schedulers.py
+5 -3
View File
@@ -14,10 +14,10 @@ class Scheduler:
@property @property
def cont(self) -> bool: def cont(self) -> bool:
return False return False
def record_loss(self, loss: float) -> None: def record_loss(self, loss: float) -> None:
self._loss_history.append(loss) self._loss_history.append(loss)
@property @property
def loss_history(self) -> np.ndarray: def loss_history(self) -> np.ndarray:
return np.array(self._loss_history) return np.array(self._loss_history)
@@ -71,7 +71,9 @@ class AdvancedScheduler(BasicScheduler):
) )
for i, (new_w_grad, new_b_grad) in enumerate(gradients) for i, (new_w_grad, new_b_grad) in enumerate(gradients)
] ]
logger.debug(f"Updates computed. Total update sum: {sum(np.sum(u) for pair in updates for u in pair)}") logger.debug(
f"Updates computed. Total update sum: {sum(np.sum(u) for pair in updates for u in pair)}"
)
self._post_update(gradients, updates) self._post_update(gradients, updates)
return updates return updates
uv.lock
Generated
+11
View File
@@ -187,6 +187,7 @@ dependencies = [
{ name = "pytest" }, { name = "pytest" },
{ name = "ruff" }, { name = "ruff" },
{ name = "scikit-learn" }, { name = "scikit-learn" },
{ name = "scikit-learn-stubs" },
{ name = "seaborn" }, { name = "seaborn" },
{ name = "tqdm" }, { name = "tqdm" },
] ]
@@ -202,6 +203,7 @@ requires-dist = [
{ name = "pytest", specifier = ">=8.4.2" }, { name = "pytest", specifier = ">=8.4.2" },
{ name = "ruff", specifier = ">=0.14.1" }, { name = "ruff", specifier = ">=0.14.1" },
{ name = "scikit-learn", specifier = ">=1.7.2" }, { name = "scikit-learn", specifier = ">=1.7.2" },
{ name = "scikit-learn-stubs", specifier = ">=0.0.3" },
{ name = "seaborn", specifier = ">=0.13.2" }, { name = "seaborn", specifier = ">=0.13.2" },
{ name = "tqdm", specifier = ">=4.67.1" }, { name = "tqdm", specifier = ">=4.67.1" },
] ]
@@ -976,6 +978,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8e/87/24f541b6d62b1794939ae6422f8023703bbf6900378b2b34e0b4384dfefd/scikit_learn-1.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:bb24510ed3f9f61476181e4db51ce801e2ba37541def12dc9333b946fc7a9cf8", size = 8820007, upload-time = "2025-09-09T08:21:26.713Z" }, { url = "https://files.pythonhosted.org/packages/8e/87/24f541b6d62b1794939ae6422f8023703bbf6900378b2b34e0b4384dfefd/scikit_learn-1.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:bb24510ed3f9f61476181e4db51ce801e2ba37541def12dc9333b946fc7a9cf8", size = 8820007, upload-time = "2025-09-09T08:21:26.713Z" },
] ]
[[package]]
name = "scikit-learn-stubs"
version = "0.0.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/0b/12/35f54848f6160ea53164ef8919d210e16c5d4f78be32a7605254651d32b7/scikit_learn_stubs-0.0.3.tar.gz", hash = "sha256:6a41fd3e26aedb923298c3f68a0746d2eed757f24615173f900ce2b942d70814", size = 123044, upload-time = "2025-08-28T01:26:49.584Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3f/ee/8c483b9384f269a14f62fa9760d0021273d1786128273ee14962468d9114/scikit_learn_stubs-0.0.3-py3-none-any.whl", hash = "sha256:5d51257ab62c79d265fa7b05e69ad64b038ac7803bfb1d6490562d4da3109aae", size = 247033, upload-time = "2025-08-28T01:26:48.199Z" },
]
[[package]] [[package]]
name = "scipy" name = "scipy"
version = "1.16.2" version = "1.16.2"