- - -
-diff --git a/doc/pub/week41/html/week41-solarized.html b/doc/pub/week41/html/week41-solarized.html index 21d63c636..34c3c70a3 100644 --- a/doc/pub/week41/html/week41-solarized.html +++ b/doc/pub/week41/html/week41-solarized.html @@ -144,14 +144,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'then-the-first-feed-forward-pass'), - ('The full Network for the Various Gates', + ('The Code using Scikit-Learn', 2, None, - 'the-full-network-for-the-various-gates'), - ('And the same using Scikit-Learn', - 2, - None, - 'and-the-same-using-scikit-learn'), + 'the-code-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1655,16 +1651,7 @@ Not an impressive result, but this was our first forward pass with randomly assi
-
- - -
-
-
-
-
diff --git a/doc/pub/week41/html/week41.html b/doc/pub/week41/html/week41.html index 73e064d97..784fd1ca8 100644 --- a/doc/pub/week41/html/week41.html +++ b/doc/pub/week41/html/week41.html @@ -149,14 +149,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'then-the-first-feed-forward-pass'), - ('The full Network for the Various Gates', + ('The Code using Scikit-Learn', 2, None, - 'the-full-network-for-the-various-gates'), - ('And the same using Scikit-Learn', - 2, - None, - 'and-the-same-using-scikit-learn'), + 'the-code-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1660,16 +1656,7 @@ Not an impressive result, but this was our first forward pass with randomly assi
-
- - -
-
-
-
-
diff --git a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz index c13aa571d..252d351a1 100644 Binary files a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz and b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz differ diff --git a/doc/pub/week41/ipynb/week41.ipynb b/doc/pub/week41/ipynb/week41.ipynb index 6ee3ddec4..c8a3eb2e2 100644 --- a/doc/pub/week41/ipynb/week41.ipynb +++ b/doc/pub/week41/ipynb/week41.ipynb @@ -1564,14 +1564,7 @@ "source": [ "Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.\n", "\n", - "## The full Network for the Various Gates" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## And the same using Scikit-Learn" + "## The Code using Scikit-Learn" ] }, { diff --git a/doc/src/week41/programs/nnown.py b/doc/src/week41/programs/nnown.py new file mode 100644 index 000000000..5be104ded --- /dev/null +++ b/doc/src/week41/programs/nnown.py @@ -0,0 +1,172 @@ + +# import necessary packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn.neural_network import MLPClassifier +from sklearn.metrics import accuracy_score +import seaborn as sns + +# ensure the same random numbers appear every time +np.random.seed(0) + +# Design matrix +X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64) + +# The XOR gate +yXOR = np.array( [ 0, 1 ,1, 0]) +# The OR gate +yOR = np.array( [ 0, 1 ,1, 1]) +# The AND gate +yAND = np.array( [ 0, 0 ,0, 1]) + + +# Defining the neural network +n_inputs, n_features = X.shape +n_hidden_neurons = 2 +n_categories = 2 +n_features = 2 + +def sigmoid(x): + return 1/(1 + np.exp(-x)) + + +class NeuralNetwork: + def __init__( + self, + X_data, + Y_data, + n_hidden_neurons=2, + n_categories=2, + epochs=10, + batch_size=100, + eta=0.1, + lmbd=0.0): + + self.X_data_full = X_data + self.Y_data_full = Y_data + + self.n_inputs = X_data.shape[0] + self.n_features = X_data.shape[1] + self.n_hidden_neurons = n_hidden_neurons + self.n_categories = n_categories + + self.epochs = epochs + self.batch_size = batch_size + self.iterations = self.n_inputs // self.batch_size + self.eta = eta + self.lmbd = lmbd + + self.create_biases_and_weights() + + + def create_biases_and_weights(self): + self.hidden_weights = np.random.randn(self.n_features, self.n_hidden_neurons) + self.hidden_bias = np.zeros(self.n_hidden_neurons) + 0.01 + + self.output_weights = np.random.randn(self.n_hidden_neurons, self.n_categories) + self.output_bias = np.zeros(self.n_categories) + 0.01 + + def feed_forward(self): + # feed-forward for training + self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias + self.a_h = sigmoid(self.z_h) + + self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias + + exp_term = sigmoid(self.z_o) + self.probabilities = exp_term + + def feed_forward_out(self, X): + # feed-forward for output + z_h = np.matmul(X, self.hidden_weights) + self.hidden_bias + a_h = sigmoid(z_h) + + z_o = np.matmul(a_h, self.output_weights) + self.output_bias + + exp_term = sigmoid(z_o) + probabilities = exp_term + return probabilities + + def backpropagation(self): + error_output = self.probabilities - self.Y_data + error_hidden = np.matmul(error_output, self.output_weights.T) * self.a_h * (1 - self.a_h) + + self.output_weights_gradient = np.matmul(self.a_h.T, error_output) + self.output_bias_gradient = np.sum(error_output, axis=0) + + self.hidden_weights_gradient = np.matmul(self.X_data.T, error_hidden) + self.hidden_bias_gradient = np.sum(error_hidden, axis=0) + + if self.lmbd > 0.0: + self.output_weights_gradient += self.lmbd * self.output_weights + self.hidden_weights_gradient += self.lmbd * self.hidden_weights + + self.output_weights -= self.eta * self.output_weights_gradient + self.output_bias -= self.eta * self.output_bias_gradient + self.hidden_weights -= self.eta * self.hidden_weights_gradient + self.hidden_bias -= self.eta * self.hidden_bias_gradient + + def predict(self, X): + probabilities = self.feed_forward_out(X) + return np.argmax(probabilities, axis=1) + + def predict_probabilities(self, X): + probabilities = self.feed_forward_out(X) + return probabilities + + def train(self): + data_indices = np.arange(self.n_inputs) + + for i in range(self.epochs): + for j in range(self.iterations): + # pick datapoints with replacement + chosen_datapoints = np.random.choice( + data_indices, size=self.batch_size, replace=False + ) + + # minibatch training data + self.X_data = self.X_data_full[chosen_datapoints] + self.Y_data = self.Y_data_full[chosen_datapoints] + + self.feed_forward() + self.backpropagation() + +epochs = 100 +batch_size = 100 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store the models for later use +DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +# grid search +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = NeuralNetwork(X, yXOR, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, + n_hidden_neurons=n_hidden_neurons, n_categories=n_categories) + dnn.train() + DNN_numpy[i][j] = dnn + test_predict = dnn.predict(X) + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on test set: ", accuracy_score(yXOR, test_predict)) + print() + +sns.set() +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_numpy[i][j] + test_pred = dnn.predict(X) + test_accuracy[i][j] = accuracy_score(yXOR, test_pred) + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + + + diff --git a/doc/src/week41/week41.do.txt b/doc/src/week41/week41.do.txt index def206c25..637c5b33a 100644 --- a/doc/src/week41/week41.do.txt +++ b/doc/src/week41/week41.do.txt @@ -1207,13 +1207,7 @@ print(predictions) Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above. !split -===== The full Network for the Various Gates ===== -!bc pycod - -!ec - -!split -===== And the same using Scikit-Learn ===== +===== The Code using Scikit-Learn ===== !bc pycod