-We define first our design matrix and the various input vectors. +We define first our design matrix and the various output vectors for the different gates.
@@ -1613,7 +1613,7 @@ np.random.seed(0) # Defining the neural network n_inputs, n_features = X.shape n_hidden_neurons = 2 -n_categories = 1 +n_categories = 2 n_features = 2 # we make the weights normally distributed using numpy.random.randn @@ -1634,7 +1634,7 @@ predictions = predict(X) print(predictions)
-Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.
+Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.
@@ -1648,6 +1648,73 @@ Not an impressive result. Let us now add the full network with the back-propagat
+
+
+
+
-We define first our design matrix and the various input vectors.
+We define first our design matrix and the various output vectors for the different gates.
@@ -1625,7 +1629,7 @@ np.random.seed(0)
# Defining the neural network
n_inputs, n_features = X.shape
n_hidden_neurons = 2
-n_categories = 1
+n_categories = 2
n_features = 2
# we make the weights normally distributed using numpy.random.randn
@@ -1646,7 +1650,7 @@ predictions = predict(X)
print(predictions)
-Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.
+Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.
+
+
+
+
diff --git a/doc/pub/week41/html/week41.html b/doc/pub/week41/html/week41.html
index 9b52c171f..73e064d97 100644
--- a/doc/pub/week41/html/week41.html
+++ b/doc/pub/week41/html/week41.html
@@ -153,6 +153,10 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'the-full-network-for-the-various-gates'),
+ ('And the same using Scikit-Learn',
+ 2,
+ None,
+ 'and-the-same-using-scikit-learn'),
('Building neural networks in Tensorflow and Keras',
2,
None,
@@ -1569,7 +1573,7 @@ while the vector of outputs is \( \boldsymbol{y}^T=[0,1,1,0] \) for the XOR gate
-We define first our design matrix and the various input vectors.
+We define first our design matrix and the various output vectors for the different gates.
@@ -1630,7 +1634,7 @@ np.random.# Defining the neural network
n_inputs, n_features = X.shape
n_hidden_neurons = 2
-n_categories = 1
+n_categories = 2
n_features = 2
# we make the weights normally distributed using numpy.random.randn
@@ -1651,7 +1655,7 @@ predictions = predict(X)
print(predictions)
-Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.
+Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.
+
+
+
+
diff --git a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz
index 18192ecbc..c13aa571d 100644
Binary files a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz and b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz differ
diff --git a/doc/pub/week41/ipynb/week41.ipynb b/doc/pub/week41/ipynb/week41.ipynb
index af4c3a2bd..6ee3ddec4 100644
--- a/doc/pub/week41/ipynb/week41.ipynb
+++ b/doc/pub/week41/ipynb/week41.ipynb
@@ -1458,7 +1458,7 @@
"\n",
"## Setting up the Neural Network\n",
"\n",
- "We define first our design matrix and the various input vectors."
+ "We define first our design matrix and the various output vectors for the different gates."
]
},
{
@@ -1537,7 +1537,7 @@
"# Defining the neural network\n",
"n_inputs, n_features = X.shape\n",
"n_hidden_neurons = 2\n",
- "n_categories = 1\n",
+ "n_categories = 2\n",
"n_features = 2\n",
"\n",
"# we make the weights normally distributed using numpy.random.randn\n",
@@ -1562,11 +1562,87 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.\n",
+ "Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.\n",
"\n",
"## The full Network for the Various Gates"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## And the same using Scikit-Learn"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "# import necessary packages\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.neural_network import MLPClassifier\n",
+ "from sklearn.metrics import accuracy_score\n",
+ "import seaborn as sns\n",
+ "\n",
+ "# ensure the same random numbers appear every time\n",
+ "np.random.seed(0)\n",
+ "\n",
+ "# Design matrix\n",
+ "X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)\n",
+ "\n",
+ "# The XOR gate\n",
+ "yXOR = np.array( [ 0, 1 ,1, 0])\n",
+ "# The OR gate\n",
+ "yOR = np.array( [ 0, 1 ,1, 1])\n",
+ "# The AND gate\n",
+ "yAND = np.array( [ 0, 0 ,0, 1])\n",
+ "\n",
+ "# Defining the neural network\n",
+ "n_inputs, n_features = X.shape\n",
+ "n_hidden_neurons = 2\n",
+ "n_categories = 2\n",
+ "n_features = 2\n",
+ "\n",
+ "eta_vals = np.logspace(-5, 1, 7)\n",
+ "lmbd_vals = np.logspace(-5, 1, 7)\n",
+ "# store models for later use\n",
+ "DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n",
+ "epochs = 100\n",
+ "\n",
+ "for i, eta in enumerate(eta_vals):\n",
+ " for j, lmbd in enumerate(lmbd_vals):\n",
+ " dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',\n",
+ " alpha=lmbd, learning_rate_init=eta, max_iter=epochs)\n",
+ " dnn.fit(X, yXOR)\n",
+ " DNN_scikit[i][j] = dnn\n",
+ " print(\"Learning rate = \", eta)\n",
+ " print(\"Lambda = \", lmbd)\n",
+ " print(\"Accuracy score on data set: \", dnn.score(X, yXOR))\n",
+ " print()\n",
+ "\n",
+ "sns.set()\n",
+ "test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))\n",
+ "for i in range(len(eta_vals)):\n",
+ " for j in range(len(lmbd_vals)):\n",
+ " dnn = DNN_scikit[i][j]\n",
+ " test_pred = dnn.predict(X)\n",
+ " test_accuracy[i][j] = accuracy_score(yXOR, test_pred)\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize = (10, 10))\n",
+ "sns.heatmap(test_accuracy, annot=True, ax=ax, cmap=\"viridis\")\n",
+ "ax.set_title(\"Test Accuracy\")\n",
+ "ax.set_ylabel(\"$\\eta$\")\n",
+ "ax.set_xlabel(\"$\\lambda$\")\n",
+ "plt.show()"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
diff --git a/doc/src/week41/programs/FFpart.py b/doc/src/week41/programs/FFpart.py
new file mode 100644
index 000000000..822977841
--- /dev/null
+++ b/doc/src/week41/programs/FFpart.py
@@ -0,0 +1,67 @@
+"""
+Simple code that tests XOR, OR and AND gates with linear regression
+"""
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
+def sigmoid(x):
+ return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ probabilities = sigmoid(z_o)
+ return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+ probabilities = feed_forward(X)
+ return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
diff --git a/doc/src/week41/nn.py b/doc/src/week41/programs/FFpart.py~
similarity index 100%
rename from doc/src/week41/nn.py
rename to doc/src/week41/programs/FFpart.py~
diff --git a/doc/src/week41/programs/ffnn.py b/doc/src/week41/programs/ffnn.py
new file mode 100644
index 000000000..c68a4fdae
--- /dev/null
+++ b/doc/src/week41/programs/ffnn.py
@@ -0,0 +1,428 @@
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+batch_size = 100
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X, yXOR)
+ DNN_scikit[i][j] = dnn
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on data set: ", dnn.score(X, yXOR))
+ print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+ test_pred = dnn.predict(X)
+ test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+
+
+
+"""
+# one-hot in numpy
+def to_categorical_numpy(integer_vector):
+ n_inputs = len(integer_vector)
+ n_categories = np.max(integer_vector) + 1
+ onehot_vector = np.zeros((n_inputs, n_categories))
+ onehot_vector[range(n_inputs), integer_vector] = 1
+
+ return onehot_vector
+
+#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test)
+Y_train_onehot, Y_test_onehot = to_categorical_numpy(Y_train), to_categorical_numpy(Y_test)
+
+def feed_forward_train(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ exp_term = np.exp(z_o)
+ probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+
+ # for backpropagation need activations in hidden and output layers
+ return a_h, probabilities
+
+def backpropagation(X, Y):
+ a_h, probabilities = feed_forward_train(X)
+
+ # error in the output layer
+ error_output = probabilities - Y
+ # error in the hidden layer
+ error_hidden = np.matmul(error_output, output_weights.T) * a_h * (1 - a_h)
+
+ # gradients for the output layer
+ output_weights_gradient = np.matmul(a_h.T, error_output)
+ output_bias_gradient = np.sum(error_output, axis=0)
+
+ # gradient for the hidden layer
+ hidden_weights_gradient = np.matmul(X.T, error_hidden)
+ hidden_bias_gradient = np.sum(error_hidden, axis=0)
+
+ return output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
+
+print("Old accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
+
+eta = 0.01
+lmbd = 0.01
+for i in range(1000):
+ # calculate gradients
+ dWo, dBo, dWh, dBh = backpropagation(X_train, Y_train_onehot)
+
+ # regularization term gradients
+ dWo += lmbd * output_weights
+ dWh += lmbd * hidden_weights
+
+ # update weights and biases
+ output_weights -= eta * dWo
+ output_bias -= eta * dBo
+ hidden_weights -= eta * dWh
+ hidden_bias -= eta * dBh
+
+print("New accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
+!ec
+
+!split
+===== Improving performance =====
+
+As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image.
+In order to obtain a network that does something useful, we will have to do a bit more work.
+
+The choice of *hyperparameters* such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a *grid-search* is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates $\eta = 10^{-6}, 10^{-5},...,10^{-1}$ with different regularization parameters $\lambda = 10^{-6},...,10^{-0}$.
+
+Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an *iteration*, and a full training period
+going through the entire dataset ($n/M$ batches) an *epoch*.
+
+If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers.
+Andrew Ng goes through some of these considerations in this "video":"https://youtu.be/F1ka6a13S9I". You can find a summary of the video "here":"https://kevinzakka.github.io/2016/09/26/applying-deep-learning/".
+
+!split
+===== Full object-oriented implementation =====
+
+It is very natural to think of the network as an object, with specific instances of the network
+being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below.
+
+
+!bc pycod
+class NeuralNetwork:
+ def __init__(
+ self,
+ X_data,
+ Y_data,
+ n_hidden_neurons=50,
+ n_categories=10,
+ epochs=10,
+ batch_size=100,
+ eta=0.1,
+ lmbd=0.0):
+
+ self.X_data_full = X_data
+ self.Y_data_full = Y_data
+
+ self.n_inputs = X_data.shape[0]
+ self.n_features = X_data.shape[1]
+ self.n_hidden_neurons = n_hidden_neurons
+ self.n_categories = n_categories
+
+ self.epochs = epochs
+ self.batch_size = batch_size
+ self.iterations = self.n_inputs // self.batch_size
+ self.eta = eta
+ self.lmbd = lmbd
+
+ self.create_biases_and_weights()
+
+ def create_biases_and_weights(self):
+ self.hidden_weights = np.random.randn(self.n_features, self.n_hidden_neurons)
+ self.hidden_bias = np.zeros(self.n_hidden_neurons) + 0.01
+
+ self.output_weights = np.random.randn(self.n_hidden_neurons, self.n_categories)
+ self.output_bias = np.zeros(self.n_categories) + 0.01
+
+ def feed_forward(self):
+ # feed-forward for training
+ self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias
+ self.a_h = sigmoid(self.z_h)
+
+ self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias
+
+ exp_term = np.exp(self.z_o)
+ self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+
+ def feed_forward_out(self, X):
+ # feed-forward for output
+ z_h = np.matmul(X, self.hidden_weights) + self.hidden_bias
+ a_h = sigmoid(z_h)
+
+ z_o = np.matmul(a_h, self.output_weights) + self.output_bias
+
+ exp_term = np.exp(z_o)
+ probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+ return probabilities
+
+ def backpropagation(self):
+ error_output = self.probabilities - self.Y_data
+ error_hidden = np.matmul(error_output, self.output_weights.T) * self.a_h * (1 - self.a_h)
+
+ self.output_weights_gradient = np.matmul(self.a_h.T, error_output)
+ self.output_bias_gradient = np.sum(error_output, axis=0)
+
+ self.hidden_weights_gradient = np.matmul(self.X_data.T, error_hidden)
+ self.hidden_bias_gradient = np.sum(error_hidden, axis=0)
+
+ if self.lmbd > 0.0:
+ self.output_weights_gradient += self.lmbd * self.output_weights
+ self.hidden_weights_gradient += self.lmbd * self.hidden_weights
+
+ self.output_weights -= self.eta * self.output_weights_gradient
+ self.output_bias -= self.eta * self.output_bias_gradient
+ self.hidden_weights -= self.eta * self.hidden_weights_gradient
+ self.hidden_bias -= self.eta * self.hidden_bias_gradient
+
+ def predict(self, X):
+ probabilities = self.feed_forward_out(X)
+ return np.argmax(probabilities, axis=1)
+
+ def predict_probabilities(self, X):
+ probabilities = self.feed_forward_out(X)
+ return probabilities
+
+ def train(self):
+ data_indices = np.arange(self.n_inputs)
+
+ for i in range(self.epochs):
+ for j in range(self.iterations):
+ # pick datapoints with replacement
+ chosen_datapoints = np.random.choice(
+ data_indices, size=self.batch_size, replace=False
+ )
+
+ # minibatch training data
+ self.X_data = self.X_data_full[chosen_datapoints]
+ self.Y_data = self.Y_data_full[chosen_datapoints]
+
+ self.feed_forward()
+ self.backpropagation()
+!ec
+
+!split
+===== Evaluate model performance on test data =====
+
+To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data.
+We measure the performance of the network using the *accuracy* score.
+The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of $1$.
+
+$$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\hat{y}_i = y_i)}{n} ,$$
+
+where $I$ is the indicator function, $1$ if $\hat{y}_i = y_i$ and $0$ otherwise.
+
+
+!bc pycod
+epochs = 100
+batch_size = 100
+
+dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
+ n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
+dnn.train()
+test_predict = dnn.predict(X_test)
+
+# accuracy score from scikit library
+print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
+
+# equivalent in numpy
+def accuracy_score_numpy(Y_test, Y_pred):
+ return np.sum(Y_test == Y_pred) / len(Y_test)
+
+#print("Accuracy score on test set: ", accuracy_score_numpy(Y_test, test_predict))
+!ec
+
+!split
+===== Adjust hyperparameters =====
+
+We now perform a grid search to find the optimal hyperparameters for the network.
+Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98\%$ ($2\%$ error rate).
+
+!bc pycod
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store the models for later use
+DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+
+# grid search
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
+ n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
+ dnn.train()
+
+ DNN_numpy[i][j] = dnn
+
+ test_predict = dnn.predict(X_test)
+
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
+ print()
+!ec
+
+!split
+===== Visualization =====
+
+!bc pycod
+# visual representation of grid search
+# uses seaborn heatmap, you can also do this with matplotlib imshow
+import seaborn as sns
+
+sns.set()
+
+train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_numpy[i][j]
+
+ train_pred = dnn.predict(X_train)
+ test_pred = dnn.predict(X_test)
+
+ train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
+ test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
+
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Training Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+!ec
+
+!split
+===== scikit-learn implementation =====
+
+_scikit-learn_ focuses more
+on traditional machine learning methods, such as regression,
+clustering, decision trees, etc. As such, it has only two types of
+neural networks: Multi Layer Perceptron outputting continuous values,
+*MPLRegressor*, and Multi Layer Perceptron outputting labels,
+*MLPClassifier*. We will see how simple it is to use these classes.
+
+_scikit-learn_ implements a few improvements from our neural network,
+such as early stopping, a varying learning rate, different
+optimization methods, etc. We would therefore expect a better
+performance overall.
+
+!bc pycod
+from sklearn.neural_network import MLPClassifier
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X_train, Y_train)
+
+ DNN_scikit[i][j] = dnn
+
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on test set: ", dnn.score(X_test, Y_test))
+ print()
+!ec
+
+
+!split
+===== Visualization =====
+!bc pycod
+# optional
+# visual representation of grid search
+# uses seaborn heatmap, could probably do this in matplotlib
+import seaborn as sns
+
+sns.set()
+
+train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+
+ train_pred = dnn.predict(X_train)
+ test_pred = dnn.predict(X_test)
+
+ train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
+ test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
+
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Training Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+!ec
+"""
+
diff --git a/doc/src/week41/programs/ffnn.py~ b/doc/src/week41/programs/ffnn.py~
new file mode 100644
index 000000000..21deb460b
--- /dev/null
+++ b/doc/src/week41/programs/ffnn.py~
@@ -0,0 +1,428 @@
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+batch_size = 100
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X, yXOR)
+ DNN_scikit[i][j] = dnn
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on data set: ", dnn.score(X, yXOR))
+ print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+ test_pred = dnn.predict(X)
+ test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+
+
+
+"""
+# one-hot in numpy
+def to_categorical_numpy(integer_vector):
+ n_inputs = len(integer_vector)
+ n_categories = np.max(integer_vector) + 1
+ onehot_vector = np.zeros((n_inputs, n_categories))
+ onehot_vector[range(n_inputs), integer_vector] = 1
+
+ return onehot_vector
+
+#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test)
+Y_train_onehot, Y_test_onehot = to_categorical_numpy(Y_train), to_categorical_numpy(Y_test)
+
+def feed_forward_train(X):
+ # weighted sum of inputs to the hidden layer
+ z_h = np.matmul(X, hidden_weights) + hidden_bias
+ # activation in the hidden layer
+ a_h = sigmoid(z_h)
+
+ # weighted sum of inputs to the output layer
+ z_o = np.matmul(a_h, output_weights) + output_bias
+ # softmax output
+ # axis 0 holds each input and axis 1 the probabilities of each category
+ exp_term = np.exp(z_o)
+ probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+
+ # for backpropagation need activations in hidden and output layers
+ return a_h, probabilities
+
+def backpropagation(X, Y):
+ a_h, probabilities = feed_forward_train(X)
+
+ # error in the output layer
+ error_output = probabilities - Y
+ # error in the hidden layer
+ error_hidden = np.matmul(error_output, output_weights.T) * a_h * (1 - a_h)
+
+ # gradients for the output layer
+ output_weights_gradient = np.matmul(a_h.T, error_output)
+ output_bias_gradient = np.sum(error_output, axis=0)
+
+ # gradient for the hidden layer
+ hidden_weights_gradient = np.matmul(X.T, error_hidden)
+ hidden_bias_gradient = np.sum(error_hidden, axis=0)
+
+ return output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
+
+print("Old accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
+
+eta = 0.01
+lmbd = 0.01
+for i in range(1000):
+ # calculate gradients
+ dWo, dBo, dWh, dBh = backpropagation(X_train, Y_train_onehot)
+
+ # regularization term gradients
+ dWo += lmbd * output_weights
+ dWh += lmbd * hidden_weights
+
+ # update weights and biases
+ output_weights -= eta * dWo
+ output_bias -= eta * dBo
+ hidden_weights -= eta * dWh
+ hidden_bias -= eta * dBh
+
+print("New accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
+!ec
+
+!split
+===== Improving performance =====
+
+As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image.
+In order to obtain a network that does something useful, we will have to do a bit more work.
+
+The choice of *hyperparameters* such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a *grid-search* is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates $\eta = 10^{-6}, 10^{-5},...,10^{-1}$ with different regularization parameters $\lambda = 10^{-6},...,10^{-0}$.
+
+Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an *iteration*, and a full training period
+going through the entire dataset ($n/M$ batches) an *epoch*.
+
+If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers.
+Andrew Ng goes through some of these considerations in this "video":"https://youtu.be/F1ka6a13S9I". You can find a summary of the video "here":"https://kevinzakka.github.io/2016/09/26/applying-deep-learning/".
+
+!split
+===== Full object-oriented implementation =====
+
+It is very natural to think of the network as an object, with specific instances of the network
+being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below.
+
+
+!bc pycod
+class NeuralNetwork:
+ def __init__(
+ self,
+ X_data,
+ Y_data,
+ n_hidden_neurons=50,
+ n_categories=10,
+ epochs=10,
+ batch_size=100,
+ eta=0.1,
+ lmbd=0.0):
+
+ self.X_data_full = X_data
+ self.Y_data_full = Y_data
+
+ self.n_inputs = X_data.shape[0]
+ self.n_features = X_data.shape[1]
+ self.n_hidden_neurons = n_hidden_neurons
+ self.n_categories = n_categories
+
+ self.epochs = epochs
+ self.batch_size = batch_size
+ self.iterations = self.n_inputs // self.batch_size
+ self.eta = eta
+ self.lmbd = lmbd
+
+ self.create_biases_and_weights()
+
+ def create_biases_and_weights(self):
+ self.hidden_weights = np.random.randn(self.n_features, self.n_hidden_neurons)
+ self.hidden_bias = np.zeros(self.n_hidden_neurons) + 0.01
+
+ self.output_weights = np.random.randn(self.n_hidden_neurons, self.n_categories)
+ self.output_bias = np.zeros(self.n_categories) + 0.01
+
+ def feed_forward(self):
+ # feed-forward for training
+ self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias
+ self.a_h = sigmoid(self.z_h)
+
+ self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias
+
+ exp_term = np.exp(self.z_o)
+ self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+
+ def feed_forward_out(self, X):
+ # feed-forward for output
+ z_h = np.matmul(X, self.hidden_weights) + self.hidden_bias
+ a_h = sigmoid(z_h)
+
+ z_o = np.matmul(a_h, self.output_weights) + self.output_bias
+
+ exp_term = np.exp(z_o)
+ probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+ return probabilities
+
+ def backpropagation(self):
+ error_output = self.probabilities - self.Y_data
+ error_hidden = np.matmul(error_output, self.output_weights.T) * self.a_h * (1 - self.a_h)
+
+ self.output_weights_gradient = np.matmul(self.a_h.T, error_output)
+ self.output_bias_gradient = np.sum(error_output, axis=0)
+
+ self.hidden_weights_gradient = np.matmul(self.X_data.T, error_hidden)
+ self.hidden_bias_gradient = np.sum(error_hidden, axis=0)
+
+ if self.lmbd > 0.0:
+ self.output_weights_gradient += self.lmbd * self.output_weights
+ self.hidden_weights_gradient += self.lmbd * self.hidden_weights
+
+ self.output_weights -= self.eta * self.output_weights_gradient
+ self.output_bias -= self.eta * self.output_bias_gradient
+ self.hidden_weights -= self.eta * self.hidden_weights_gradient
+ self.hidden_bias -= self.eta * self.hidden_bias_gradient
+
+ def predict(self, X):
+ probabilities = self.feed_forward_out(X)
+ return np.argmax(probabilities, axis=1)
+
+ def predict_probabilities(self, X):
+ probabilities = self.feed_forward_out(X)
+ return probabilities
+
+ def train(self):
+ data_indices = np.arange(self.n_inputs)
+
+ for i in range(self.epochs):
+ for j in range(self.iterations):
+ # pick datapoints with replacement
+ chosen_datapoints = np.random.choice(
+ data_indices, size=self.batch_size, replace=False
+ )
+
+ # minibatch training data
+ self.X_data = self.X_data_full[chosen_datapoints]
+ self.Y_data = self.Y_data_full[chosen_datapoints]
+
+ self.feed_forward()
+ self.backpropagation()
+!ec
+
+!split
+===== Evaluate model performance on test data =====
+
+To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data.
+We measure the performance of the network using the *accuracy* score.
+The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of $1$.
+
+$$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\hat{y}_i = y_i)}{n} ,$$
+
+where $I$ is the indicator function, $1$ if $\hat{y}_i = y_i$ and $0$ otherwise.
+
+
+!bc pycod
+epochs = 100
+batch_size = 100
+
+dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
+ n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
+dnn.train()
+test_predict = dnn.predict(X_test)
+
+# accuracy score from scikit library
+print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
+
+# equivalent in numpy
+def accuracy_score_numpy(Y_test, Y_pred):
+ return np.sum(Y_test == Y_pred) / len(Y_test)
+
+#print("Accuracy score on test set: ", accuracy_score_numpy(Y_test, test_predict))
+!ec
+
+!split
+===== Adjust hyperparameters =====
+
+We now perform a grid search to find the optimal hyperparameters for the network.
+Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98\%$ ($2\%$ error rate).
+
+!bc pycod
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store the models for later use
+DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+
+# grid search
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
+ n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
+ dnn.train()
+
+ DNN_numpy[i][j] = dnn
+
+ test_predict = dnn.predict(X_test)
+
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
+ print()
+!ec
+
+!split
+===== Visualization =====
+
+!bc pycod
+# visual representation of grid search
+# uses seaborn heatmap, you can also do this with matplotlib imshow
+import seaborn as sns
+
+sns.set()
+
+train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_numpy[i][j]
+
+ train_pred = dnn.predict(X_train)
+ test_pred = dnn.predict(X_test)
+
+ train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
+ test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
+
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Training Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+!ec
+
+!split
+===== scikit-learn implementation =====
+
+_scikit-learn_ focuses more
+on traditional machine learning methods, such as regression,
+clustering, decision trees, etc. As such, it has only two types of
+neural networks: Multi Layer Perceptron outputting continuous values,
+*MPLRegressor*, and Multi Layer Perceptron outputting labels,
+*MLPClassifier*. We will see how simple it is to use these classes.
+
+_scikit-learn_ implements a few improvements from our neural network,
+such as early stopping, a varying learning rate, different
+optimization methods, etc. We would therefore expect a better
+performance overall.
+
+!bc pycod
+from sklearn.neural_network import MLPClassifier
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X_train, Y_train)
+
+ DNN_scikit[i][j] = dnn
+
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on test set: ", dnn.score(X_test, Y_test))
+ print()
+!ec
+
+
+!split
+===== Visualization =====
+!bc pycod
+# optional
+# visual representation of grid search
+# uses seaborn heatmap, could probably do this in matplotlib
+import seaborn as sns
+
+sns.set()
+
+train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+
+ train_pred = dnn.predict(X_train)
+ test_pred = dnn.predict(X_test)
+
+ train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
+ test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
+
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Training Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+!ec
+"""
+
diff --git a/doc/src/week41/week41.do.txt b/doc/src/week41/week41.do.txt
index 442c8902a..def206c25 100644
--- a/doc/src/week41/week41.do.txt
+++ b/doc/src/week41/week41.do.txt
@@ -1125,7 +1125,7 @@ while the vector of outputs is $\bm{y}^T=[0,1,1,0]$ for the XOR gate, $\bm{y}^T=
!split
===== Setting up the Neural Network =====
-We define first our design matrix and the various input vectors.
+We define first our design matrix and the various output vectors for the different gates.
!bc pycod
"""
@@ -1184,7 +1184,7 @@ np.random.seed(0)
# Defining the neural network
n_inputs, n_features = X.shape
n_hidden_neurons = 2
-n_categories = 1
+n_categories = 2
n_features = 2
# we make the weights normally distributed using numpy.random.randn
@@ -1204,7 +1204,7 @@ print(probabilities)
predictions = predict(X)
print(predictions)
!ec
-Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.
+Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.
!split
===== The full Network for the Various Gates =====
@@ -1212,6 +1212,71 @@ Not an impressive result. Let us now add the full network with the back-propagat
!ec
+!split
+===== And the same using Scikit-Learn =====
+
+!bc pycod
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X, yXOR)
+ DNN_scikit[i][j] = dnn
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on data set: ", dnn.score(X, yXOR))
+ print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+ test_pred = dnn.predict(X)
+ test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+!ec
+
!split
===== Building neural networks in Tensorflow and Keras =====
And the same using Scikit-Learn
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X, yXOR)
+ DNN_scikit[i][j] = dnn
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on data set: ", dnn.score(X, yXOR))
+ print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+ test_pred = dnn.predict(X)
+ test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
Building neural networks in Tensorflow and Keras
diff --git a/doc/pub/week41/html/week41-solarized.html b/doc/pub/week41/html/week41-solarized.html
index 8a5cdc93d..21d63c636 100644
--- a/doc/pub/week41/html/week41-solarized.html
+++ b/doc/pub/week41/html/week41-solarized.html
@@ -148,6 +148,10 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'the-full-network-for-the-various-gates'),
+ ('And the same using Scikit-Learn',
+ 2,
+ None,
+ 'and-the-same-using-scikit-learn'),
('Building neural networks in Tensorflow and Keras',
2,
None,
@@ -1564,7 +1568,7 @@ while the vector of outputs is \( \boldsymbol{y}^T=[0,1,1,0] \) for the XOR gate
Setting up the Neural Network
@@ -1660,6 +1664,72 @@ Not an impressive result. Let us now add the full network with the back-propagat
+And the same using Scikit-Learn
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X, yXOR)
+ DNN_scikit[i][j] = dnn
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on data set: ", dnn.score(X, yXOR))
+ print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+ test_pred = dnn.predict(X)
+ test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+
Building neural networks in Tensorflow and Keras
Setting up the Neural Network
@@ -1665,6 +1669,72 @@ Not an impressive result. Let us now add the full network with the back-propagat
+And the same using Scikit-Learn
+
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+ for j, lmbd in enumerate(lmbd_vals):
+ dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+ alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+ dnn.fit(X, yXOR)
+ DNN_scikit[i][j] = dnn
+ print("Learning rate = ", eta)
+ print("Lambda = ", lmbd)
+ print("Accuracy score on data set: ", dnn.score(X, yXOR))
+ print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+ for j in range(len(lmbd_vals)):
+ dnn = DNN_scikit[i][j]
+ test_pred = dnn.predict(X)
+ test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+
Building neural networks in Tensorflow and Keras