From d70ccf7b9afdf5e3c60aea3682316ef1d0402dae Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Wed, 13 Oct 2021 22:28:17 +0200 Subject: [PATCH] update --- doc/pub/week41/html/week41-bs.html | 87 ++-- doc/pub/week41/html/week41-reveal.html | 73 ++- doc/pub/week41/html/week41-solarized.html | 76 +++- doc/pub/week41/html/week41.html | 76 +++- doc/pub/week41/ipynb/ipynb-week41-src.tar.gz | Bin 534110 -> 534110 bytes doc/pub/week41/ipynb/week41.ipynb | 82 +++- doc/src/week41/programs/FFpart.py | 67 +++ doc/src/week41/{nn.py => programs/FFpart.py~} | 0 doc/src/week41/programs/ffnn.py | 428 ++++++++++++++++++ doc/src/week41/programs/ffnn.py~ | 428 ++++++++++++++++++ doc/src/week41/week41.do.txt | 71 ++- 11 files changed, 1332 insertions(+), 56 deletions(-) create mode 100644 doc/src/week41/programs/FFpart.py rename doc/src/week41/{nn.py => programs/FFpart.py~} (100%) create mode 100644 doc/src/week41/programs/ffnn.py create mode 100644 doc/src/week41/programs/ffnn.py~ diff --git a/doc/pub/week41/html/week41-bs.html b/doc/pub/week41/html/week41-bs.html index f534b7038..ce099accf 100644 --- a/doc/pub/week41/html/week41-bs.html +++ b/doc/pub/week41/html/week41-bs.html @@ -128,6 +128,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'the-full-network-for-the-various-gates'), + ('And the same using Scikit-Learn', + 2, + None, + 'and-the-same-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -273,7 +277,7 @@ MathJax.Hub.Config({
  • Example: binary classification problem
  • The Softmax function
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • +
  • Collect and pre-process data
  • Train and test datasets
  • Define model and architecture
  • Layers
    • @@ -297,45 +301,46 @@ MathJax.Hub.Config({
  • Setting up the Neural Network
  • Then the first Feed Forward pass
  • The full Network for the Various Gates
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Which activation function should I use?
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • The derivative of the Logistic funtion
    • -
  • The RELU function family
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A very nice website on Neural Networks
    • -
  • A top-down perspective on Neural networks
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Overarching Views, a personal note
    • -
  • From a Spherical Cow to a real one
    • -
  • Convolutional Neural Networks (recognizing images)
    • -
  • Regular NNs don’t scale well to full images
    • -
  • 3D volumes of neurons
    • -
  • Layers used to build CNNs
    • -
  • Transforming images
    • -
  • CNNs in brief
    • -
  • CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
    • -
  • Setting it up
    • -
  • The MNIST dataset again
    • -
  • Strong correlations
    • -
  • Layers of a CNN
    • -
  • Systematic reduction
    • -
  • Prerequisites: Collect and pre-process data
    • -
  • Importing Keras and Tensorflow
    • -
  • Running with Keras
    • -
  • Final part
    • -
  • Final visualization
    • -
  • Fun links
    • +
  • And the same using Scikit-Learn
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Which activation function should I use?
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • The derivative of the Logistic funtion
    • +
  • The RELU function family
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A very nice website on Neural Networks
    • +
  • A top-down perspective on Neural networks
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Overarching Views, a personal note
    • +
  • From a Spherical Cow to a real one
    • +
  • Convolutional Neural Networks (recognizing images)
    • +
  • Regular NNs don’t scale well to full images
    • +
  • 3D volumes of neurons
    • +
  • Layers used to build CNNs
    • +
  • Transforming images
    • +
  • CNNs in brief
    • +
  • CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
    • +
  • Setting it up
    • +
  • The MNIST dataset again
    • +
  • Strong correlations
    • +
  • Layers of a CNN
    • +
  • Systematic reduction
    • +
  • Prerequisites: Collect and pre-process data
    • +
  • Importing Keras and Tensorflow
    • +
  • Running with Keras
    • +
  • Final part
    • +
  • Final visualization
    • +
  • Fun links
    • @@ -394,7 +399,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 72
    • +
  • 73
  • »
    • diff --git a/doc/pub/week41/html/week41-reveal.html b/doc/pub/week41/html/week41-reveal.html index afd96fa9b..ace121a72 100644 --- a/doc/pub/week41/html/week41-reveal.html +++ b/doc/pub/week41/html/week41-reveal.html @@ -1551,7 +1551,7 @@ while the vector of outputs is \( \boldsymbol{y}^T=[0,1,1,0] \) for the XOR gate

    Setting up the Neural Network

    -We define first our design matrix and the various input vectors. +We define first our design matrix and the various output vectors for the different gates.

    @@ -1613,7 +1613,7 @@ np.random.seed(0) # Defining the neural network n_inputs, n_features = X.shape n_hidden_neurons = 2 -n_categories = 1 +n_categories = 2 n_features = 2 # we make the weights normally distributed using numpy.random.randn @@ -1634,7 +1634,7 @@ predictions = predict(X) print(predictions)

    -Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. +Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above. @@ -1648,6 +1648,73 @@ Not an impressive result. Let us now add the full network with the back-propagat +

    +

    And the same using Scikit-Learn

    + +

    + + +

    # import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+    for j, lmbd in enumerate(lmbd_vals):
+        dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+                            alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+        dnn.fit(X, yXOR)
+        DNN_scikit[i][j] = dnn
+        print("Learning rate  = ", eta)
+        print("Lambda = ", lmbd)
+        print("Accuracy score on data set: ", dnn.score(X, yXOR))
+        print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+    for j in range(len(lmbd_vals)):
+        dnn = DNN_scikit[i][j]
+        test_pred = dnn.predict(X)
+        test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
    +
    + +

    Building neural networks in Tensorflow and Keras

    diff --git a/doc/pub/week41/html/week41-solarized.html b/doc/pub/week41/html/week41-solarized.html index 8a5cdc93d..21d63c636 100644 --- a/doc/pub/week41/html/week41-solarized.html +++ b/doc/pub/week41/html/week41-solarized.html @@ -148,6 +148,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'the-full-network-for-the-various-gates'), + ('And the same using Scikit-Learn', + 2, + None, + 'and-the-same-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1564,7 +1568,7 @@ while the vector of outputs is \( \boldsymbol{y}^T=[0,1,1,0] \) for the XOR gate

    Setting up the Neural Network

    -We define first our design matrix and the various input vectors. +We define first our design matrix and the various output vectors for the different gates.

    @@ -1625,7 +1629,7 @@ np.random.seed(0) # Defining the neural network n_inputs, n_features = X.shape n_hidden_neurons = 2 -n_categories = 1 +n_categories = 2 n_features = 2 # we make the weights normally distributed using numpy.random.randn @@ -1646,7 +1650,7 @@ predictions = predict(X) print(predictions)

    -Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. +Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.











    @@ -1660,6 +1664,72 @@ Not an impressive result. Let us now add the full network with the back-propagat











    +

    And the same using Scikit-Learn

    + +

    + + +

    # import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+    for j, lmbd in enumerate(lmbd_vals):
+        dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+                            alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+        dnn.fit(X, yXOR)
+        DNN_scikit[i][j] = dnn
+        print("Learning rate  = ", eta)
+        print("Lambda = ", lmbd)
+        print("Accuracy score on data set: ", dnn.score(X, yXOR))
+        print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+    for j in range(len(lmbd_vals)):
+        dnn = DNN_scikit[i][j]
+        test_pred = dnn.predict(X)
+        test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
    +

    +









    +

    Building neural networks in Tensorflow and Keras

    diff --git a/doc/pub/week41/html/week41.html b/doc/pub/week41/html/week41.html index 9b52c171f..73e064d97 100644 --- a/doc/pub/week41/html/week41.html +++ b/doc/pub/week41/html/week41.html @@ -153,6 +153,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'the-full-network-for-the-various-gates'), + ('And the same using Scikit-Learn', + 2, + None, + 'and-the-same-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1569,7 +1573,7 @@ while the vector of outputs is \( \boldsymbol{y}^T=[0,1,1,0] \) for the XOR gate

    Setting up the Neural Network

    -We define first our design matrix and the various input vectors. +We define first our design matrix and the various output vectors for the different gates.

    @@ -1630,7 +1634,7 @@ np.random.# Defining the neural network n_inputs, n_features = X.shape n_hidden_neurons = 2 -n_categories = 1 +n_categories = 2 n_features = 2 # we make the weights normally distributed using numpy.random.randn @@ -1651,7 +1655,7 @@ predictions = predict(X) print(predictions)

    -Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. +Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.











    @@ -1665,6 +1669,72 @@ Not an impressive result. Let us now add the full network with the back-propagat











    +

    And the same using Scikit-Learn

    + +

    + + +

    # import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.neural_network import MLPClassifier
+from sklearn.metrics import accuracy_score
+import seaborn as sns
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+# Design matrix
+X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)
+
+# The XOR gate
+yXOR = np.array( [ 0, 1 ,1, 0])
+# The OR gate
+yOR = np.array( [ 0, 1 ,1, 1])
+# The AND gate
+yAND = np.array( [ 0, 0 ,0, 1])
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 2
+n_features = 2
+
+eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+epochs = 100
+
+for i, eta in enumerate(eta_vals):
+    for j, lmbd in enumerate(lmbd_vals):
+        dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+                            alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+        dnn.fit(X, yXOR)
+        DNN_scikit[i][j] = dnn
+        print("Learning rate  = ", eta)
+        print("Lambda = ", lmbd)
+        print("Accuracy score on data set: ", dnn.score(X, yXOR))
+        print()
+
+sns.set()
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+for i in range(len(eta_vals)):
+    for j in range(len(lmbd_vals)):
+        dnn = DNN_scikit[i][j]
+        test_pred = dnn.predict(X)
+        test_accuracy[i][j] = accuracy_score(yXOR, test_pred)
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
    +

    +









    +

    Building neural networks in Tensorflow and Keras

    diff --git a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz index 18192ecbc9e4a1bcd857dbeea7a77e9b7d9f947d..c13aa571d94ee79c4422ec4a23b279d03ff0c38d 100644 GIT binary patch delta 42 xcmcb2M&aHW1$Oyv4hApB^hWkpcE(nArdD?5R(6(FcGgyQwpMobt?V4(Y5*CY3i$v4 delta 42 ycmcb2M&aHW1$Oyv4u(sw(i+)Y*%@2enOfPITiID!*;!lJ*;?7zx3Y7Ds{sH##|$X| diff --git a/doc/pub/week41/ipynb/week41.ipynb b/doc/pub/week41/ipynb/week41.ipynb index af4c3a2bd..6ee3ddec4 100644 --- a/doc/pub/week41/ipynb/week41.ipynb +++ b/doc/pub/week41/ipynb/week41.ipynb @@ -1458,7 +1458,7 @@ "\n", "## Setting up the Neural Network\n", "\n", - "We define first our design matrix and the various input vectors." + "We define first our design matrix and the various output vectors for the different gates." ] }, { @@ -1537,7 +1537,7 @@ "# Defining the neural network\n", "n_inputs, n_features = X.shape\n", "n_hidden_neurons = 2\n", - "n_categories = 1\n", + "n_categories = 2\n", "n_features = 2\n", "\n", "# we make the weights normally distributed using numpy.random.randn\n", @@ -1562,11 +1562,87 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above.\n", + "Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above.\n", "\n", "## The full Network for the Various Gates" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## And the same using Scikit-Learn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\n", + "# import necessary packages\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.neural_network import MLPClassifier\n", + "from sklearn.metrics import accuracy_score\n", + "import seaborn as sns\n", + "\n", + "# ensure the same random numbers appear every time\n", + "np.random.seed(0)\n", + "\n", + "# Design matrix\n", + "X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)\n", + "\n", + "# The XOR gate\n", + "yXOR = np.array( [ 0, 1 ,1, 0])\n", + "# The OR gate\n", + "yOR = np.array( [ 0, 1 ,1, 1])\n", + "# The AND gate\n", + "yAND = np.array( [ 0, 0 ,0, 1])\n", + "\n", + "# Defining the neural network\n", + "n_inputs, n_features = X.shape\n", + "n_hidden_neurons = 2\n", + "n_categories = 2\n", + "n_features = 2\n", + "\n", + "eta_vals = np.logspace(-5, 1, 7)\n", + "lmbd_vals = np.logspace(-5, 1, 7)\n", + "# store models for later use\n", + "DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", + "epochs = 100\n", + "\n", + "for i, eta in enumerate(eta_vals):\n", + " for j, lmbd in enumerate(lmbd_vals):\n", + " dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',\n", + " alpha=lmbd, learning_rate_init=eta, max_iter=epochs)\n", + " dnn.fit(X, yXOR)\n", + " DNN_scikit[i][j] = dnn\n", + " print(\"Learning rate = \", eta)\n", + " print(\"Lambda = \", lmbd)\n", + " print(\"Accuracy score on data set: \", dnn.score(X, yXOR))\n", + " print()\n", + "\n", + "sns.set()\n", + "test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))\n", + "for i in range(len(eta_vals)):\n", + " for j in range(len(lmbd_vals)):\n", + " dnn = DNN_scikit[i][j]\n", + " test_pred = dnn.predict(X)\n", + " test_accuracy[i][j] = accuracy_score(yXOR, test_pred)\n", + "\n", + "fig, ax = plt.subplots(figsize = (10, 10))\n", + "sns.heatmap(test_accuracy, annot=True, ax=ax, cmap=\"viridis\")\n", + "ax.set_title(\"Test Accuracy\")\n", + "ax.set_ylabel(\"$\\eta$\")\n", + "ax.set_xlabel(\"$\\lambda$\")\n", + "plt.show()" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/doc/src/week41/programs/FFpart.py b/doc/src/week41/programs/FFpart.py new file mode 100644 index 000000000..822977841 --- /dev/null +++ b/doc/src/week41/programs/FFpart.py @@ -0,0 +1,67 @@ +""" +Simple code that tests XOR, OR and AND gates with linear regression +""" + +# import necessary packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn import datasets + +def sigmoid(x): + return 1/(1 + np.exp(-x)) + +def feed_forward(X): + # weighted sum of inputs to the hidden layer + z_h = np.matmul(X, hidden_weights) + hidden_bias + # activation in the hidden layer + a_h = sigmoid(z_h) + + # weighted sum of inputs to the output layer + z_o = np.matmul(a_h, output_weights) + output_bias + # softmax output + # axis 0 holds each input and axis 1 the probabilities of each category + probabilities = sigmoid(z_o) + return probabilities + +# we obtain a prediction by taking the class with the highest likelihood +def predict(X): + probabilities = feed_forward(X) + return np.argmax(probabilities, axis=1) + + + +# ensure the same random numbers appear every time +np.random.seed(0) + +# Design matrix +X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64) + +# The XOR gate +yXOR = np.array( [ 0, 1 ,1, 0]) +# The OR gate +yOR = np.array( [ 0, 1 ,1, 1]) +# The AND gate +yAND = np.array( [ 0, 0 ,0, 1]) + +# Defining the neural network +n_inputs, n_features = X.shape +n_hidden_neurons = 2 +n_categories = 2 +n_features = 2 + +# we make the weights normally distributed using numpy.random.randn + +# weights and bias in the hidden layer +hidden_weights = np.random.randn(n_features, n_hidden_neurons) +hidden_bias = np.zeros(n_hidden_neurons) + 0.01 + +# weights and bias in the output layer +output_weights = np.random.randn(n_hidden_neurons, n_categories) +output_bias = np.zeros(n_categories) + 0.01 + +probabilities = feed_forward(X) +print(probabilities) + + +predictions = predict(X) +print(predictions) diff --git a/doc/src/week41/nn.py b/doc/src/week41/programs/FFpart.py~ similarity index 100% rename from doc/src/week41/nn.py rename to doc/src/week41/programs/FFpart.py~ diff --git a/doc/src/week41/programs/ffnn.py b/doc/src/week41/programs/ffnn.py new file mode 100644 index 000000000..c68a4fdae --- /dev/null +++ b/doc/src/week41/programs/ffnn.py @@ -0,0 +1,428 @@ + +# import necessary packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn.neural_network import MLPClassifier +from sklearn.metrics import accuracy_score +import seaborn as sns + +# ensure the same random numbers appear every time +np.random.seed(0) + +# Design matrix +X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64) + +# The XOR gate +yXOR = np.array( [ 0, 1 ,1, 0]) +# The OR gate +yOR = np.array( [ 0, 1 ,1, 1]) +# The AND gate +yAND = np.array( [ 0, 0 ,0, 1]) + +# Defining the neural network +n_inputs, n_features = X.shape +n_hidden_neurons = 2 +n_categories = 2 +n_features = 2 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store models for later use +DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) +epochs = 100 +batch_size = 100 + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic', + alpha=lmbd, learning_rate_init=eta, max_iter=epochs) + dnn.fit(X, yXOR) + DNN_scikit[i][j] = dnn + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on data set: ", dnn.score(X, yXOR)) + print() + +sns.set() +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_scikit[i][j] + test_pred = dnn.predict(X) + test_accuracy[i][j] = accuracy_score(yXOR, test_pred) + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + + + + +""" +# one-hot in numpy +def to_categorical_numpy(integer_vector): + n_inputs = len(integer_vector) + n_categories = np.max(integer_vector) + 1 + onehot_vector = np.zeros((n_inputs, n_categories)) + onehot_vector[range(n_inputs), integer_vector] = 1 + + return onehot_vector + +#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test) +Y_train_onehot, Y_test_onehot = to_categorical_numpy(Y_train), to_categorical_numpy(Y_test) + +def feed_forward_train(X): + # weighted sum of inputs to the hidden layer + z_h = np.matmul(X, hidden_weights) + hidden_bias + # activation in the hidden layer + a_h = sigmoid(z_h) + + # weighted sum of inputs to the output layer + z_o = np.matmul(a_h, output_weights) + output_bias + # softmax output + # axis 0 holds each input and axis 1 the probabilities of each category + exp_term = np.exp(z_o) + probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True) + + # for backpropagation need activations in hidden and output layers + return a_h, probabilities + +def backpropagation(X, Y): + a_h, probabilities = feed_forward_train(X) + + # error in the output layer + error_output = probabilities - Y + # error in the hidden layer + error_hidden = np.matmul(error_output, output_weights.T) * a_h * (1 - a_h) + + # gradients for the output layer + output_weights_gradient = np.matmul(a_h.T, error_output) + output_bias_gradient = np.sum(error_output, axis=0) + + # gradient for the hidden layer + hidden_weights_gradient = np.matmul(X.T, error_hidden) + hidden_bias_gradient = np.sum(error_hidden, axis=0) + + return output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient + +print("Old accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train))) + +eta = 0.01 +lmbd = 0.01 +for i in range(1000): + # calculate gradients + dWo, dBo, dWh, dBh = backpropagation(X_train, Y_train_onehot) + + # regularization term gradients + dWo += lmbd * output_weights + dWh += lmbd * hidden_weights + + # update weights and biases + output_weights -= eta * dWo + output_bias -= eta * dBo + hidden_weights -= eta * dWh + hidden_bias -= eta * dBh + +print("New accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train))) +!ec + +!split +===== Improving performance ===== + +As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image. +In order to obtain a network that does something useful, we will have to do a bit more work. + +The choice of *hyperparameters* such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a *grid-search* is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates $\eta = 10^{-6}, 10^{-5},...,10^{-1}$ with different regularization parameters $\lambda = 10^{-6},...,10^{-0}$. + +Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an *iteration*, and a full training period +going through the entire dataset ($n/M$ batches) an *epoch*. + +If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers. +Andrew Ng goes through some of these considerations in this "video":"https://youtu.be/F1ka6a13S9I". You can find a summary of the video "here":"https://kevinzakka.github.io/2016/09/26/applying-deep-learning/". + +!split +===== Full object-oriented implementation ===== + +It is very natural to think of the network as an object, with specific instances of the network +being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below. + + +!bc pycod +class NeuralNetwork: + def __init__( + self, + X_data, + Y_data, + n_hidden_neurons=50, + n_categories=10, + epochs=10, + batch_size=100, + eta=0.1, + lmbd=0.0): + + self.X_data_full = X_data + self.Y_data_full = Y_data + + self.n_inputs = X_data.shape[0] + self.n_features = X_data.shape[1] + self.n_hidden_neurons = n_hidden_neurons + self.n_categories = n_categories + + self.epochs = epochs + self.batch_size = batch_size + self.iterations = self.n_inputs // self.batch_size + self.eta = eta + self.lmbd = lmbd + + self.create_biases_and_weights() + + def create_biases_and_weights(self): + self.hidden_weights = np.random.randn(self.n_features, self.n_hidden_neurons) + self.hidden_bias = np.zeros(self.n_hidden_neurons) + 0.01 + + self.output_weights = np.random.randn(self.n_hidden_neurons, self.n_categories) + self.output_bias = np.zeros(self.n_categories) + 0.01 + + def feed_forward(self): + # feed-forward for training + self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias + self.a_h = sigmoid(self.z_h) + + self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias + + exp_term = np.exp(self.z_o) + self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True) + + def feed_forward_out(self, X): + # feed-forward for output + z_h = np.matmul(X, self.hidden_weights) + self.hidden_bias + a_h = sigmoid(z_h) + + z_o = np.matmul(a_h, self.output_weights) + self.output_bias + + exp_term = np.exp(z_o) + probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True) + return probabilities + + def backpropagation(self): + error_output = self.probabilities - self.Y_data + error_hidden = np.matmul(error_output, self.output_weights.T) * self.a_h * (1 - self.a_h) + + self.output_weights_gradient = np.matmul(self.a_h.T, error_output) + self.output_bias_gradient = np.sum(error_output, axis=0) + + self.hidden_weights_gradient = np.matmul(self.X_data.T, error_hidden) + self.hidden_bias_gradient = np.sum(error_hidden, axis=0) + + if self.lmbd > 0.0: + self.output_weights_gradient += self.lmbd * self.output_weights + self.hidden_weights_gradient += self.lmbd * self.hidden_weights + + self.output_weights -= self.eta * self.output_weights_gradient + self.output_bias -= self.eta * self.output_bias_gradient + self.hidden_weights -= self.eta * self.hidden_weights_gradient + self.hidden_bias -= self.eta * self.hidden_bias_gradient + + def predict(self, X): + probabilities = self.feed_forward_out(X) + return np.argmax(probabilities, axis=1) + + def predict_probabilities(self, X): + probabilities = self.feed_forward_out(X) + return probabilities + + def train(self): + data_indices = np.arange(self.n_inputs) + + for i in range(self.epochs): + for j in range(self.iterations): + # pick datapoints with replacement + chosen_datapoints = np.random.choice( + data_indices, size=self.batch_size, replace=False + ) + + # minibatch training data + self.X_data = self.X_data_full[chosen_datapoints] + self.Y_data = self.Y_data_full[chosen_datapoints] + + self.feed_forward() + self.backpropagation() +!ec + +!split +===== Evaluate model performance on test data ===== + +To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data. +We measure the performance of the network using the *accuracy* score. +The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of $1$. + +$$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\hat{y}_i = y_i)}{n} ,$$ + +where $I$ is the indicator function, $1$ if $\hat{y}_i = y_i$ and $0$ otherwise. + + +!bc pycod +epochs = 100 +batch_size = 100 + +dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, + n_hidden_neurons=n_hidden_neurons, n_categories=n_categories) +dnn.train() +test_predict = dnn.predict(X_test) + +# accuracy score from scikit library +print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict)) + +# equivalent in numpy +def accuracy_score_numpy(Y_test, Y_pred): + return np.sum(Y_test == Y_pred) / len(Y_test) + +#print("Accuracy score on test set: ", accuracy_score_numpy(Y_test, test_predict)) +!ec + +!split +===== Adjust hyperparameters ===== + +We now perform a grid search to find the optimal hyperparameters for the network. +Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98\%$ ($2\%$ error rate). + +!bc pycod +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store the models for later use +DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +# grid search +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, + n_hidden_neurons=n_hidden_neurons, n_categories=n_categories) + dnn.train() + + DNN_numpy[i][j] = dnn + + test_predict = dnn.predict(X_test) + + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict)) + print() +!ec + +!split +===== Visualization ===== + +!bc pycod +# visual representation of grid search +# uses seaborn heatmap, you can also do this with matplotlib imshow +import seaborn as sns + +sns.set() + +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_numpy[i][j] + + train_pred = dnn.predict(X_train) + test_pred = dnn.predict(X_test) + + train_accuracy[i][j] = accuracy_score(Y_train, train_pred) + test_accuracy[i][j] = accuracy_score(Y_test, test_pred) + + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() +!ec + +!split +===== scikit-learn implementation ===== + +_scikit-learn_ focuses more +on traditional machine learning methods, such as regression, +clustering, decision trees, etc. As such, it has only two types of +neural networks: Multi Layer Perceptron outputting continuous values, +*MPLRegressor*, and Multi Layer Perceptron outputting labels, +*MLPClassifier*. We will see how simple it is to use these classes. + +_scikit-learn_ implements a few improvements from our neural network, +such as early stopping, a varying learning rate, different +optimization methods, etc. We would therefore expect a better +performance overall. + +!bc pycod +from sklearn.neural_network import MLPClassifier +# store models for later use +DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic', + alpha=lmbd, learning_rate_init=eta, max_iter=epochs) + dnn.fit(X_train, Y_train) + + DNN_scikit[i][j] = dnn + + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on test set: ", dnn.score(X_test, Y_test)) + print() +!ec + + +!split +===== Visualization ===== +!bc pycod +# optional +# visual representation of grid search +# uses seaborn heatmap, could probably do this in matplotlib +import seaborn as sns + +sns.set() + +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_scikit[i][j] + + train_pred = dnn.predict(X_train) + test_pred = dnn.predict(X_test) + + train_accuracy[i][j] = accuracy_score(Y_train, train_pred) + test_accuracy[i][j] = accuracy_score(Y_test, test_pred) + + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() +!ec +""" + diff --git a/doc/src/week41/programs/ffnn.py~ b/doc/src/week41/programs/ffnn.py~ new file mode 100644 index 000000000..21deb460b --- /dev/null +++ b/doc/src/week41/programs/ffnn.py~ @@ -0,0 +1,428 @@ + +# import necessary packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn.neural_network import MLPClassifier +from sklearn.metrics import accuracy_score +import seaborn as sns + +# ensure the same random numbers appear every time +np.random.seed(0) + +# Design matrix +X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64) + +# The XOR gate +yXOR = np.array( [ 0, 1 ,1, 0]) +# The OR gate +yOR = np.array( [ 0, 1 ,1, 1]) +# The AND gate +yAND = np.array( [ 0, 0 ,0, 1]) + +# Defining the neural network +n_inputs, n_features = X.shape +n_hidden_neurons = 2 +n_categories = 1 +n_features = 2 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store models for later use +DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) +epochs = 100 +batch_size = 100 + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic', + alpha=lmbd, learning_rate_init=eta, max_iter=epochs) + dnn.fit(X, yXOR) + DNN_scikit[i][j] = dnn + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on data set: ", dnn.score(X, yXOR)) + print() + +sns.set() +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_scikit[i][j] + test_pred = dnn.predict(X) + test_accuracy[i][j] = accuracy_score(yXOR, test_pred) + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + + + + +""" +# one-hot in numpy +def to_categorical_numpy(integer_vector): + n_inputs = len(integer_vector) + n_categories = np.max(integer_vector) + 1 + onehot_vector = np.zeros((n_inputs, n_categories)) + onehot_vector[range(n_inputs), integer_vector] = 1 + + return onehot_vector + +#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test) +Y_train_onehot, Y_test_onehot = to_categorical_numpy(Y_train), to_categorical_numpy(Y_test) + +def feed_forward_train(X): + # weighted sum of inputs to the hidden layer + z_h = np.matmul(X, hidden_weights) + hidden_bias + # activation in the hidden layer + a_h = sigmoid(z_h) + + # weighted sum of inputs to the output layer + z_o = np.matmul(a_h, output_weights) + output_bias + # softmax output + # axis 0 holds each input and axis 1 the probabilities of each category + exp_term = np.exp(z_o) + probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True) + + # for backpropagation need activations in hidden and output layers + return a_h, probabilities + +def backpropagation(X, Y): + a_h, probabilities = feed_forward_train(X) + + # error in the output layer + error_output = probabilities - Y + # error in the hidden layer + error_hidden = np.matmul(error_output, output_weights.T) * a_h * (1 - a_h) + + # gradients for the output layer + output_weights_gradient = np.matmul(a_h.T, error_output) + output_bias_gradient = np.sum(error_output, axis=0) + + # gradient for the hidden layer + hidden_weights_gradient = np.matmul(X.T, error_hidden) + hidden_bias_gradient = np.sum(error_hidden, axis=0) + + return output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient + +print("Old accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train))) + +eta = 0.01 +lmbd = 0.01 +for i in range(1000): + # calculate gradients + dWo, dBo, dWh, dBh = backpropagation(X_train, Y_train_onehot) + + # regularization term gradients + dWo += lmbd * output_weights + dWh += lmbd * hidden_weights + + # update weights and biases + output_weights -= eta * dWo + output_bias -= eta * dBo + hidden_weights -= eta * dWh + hidden_bias -= eta * dBh + +print("New accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train))) +!ec + +!split +===== Improving performance ===== + +As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image. +In order to obtain a network that does something useful, we will have to do a bit more work. + +The choice of *hyperparameters* such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a *grid-search* is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates $\eta = 10^{-6}, 10^{-5},...,10^{-1}$ with different regularization parameters $\lambda = 10^{-6},...,10^{-0}$. + +Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an *iteration*, and a full training period +going through the entire dataset ($n/M$ batches) an *epoch*. + +If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers. +Andrew Ng goes through some of these considerations in this "video":"https://youtu.be/F1ka6a13S9I". You can find a summary of the video "here":"https://kevinzakka.github.io/2016/09/26/applying-deep-learning/". + +!split +===== Full object-oriented implementation ===== + +It is very natural to think of the network as an object, with specific instances of the network +being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below. + + +!bc pycod +class NeuralNetwork: + def __init__( + self, + X_data, + Y_data, + n_hidden_neurons=50, + n_categories=10, + epochs=10, + batch_size=100, + eta=0.1, + lmbd=0.0): + + self.X_data_full = X_data + self.Y_data_full = Y_data + + self.n_inputs = X_data.shape[0] + self.n_features = X_data.shape[1] + self.n_hidden_neurons = n_hidden_neurons + self.n_categories = n_categories + + self.epochs = epochs + self.batch_size = batch_size + self.iterations = self.n_inputs // self.batch_size + self.eta = eta + self.lmbd = lmbd + + self.create_biases_and_weights() + + def create_biases_and_weights(self): + self.hidden_weights = np.random.randn(self.n_features, self.n_hidden_neurons) + self.hidden_bias = np.zeros(self.n_hidden_neurons) + 0.01 + + self.output_weights = np.random.randn(self.n_hidden_neurons, self.n_categories) + self.output_bias = np.zeros(self.n_categories) + 0.01 + + def feed_forward(self): + # feed-forward for training + self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias + self.a_h = sigmoid(self.z_h) + + self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias + + exp_term = np.exp(self.z_o) + self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True) + + def feed_forward_out(self, X): + # feed-forward for output + z_h = np.matmul(X, self.hidden_weights) + self.hidden_bias + a_h = sigmoid(z_h) + + z_o = np.matmul(a_h, self.output_weights) + self.output_bias + + exp_term = np.exp(z_o) + probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True) + return probabilities + + def backpropagation(self): + error_output = self.probabilities - self.Y_data + error_hidden = np.matmul(error_output, self.output_weights.T) * self.a_h * (1 - self.a_h) + + self.output_weights_gradient = np.matmul(self.a_h.T, error_output) + self.output_bias_gradient = np.sum(error_output, axis=0) + + self.hidden_weights_gradient = np.matmul(self.X_data.T, error_hidden) + self.hidden_bias_gradient = np.sum(error_hidden, axis=0) + + if self.lmbd > 0.0: + self.output_weights_gradient += self.lmbd * self.output_weights + self.hidden_weights_gradient += self.lmbd * self.hidden_weights + + self.output_weights -= self.eta * self.output_weights_gradient + self.output_bias -= self.eta * self.output_bias_gradient + self.hidden_weights -= self.eta * self.hidden_weights_gradient + self.hidden_bias -= self.eta * self.hidden_bias_gradient + + def predict(self, X): + probabilities = self.feed_forward_out(X) + return np.argmax(probabilities, axis=1) + + def predict_probabilities(self, X): + probabilities = self.feed_forward_out(X) + return probabilities + + def train(self): + data_indices = np.arange(self.n_inputs) + + for i in range(self.epochs): + for j in range(self.iterations): + # pick datapoints with replacement + chosen_datapoints = np.random.choice( + data_indices, size=self.batch_size, replace=False + ) + + # minibatch training data + self.X_data = self.X_data_full[chosen_datapoints] + self.Y_data = self.Y_data_full[chosen_datapoints] + + self.feed_forward() + self.backpropagation() +!ec + +!split +===== Evaluate model performance on test data ===== + +To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data. +We measure the performance of the network using the *accuracy* score. +The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of $1$. + +$$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\hat{y}_i = y_i)}{n} ,$$ + +where $I$ is the indicator function, $1$ if $\hat{y}_i = y_i$ and $0$ otherwise. + + +!bc pycod +epochs = 100 +batch_size = 100 + +dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, + n_hidden_neurons=n_hidden_neurons, n_categories=n_categories) +dnn.train() +test_predict = dnn.predict(X_test) + +# accuracy score from scikit library +print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict)) + +# equivalent in numpy +def accuracy_score_numpy(Y_test, Y_pred): + return np.sum(Y_test == Y_pred) / len(Y_test) + +#print("Accuracy score on test set: ", accuracy_score_numpy(Y_test, test_predict)) +!ec + +!split +===== Adjust hyperparameters ===== + +We now perform a grid search to find the optimal hyperparameters for the network. +Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around $98\%$ ($2\%$ error rate). + +!bc pycod +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store the models for later use +DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +# grid search +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, + n_hidden_neurons=n_hidden_neurons, n_categories=n_categories) + dnn.train() + + DNN_numpy[i][j] = dnn + + test_predict = dnn.predict(X_test) + + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict)) + print() +!ec + +!split +===== Visualization ===== + +!bc pycod +# visual representation of grid search +# uses seaborn heatmap, you can also do this with matplotlib imshow +import seaborn as sns + +sns.set() + +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_numpy[i][j] + + train_pred = dnn.predict(X_train) + test_pred = dnn.predict(X_test) + + train_accuracy[i][j] = accuracy_score(Y_train, train_pred) + test_accuracy[i][j] = accuracy_score(Y_test, test_pred) + + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() +!ec + +!split +===== scikit-learn implementation ===== + +_scikit-learn_ focuses more +on traditional machine learning methods, such as regression, +clustering, decision trees, etc. As such, it has only two types of +neural networks: Multi Layer Perceptron outputting continuous values, +*MPLRegressor*, and Multi Layer Perceptron outputting labels, +*MLPClassifier*. We will see how simple it is to use these classes. + +_scikit-learn_ implements a few improvements from our neural network, +such as early stopping, a varying learning rate, different +optimization methods, etc. We would therefore expect a better +performance overall. + +!bc pycod +from sklearn.neural_network import MLPClassifier +# store models for later use +DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic', + alpha=lmbd, learning_rate_init=eta, max_iter=epochs) + dnn.fit(X_train, Y_train) + + DNN_scikit[i][j] = dnn + + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on test set: ", dnn.score(X_test, Y_test)) + print() +!ec + + +!split +===== Visualization ===== +!bc pycod +# optional +# visual representation of grid search +# uses seaborn heatmap, could probably do this in matplotlib +import seaborn as sns + +sns.set() + +train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_scikit[i][j] + + train_pred = dnn.predict(X_train) + test_pred = dnn.predict(X_test) + + train_accuracy[i][j] = accuracy_score(Y_train, train_pred) + test_accuracy[i][j] = accuracy_score(Y_test, test_pred) + + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Training Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() +!ec +""" + diff --git a/doc/src/week41/week41.do.txt b/doc/src/week41/week41.do.txt index 442c8902a..def206c25 100644 --- a/doc/src/week41/week41.do.txt +++ b/doc/src/week41/week41.do.txt @@ -1125,7 +1125,7 @@ while the vector of outputs is $\bm{y}^T=[0,1,1,0]$ for the XOR gate, $\bm{y}^T= !split ===== Setting up the Neural Network ===== -We define first our design matrix and the various input vectors. +We define first our design matrix and the various output vectors for the different gates. !bc pycod """ @@ -1184,7 +1184,7 @@ np.random.seed(0) # Defining the neural network n_inputs, n_features = X.shape n_hidden_neurons = 2 -n_categories = 1 +n_categories = 2 n_features = 2 # we make the weights normally distributed using numpy.random.randn @@ -1204,7 +1204,7 @@ print(probabilities) predictions = predict(X) print(predictions) !ec -Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. +Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above. !split ===== The full Network for the Various Gates ===== @@ -1212,6 +1212,71 @@ Not an impressive result. Let us now add the full network with the back-propagat !ec +!split +===== And the same using Scikit-Learn ===== + +!bc pycod + +# import necessary packages +import numpy as np +import matplotlib.pyplot as plt +from sklearn.neural_network import MLPClassifier +from sklearn.metrics import accuracy_score +import seaborn as sns + +# ensure the same random numbers appear every time +np.random.seed(0) + +# Design matrix +X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64) + +# The XOR gate +yXOR = np.array( [ 0, 1 ,1, 0]) +# The OR gate +yOR = np.array( [ 0, 1 ,1, 1]) +# The AND gate +yAND = np.array( [ 0, 0 ,0, 1]) + +# Defining the neural network +n_inputs, n_features = X.shape +n_hidden_neurons = 2 +n_categories = 2 +n_features = 2 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store models for later use +DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) +epochs = 100 + +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic', + alpha=lmbd, learning_rate_init=eta, max_iter=epochs) + dnn.fit(X, yXOR) + DNN_scikit[i][j] = dnn + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on data set: ", dnn.score(X, yXOR)) + print() + +sns.set() +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_scikit[i][j] + test_pred = dnn.predict(X) + test_accuracy[i][j] = accuracy_score(yXOR, test_pred) + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + +!ec + !split ===== Building neural networks in Tensorflow and Keras =====