From a462464a0856d1f5f06ae8f671e16987b54a2b94 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Wed, 13 Oct 2021 16:19:10 +0200 Subject: [PATCH] update --- doc/pub/week41/html/week41-bs.html | 92 ++++++++++--------- doc/pub/week41/html/week41-reveal.html | 77 +++++++++++++++- doc/pub/week41/html/week41-solarized.html | 84 ++++++++++++++++- doc/pub/week41/html/week41.html | 84 ++++++++++++++++- doc/pub/week41/ipynb/ipynb-week41-src.tar.gz | Bin 534110 -> 534110 bytes doc/pub/week41/ipynb/week41.ipynb | 86 ++++++++++++++++- doc/src/week41/nn.py | 67 ++++++++++++++ doc/src/week41/week41.do.txt | 72 ++++++++++++++- 8 files changed, 504 insertions(+), 58 deletions(-) create mode 100644 doc/src/week41/nn.py diff --git a/doc/pub/week41/html/week41-bs.html b/doc/pub/week41/html/week41-bs.html index 64647fae9..f534b7038 100644 --- a/doc/pub/week41/html/week41-bs.html +++ b/doc/pub/week41/html/week41-bs.html @@ -120,6 +120,14 @@ Automatically generated HTML file from DocOnce source 2, None, 'setting-up-the-neural-network'), + ('Then the first Feed Forward pass', + 2, + None, + 'then-the-first-feed-forward-pass'), + ('The full Network for the Various Gates', + 2, + None, + 'the-full-network-for-the-various-gates'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -265,7 +273,7 @@ MathJax.Hub.Config({
  • Example: binary classification problem
  • The Softmax function
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • +
  • Collect and pre-process data
  • Train and test datasets
  • Define model and architecture
  • Layers
    • @@ -287,45 +295,47 @@ MathJax.Hub.Config({
  • The AND and XOR Gates
  • Representing the Data Sets
  • Setting up the Neural Network
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Which activation function should I use?
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • The derivative of the Logistic funtion
    • -
  • The RELU function family
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A very nice website on Neural Networks
    • -
  • A top-down perspective on Neural networks
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Overarching Views, a personal note
    • -
  • From a Spherical Cow to a real one
    • -
  • Convolutional Neural Networks (recognizing images)
    • -
  • Regular NNs don’t scale well to full images
    • -
  • 3D volumes of neurons
    • -
  • Layers used to build CNNs
    • -
  • Transforming images
    • -
  • CNNs in brief
    • -
  • CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
    • -
  • Setting it up
    • -
  • The MNIST dataset again
    • -
  • Strong correlations
    • -
  • Layers of a CNN
    • -
  • Systematic reduction
    • -
  • Prerequisites: Collect and pre-process data
    • -
  • Importing Keras and Tensorflow
    • -
  • Running with Keras
    • -
  • Final part
    • -
  • Final visualization
    • -
  • Fun links
    • +
  • Then the first Feed Forward pass
    • +
  • The full Network for the Various Gates
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Which activation function should I use?
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • The derivative of the Logistic funtion
    • +
  • The RELU function family
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A very nice website on Neural Networks
    • +
  • A top-down perspective on Neural networks
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Overarching Views, a personal note
    • +
  • From a Spherical Cow to a real one
    • +
  • Convolutional Neural Networks (recognizing images)
    • +
  • Regular NNs don’t scale well to full images
    • +
  • 3D volumes of neurons
    • +
  • Layers used to build CNNs
    • +
  • Transforming images
    • +
  • CNNs in brief
    • +
  • CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
    • +
  • Setting it up
    • +
  • The MNIST dataset again
    • +
  • Strong correlations
    • +
  • Layers of a CNN
    • +
  • Systematic reduction
    • +
  • Prerequisites: Collect and pre-process data
    • +
  • Importing Keras and Tensorflow
    • +
  • Running with Keras
    • +
  • Final part
    • +
  • Final visualization
    • +
  • Fun links
    • @@ -384,7 +394,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 70
    • +
  • 72
  • »
    • diff --git a/doc/pub/week41/html/week41-reveal.html b/doc/pub/week41/html/week41-reveal.html index 5c4fc0f15..afd96fa9b 100644 --- a/doc/pub/week41/html/week41-reveal.html +++ b/doc/pub/week41/html/week41-reveal.html @@ -1559,8 +1559,11 @@ We define first our design matrix and the various input vectors. 
    """
 Simple code that tests XOR, OR and AND gates with linear regression
 """
-
+# import necessary packages
 import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
 # Design matrix
 X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
 
@@ -1570,9 +1573,77 @@ yXOR = np.array( [ 0, 0, 1 ,1, 1])
 # The AND gate 
 yAND = np.array( [ 0, 0 ,0, 1])
+
    + -#print(f"The values of theta for the AND gate:{ThetaAND}") -#print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}") + +
    +

    Then the first Feed Forward pass

    + +

    + + +

    def sigmoid(x):
+    return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+    # weighted sum of inputs to the hidden layer
+    z_h = np.matmul(X, hidden_weights) + hidden_bias
+    # activation in the hidden layer
+    a_h = sigmoid(z_h)
+    
+    # weighted sum of inputs to the output layer
+    z_o = np.matmul(a_h, output_weights) + output_bias
+    # softmax output
+    # axis 0 holds each input and axis 1 the probabilities of each category
+    probabilities = sigmoid(z_o)
+    return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+    probabilities = feed_forward(X)
+    return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
+
    +

    +Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. +

    + + +
    +

    The full Network for the Various Gates

    +

    + + +
    diff --git a/doc/pub/week41/html/week41-solarized.html b/doc/pub/week41/html/week41-solarized.html index 93ae88671..8a5cdc93d 100644 --- a/doc/pub/week41/html/week41-solarized.html +++ b/doc/pub/week41/html/week41-solarized.html @@ -140,6 +140,14 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'setting-up-the-neural-network'), + ('Then the first Feed Forward pass', + 2, + None, + 'then-the-first-feed-forward-pass'), + ('The full Network for the Various Gates', + 2, + None, + 'the-full-network-for-the-various-gates'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1564,8 +1572,11 @@ We define first our design matrix and the various input vectors. 
    """
 Simple code that tests XOR, OR and AND gates with linear regression
 """
-
+# import necessary packages
 import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
 # Design matrix
 X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
 
@@ -1575,9 +1586,76 @@ yXOR = np.array( [ 0, 0, 1 ,1, 1])
 # The AND gate 
 yAND = np.array( [ 0, 0 ,0, 1])
+
    +

    +









    -#print(f"The values of theta for the AND gate:{ThetaAND}") -#print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}") +

    Then the first Feed Forward pass

    + +

    + + +

    def sigmoid(x):
+    return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+    # weighted sum of inputs to the hidden layer
+    z_h = np.matmul(X, hidden_weights) + hidden_bias
+    # activation in the hidden layer
+    a_h = sigmoid(z_h)
+    
+    # weighted sum of inputs to the output layer
+    z_o = np.matmul(a_h, output_weights) + output_bias
+    # softmax output
+    # axis 0 holds each input and axis 1 the probabilities of each category
+    probabilities = sigmoid(z_o)
+    return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+    probabilities = feed_forward(X)
+    return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
+
    +

    +Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. + +

    +









    + +

    The full Network for the Various Gates

    +

    + + +











    diff --git a/doc/pub/week41/html/week41.html b/doc/pub/week41/html/week41.html index c152409d4..9b52c171f 100644 --- a/doc/pub/week41/html/week41.html +++ b/doc/pub/week41/html/week41.html @@ -145,6 +145,14 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'setting-up-the-neural-network'), + ('Then the first Feed Forward pass', + 2, + None, + 'then-the-first-feed-forward-pass'), + ('The full Network for the Various Gates', + 2, + None, + 'the-full-network-for-the-various-gates'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1569,8 +1577,11 @@ We define first our design matrix and the various input vectors. 

    """
 Simple code that tests XOR, OR and AND gates with linear regression
 """
-
+# import necessary packages
 import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
+
 # Design matrix
 X = np.array([ [1, 0, 0], [1, 0, 1], [1, 1, 0],[1, 1, 1]],dtype=np.float64)
 
@@ -1580,9 +1591,76 @@ yXOR = np.= np.array( [ 0, 1 ,1, 1])
 # The AND gate 
 yAND = np.array( [ 0, 0 ,0, 1])
+
    +

    +









    -#print(f"The values of theta for the AND gate:{ThetaAND}") -#print(f"The linear regression prediction for the AND gate:{X @ ThetaAND}") +

    Then the first Feed Forward pass

    + +

    + + +

    def sigmoid(x):
+    return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+    # weighted sum of inputs to the hidden layer
+    z_h = np.matmul(X, hidden_weights) + hidden_bias
+    # activation in the hidden layer
+    a_h = sigmoid(z_h)
+    
+    # weighted sum of inputs to the output layer
+    z_o = np.matmul(a_h, output_weights) + output_bias
+    # softmax output
+    # axis 0 holds each input and axis 1 the probabilities of each category
+    probabilities = sigmoid(z_o)
+    return probabilities
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+    probabilities = feed_forward(X)
+    return np.argmax(probabilities, axis=1)
+
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+
+
+# Defining the neural network
+n_inputs, n_features = X.shape
+n_hidden_neurons = 2
+n_categories = 1
+n_features = 2
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
+probabilities = feed_forward(X)
+print(probabilities)
+
+
+predictions = predict(X)
+print(predictions)
+
    +

    +Not an impressive result. Let us now add the full network with the back-propagation algorithm discussed above. + +

    +









    + +

    The full Network for the Various Gates

    +

    + + +











    diff --git a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz index f66080706086d7910e7035c4b30af48719cb7cde..18192ecbc9e4a1bcd857dbeea7a77e9b7d9f947d 100644 GIT binary patch delta 41 vcmcb2M&aHW1vdF^4u(sw8rfRe7+cwxTG^Oe*;rcHSX