From 7639c59ca171218e06f8179bf1acf4b8a9d6f099 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Wed, 13 Oct 2021 22:51:42 +0200 Subject: [PATCH] update --- doc/pub/week41/html/week41-bs.html | 93 +++++----- doc/pub/week41/html/week41-reveal.html | 12 +- doc/pub/week41/html/week41-solarized.html | 19 +- doc/pub/week41/html/week41.html | 19 +- doc/pub/week41/ipynb/ipynb-week41-src.tar.gz | Bin 534110 -> 534110 bytes doc/pub/week41/ipynb/week41.ipynb | 9 +- doc/src/week41/programs/nnown.py | 172 +++++++++++++++++++ doc/src/week41/week41.do.txt | 8 +- 8 files changed, 225 insertions(+), 107 deletions(-) create mode 100644 doc/src/week41/programs/nnown.py diff --git a/doc/pub/week41/html/week41-bs.html b/doc/pub/week41/html/week41-bs.html index ce099accf..75e897c57 100644 --- a/doc/pub/week41/html/week41-bs.html +++ b/doc/pub/week41/html/week41-bs.html @@ -124,14 +124,10 @@ Automatically generated HTML file from DocOnce source 2, None, 'then-the-first-feed-forward-pass'), - ('The full Network for the Various Gates', + ('The Code using Scikit-Learn', 2, None, - 'the-full-network-for-the-various-gates'), - ('And the same using Scikit-Learn', - 2, - None, - 'and-the-same-using-scikit-learn'), + 'the-code-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -277,7 +273,7 @@ MathJax.Hub.Config({
  • Example: binary classification problem
  • The Softmax function
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • +
  • Collect and pre-process data
  • Train and test datasets
  • Define model and architecture
  • Layers
    • @@ -300,47 +296,46 @@ MathJax.Hub.Config({
  • Representing the Data Sets
  • Setting up the Neural Network
  • Then the first Feed Forward pass
    • -
  • The full Network for the Various Gates
    • -
  • And the same using Scikit-Learn
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Which activation function should I use?
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • The derivative of the Logistic funtion
    • -
  • The RELU function family
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A very nice website on Neural Networks
    • -
  • A top-down perspective on Neural networks
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Overarching Views, a personal note
    • -
  • From a Spherical Cow to a real one
    • -
  • Convolutional Neural Networks (recognizing images)
    • -
  • Regular NNs don’t scale well to full images
    • -
  • 3D volumes of neurons
    • -
  • Layers used to build CNNs
    • -
  • Transforming images
    • -
  • CNNs in brief
    • -
  • CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
    • -
  • Setting it up
    • -
  • The MNIST dataset again
    • -
  • Strong correlations
    • -
  • Layers of a CNN
    • -
  • Systematic reduction
    • -
  • Prerequisites: Collect and pre-process data
    • -
  • Importing Keras and Tensorflow
    • -
  • Running with Keras
    • -
  • Final part
    • -
  • Final visualization
    • -
  • Fun links
    • +
  • The Code using Scikit-Learn
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Which activation function should I use?
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • The derivative of the Logistic funtion
    • +
  • The RELU function family
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A very nice website on Neural Networks
    • +
  • A top-down perspective on Neural networks
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Overarching Views, a personal note
    • +
  • From a Spherical Cow to a real one
    • +
  • Convolutional Neural Networks (recognizing images)
    • +
  • Regular NNs don’t scale well to full images
    • +
  • 3D volumes of neurons
    • +
  • Layers used to build CNNs
    • +
  • Transforming images
    • +
  • CNNs in brief
    • +
  • CNNs in more detail, building convolutional neural networks in Tensorflow and Keras
    • +
  • Setting it up
    • +
  • The MNIST dataset again
    • +
  • Strong correlations
    • +
  • Layers of a CNN
    • +
  • Systematic reduction
    • +
  • Prerequisites: Collect and pre-process data
    • +
  • Importing Keras and Tensorflow
    • +
  • Running with Keras
    • +
  • Final part
    • +
  • Final visualization
    • +
  • Fun links
    • @@ -399,7 +394,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 73
    • +
  • 72
  • »
    • diff --git a/doc/pub/week41/html/week41-reveal.html b/doc/pub/week41/html/week41-reveal.html index ace121a72..9571a5c2a 100644 --- a/doc/pub/week41/html/week41-reveal.html +++ b/doc/pub/week41/html/week41-reveal.html @@ -1639,17 +1639,7 @@ Not an impressive result, but this was our first forward pass with randomly assi
    -

    The full Network for the Various Gates

    -

    - - -

    
-
    -
    - - -
    -

    And the same using Scikit-Learn

    +

    The Code using Scikit-Learn

    diff --git a/doc/pub/week41/html/week41-solarized.html b/doc/pub/week41/html/week41-solarized.html index 21d63c636..34c3c70a3 100644 --- a/doc/pub/week41/html/week41-solarized.html +++ b/doc/pub/week41/html/week41-solarized.html @@ -144,14 +144,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'then-the-first-feed-forward-pass'), - ('The full Network for the Various Gates', + ('The Code using Scikit-Learn', 2, None, - 'the-full-network-for-the-various-gates'), - ('And the same using Scikit-Learn', - 2, - None, - 'and-the-same-using-scikit-learn'), + 'the-code-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1655,16 +1651,7 @@ Not an impressive result, but this was our first forward pass with randomly assi











    -

    The full Network for the Various Gates

    -

    - - -

    
-
    -

    -









    - -

    And the same using Scikit-Learn

    +

    The Code using Scikit-Learn

    diff --git a/doc/pub/week41/html/week41.html b/doc/pub/week41/html/week41.html index 73e064d97..784fd1ca8 100644 --- a/doc/pub/week41/html/week41.html +++ b/doc/pub/week41/html/week41.html @@ -149,14 +149,10 @@ div { text-align: justify; text-justify: inter-word; } 2, None, 'then-the-first-feed-forward-pass'), - ('The full Network for the Various Gates', + ('The Code using Scikit-Learn', 2, None, - 'the-full-network-for-the-various-gates'), - ('And the same using Scikit-Learn', - 2, - None, - 'and-the-same-using-scikit-learn'), + 'the-code-using-scikit-learn'), ('Building neural networks in Tensorflow and Keras', 2, None, @@ -1660,16 +1656,7 @@ Not an impressive result, but this was our first forward pass with randomly assi











    -

    The full Network for the Various Gates

    -

    - - -

    
-
    -

    -









    - -

    And the same using Scikit-Learn

    +

    The Code using Scikit-Learn

    diff --git a/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz b/doc/pub/week41/ipynb/ipynb-week41-src.tar.gz index c13aa571d94ee79c4422ec4a23b279d03ff0c38d..252d351a17a1055314c27743ed3f6fa8ff8d2a57 100644 GIT binary patch delta 41 vcmcb2M&aHW1vdF^4u*|xjcl!KjIC@;t!&J#Y%Hy8tgUQ75%#TY91&^&6%7jY delta 41 vcmcb2M&aHW1vdF^4hApBMz&Tq##T0_RyO8VHkMX4)>by42>VtxjtDgX2x 0.0: + self.output_weights_gradient += self.lmbd * self.output_weights + self.hidden_weights_gradient += self.lmbd * self.hidden_weights + + self.output_weights -= self.eta * self.output_weights_gradient + self.output_bias -= self.eta * self.output_bias_gradient + self.hidden_weights -= self.eta * self.hidden_weights_gradient + self.hidden_bias -= self.eta * self.hidden_bias_gradient + + def predict(self, X): + probabilities = self.feed_forward_out(X) + return np.argmax(probabilities, axis=1) + + def predict_probabilities(self, X): + probabilities = self.feed_forward_out(X) + return probabilities + + def train(self): + data_indices = np.arange(self.n_inputs) + + for i in range(self.epochs): + for j in range(self.iterations): + # pick datapoints with replacement + chosen_datapoints = np.random.choice( + data_indices, size=self.batch_size, replace=False + ) + + # minibatch training data + self.X_data = self.X_data_full[chosen_datapoints] + self.Y_data = self.Y_data_full[chosen_datapoints] + + self.feed_forward() + self.backpropagation() + +epochs = 100 +batch_size = 100 + +eta_vals = np.logspace(-5, 1, 7) +lmbd_vals = np.logspace(-5, 1, 7) +# store the models for later use +DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object) + +# grid search +for i, eta in enumerate(eta_vals): + for j, lmbd in enumerate(lmbd_vals): + dnn = NeuralNetwork(X, yXOR, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size, + n_hidden_neurons=n_hidden_neurons, n_categories=n_categories) + dnn.train() + DNN_numpy[i][j] = dnn + test_predict = dnn.predict(X) + print("Learning rate = ", eta) + print("Lambda = ", lmbd) + print("Accuracy score on test set: ", accuracy_score(yXOR, test_predict)) + print() + +sns.set() +test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals))) + +for i in range(len(eta_vals)): + for j in range(len(lmbd_vals)): + dnn = DNN_numpy[i][j] + test_pred = dnn.predict(X) + test_accuracy[i][j] = accuracy_score(yXOR, test_pred) + +fig, ax = plt.subplots(figsize = (10, 10)) +sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis") +ax.set_title("Test Accuracy") +ax.set_ylabel("$\eta$") +ax.set_xlabel("$\lambda$") +plt.show() + + + diff --git a/doc/src/week41/week41.do.txt b/doc/src/week41/week41.do.txt index def206c25..637c5b33a 100644 --- a/doc/src/week41/week41.do.txt +++ b/doc/src/week41/week41.do.txt @@ -1207,13 +1207,7 @@ print(predictions) Not an impressive result, but this was our first forward pass with randomly assigned weights. Let us now add the full network with the back-propagation algorithm discussed above. !split -===== The full Network for the Various Gates ===== -!bc pycod - -!ec - -!split -===== And the same using Scikit-Learn ===== +===== The Code using Scikit-Learn ===== !bc pycod