From ad9964dcac5da719d17b27fbbfd9a2217eeb6899 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Sun, 13 Oct 2024 17:09:49 +0200 Subject: [PATCH] update --- doc/LectureNotes/_toc.yml | 2 + doc/pub/week42/html/._week42-bs000.html | 163 ++--- doc/pub/week42/html/._week42-bs001.html | 163 ++--- doc/pub/week42/html/._week42-bs002.html | 163 ++--- doc/pub/week42/html/._week42-bs003.html | 163 ++--- doc/pub/week42/html/._week42-bs004.html | 163 ++--- doc/pub/week42/html/._week42-bs005.html | 163 ++--- doc/pub/week42/html/._week42-bs006.html | 163 ++--- doc/pub/week42/html/._week42-bs007.html | 163 ++--- doc/pub/week42/html/._week42-bs008.html | 163 ++--- doc/pub/week42/html/._week42-bs009.html | 163 ++--- doc/pub/week42/html/._week42-bs010.html | 163 ++--- doc/pub/week42/html/._week42-bs011.html | 163 ++--- doc/pub/week42/html/._week42-bs012.html | 163 ++--- doc/pub/week42/html/._week42-bs013.html | 163 ++--- doc/pub/week42/html/._week42-bs014.html | 163 ++--- doc/pub/week42/html/._week42-bs015.html | 163 ++--- doc/pub/week42/html/._week42-bs016.html | 163 ++--- doc/pub/week42/html/._week42-bs017.html | 163 ++--- doc/pub/week42/html/._week42-bs018.html | 163 ++--- doc/pub/week42/html/._week42-bs019.html | 163 ++--- doc/pub/week42/html/._week42-bs020.html | 163 ++--- doc/pub/week42/html/._week42-bs021.html | 163 ++--- doc/pub/week42/html/._week42-bs022.html | 163 ++--- doc/pub/week42/html/._week42-bs023.html | 163 ++--- doc/pub/week42/html/._week42-bs024.html | 163 ++--- doc/pub/week42/html/._week42-bs025.html | 163 ++--- doc/pub/week42/html/._week42-bs026.html | 163 ++--- doc/pub/week42/html/._week42-bs027.html | 163 ++--- doc/pub/week42/html/._week42-bs028.html | 167 ++--- doc/pub/week42/html/._week42-bs029.html | 163 ++--- doc/pub/week42/html/._week42-bs030.html | 163 ++--- doc/pub/week42/html/._week42-bs031.html | 185 +++-- doc/pub/week42/html/._week42-bs032.html | 180 ++--- doc/pub/week42/html/._week42-bs033.html | 180 ++--- doc/pub/week42/html/._week42-bs034.html | 188 +++--- doc/pub/week42/html/._week42-bs035.html | 189 +++--- doc/pub/week42/html/._week42-bs036.html | 181 ++--- doc/pub/week42/html/._week42-bs037.html | 189 +++--- doc/pub/week42/html/._week42-bs038.html | 191 +++--- doc/pub/week42/html/._week42-bs039.html | 183 +++-- doc/pub/week42/html/._week42-bs040.html | 191 +++--- doc/pub/week42/html/._week42-bs041.html | 185 ++--- doc/pub/week42/html/._week42-bs042.html | 178 ++--- doc/pub/week42/html/._week42-bs043.html | 175 ++--- doc/pub/week42/html/._week42-bs044.html | 191 +++--- doc/pub/week42/html/._week42-bs045.html | 194 +++--- doc/pub/week42/html/._week42-bs046.html | 275 +++----- doc/pub/week42/html/._week42-bs047.html | 282 +++++--- doc/pub/week42/html/._week42-bs048.html | 197 +++--- doc/pub/week42/html/._week42-bs049.html | 191 +++--- doc/pub/week42/html/._week42-bs050.html | 186 ++--- doc/pub/week42/html/._week42-bs051.html | 187 +++--- doc/pub/week42/html/._week42-bs052.html | 188 +++--- doc/pub/week42/html/._week42-bs053.html | 191 +++--- doc/pub/week42/html/._week42-bs054.html | 192 +++--- doc/pub/week42/html/._week42-bs055.html | 185 ++--- doc/pub/week42/html/._week42-bs056.html | 182 ++--- doc/pub/week42/html/._week42-bs057.html | 183 ++--- doc/pub/week42/html/._week42-bs058.html | 185 ++--- doc/pub/week42/html/._week42-bs059.html | 189 +++--- doc/pub/week42/html/._week42-bs060.html | 187 +++--- doc/pub/week42/html/._week42-bs061.html | 181 ++--- doc/pub/week42/html/._week42-bs062.html | 191 +++--- doc/pub/week42/html/._week42-bs063.html | 200 +++--- doc/pub/week42/html/._week42-bs064.html | 191 +++--- doc/pub/week42/html/._week42-bs065.html | 178 ++--- doc/pub/week42/html/._week42-bs066.html | 170 ++--- doc/pub/week42/html/._week42-bs067.html | 169 ++--- doc/pub/week42/html/._week42-bs068.html | 235 +++---- doc/pub/week42/html/._week42-bs069.html | 248 ++++--- doc/pub/week42/html/._week42-bs070.html | 230 +++---- doc/pub/week42/html/._week42-bs071.html | 229 ++++--- doc/pub/week42/html/._week42-bs072.html | 207 +++--- doc/pub/week42/html/._week42-bs073.html | 192 +++--- doc/pub/week42/html/._week42-bs074.html | 284 +++----- doc/pub/week42/html/._week42-bs075.html | 273 +++++--- doc/pub/week42/html/._week42-bs076.html | 243 ++++--- doc/pub/week42/html/._week42-bs077.html | 222 +++--- doc/pub/week42/html/._week42-bs078.html | 232 ++++--- doc/pub/week42/html/._week42-bs079.html | 228 ++++--- doc/pub/week42/html/._week42-bs080.html | 264 +++----- doc/pub/week42/html/._week42-bs081.html | 263 +++++--- doc/pub/week42/html/._week42-bs082.html | 206 +++--- doc/pub/week42/html/._week42-bs083.html | 212 +++--- doc/pub/week42/html/._week42-bs084.html | 302 +++------ doc/pub/week42/html/._week42-bs085.html | 289 +++++--- doc/pub/week42/html/._week42-bs086.html | 295 +++----- doc/pub/week42/html/._week42-bs087.html | 281 +++++--- doc/pub/week42/html/._week42-bs088.html | 208 +++--- doc/pub/week42/html/._week42-bs089.html | 216 +++--- doc/pub/week42/html/._week42-bs090.html | 221 +++--- doc/pub/week42/html/._week42-bs091.html | 223 +++--- doc/pub/week42/html/._week42-bs092.html | 226 ++++--- doc/pub/week42/html/week42-bs.html | 163 ++--- doc/pub/week42/html/week42-reveal.html | 14 +- doc/pub/week42/html/week42-solarized.html | 23 +- doc/pub/week42/html/week42.html | 23 +- doc/pub/week42/ipynb/ipynb-week42-src.tar.gz | Bin 960220 -> 978624 bytes doc/pub/week42/ipynb/week42.ipynb | 672 ++++++++++--------- doc/src/week42/exercisesweek42.ipynb | 467 ------------- doc/src/week42/figures/nn2.pdf | Bin 0 -> 35328 bytes doc/src/week42/figures/nn2.png | Bin 0 -> 41803 bytes doc/src/week42/figures/simplenn3.pdf | Bin 0 -> 30861 bytes doc/src/week42/figures/structure.pdf | Bin 0 -> 60903 bytes doc/src/week42/figures/structure.png | Bin 0 -> 28827 bytes doc/src/week42/week42.do.txt | 13 +- 107 files changed, 9890 insertions(+), 9633 deletions(-) delete mode 100644 doc/src/week42/exercisesweek42.ipynb create mode 100644 doc/src/week42/figures/nn2.pdf create mode 100644 doc/src/week42/figures/nn2.png create mode 100644 doc/src/week42/figures/simplenn3.pdf create mode 100644 doc/src/week42/figures/structure.pdf create mode 100644 doc/src/week42/figures/structure.png diff --git a/doc/LectureNotes/_toc.yml b/doc/LectureNotes/_toc.yml index a3165d0ef..eee8e4387 100644 --- a/doc/LectureNotes/_toc.yml +++ b/doc/LectureNotes/_toc.yml @@ -56,6 +56,8 @@ parts: - file: week40.ipynb - file: exercisesweek41.ipynb - file: week41.ipynb + - file: exercisesweek42.ipynb + - file: week42.ipynb - caption: Projects numbered: false chapters: diff --git a/doc/pub/week42/html/._week42-bs000.html b/doc/pub/week42/html/._week42-bs000.html index dddce8a8e..c33716b7d 100644 --- a/doc/pub/week42/html/._week42-bs000.html +++ b/doc/pub/week42/html/._week42-bs000.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -500,7 +507,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs001.html b/doc/pub/week42/html/._week42-bs001.html index afd7261b3..be51bb83a 100644 --- a/doc/pub/week42/html/._week42-bs001.html +++ b/doc/pub/week42/html/._week42-bs001.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -501,7 +508,7 @@ MathJax.Hub.Config({
  • 10
  • 11
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs002.html b/doc/pub/week42/html/._week42-bs002.html index 9976d9706..ecd3c51c9 100644 --- a/doc/pub/week42/html/._week42-bs002.html +++ b/doc/pub/week42/html/._week42-bs002.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -488,7 +495,7 @@ MathJax.Hub.Config({
  • 11
  • 12
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs003.html b/doc/pub/week42/html/._week42-bs003.html index a3e1cd48d..55c376349 100644 --- a/doc/pub/week42/html/._week42-bs003.html +++ b/doc/pub/week42/html/._week42-bs003.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -488,7 +495,7 @@ hidden nodes but only one output node.
  • 12
  • 13
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs004.html b/doc/pub/week42/html/._week42-bs004.html index dc4af5349..c0e0169e6 100644 --- a/doc/pub/week42/html/._week42-bs004.html +++ b/doc/pub/week42/html/._week42-bs004.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -489,7 +496,7 @@ MathJax.Hub.Config({
  • 13
  • 14
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs005.html b/doc/pub/week42/html/._week42-bs005.html index dcab679d1..dedd4b8b1 100644 --- a/doc/pub/week42/html/._week42-bs005.html +++ b/doc/pub/week42/html/._week42-bs005.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -488,7 +495,7 @@ MathJax.Hub.Config({
  • 14
  • 15
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs006.html b/doc/pub/week42/html/._week42-bs006.html index 95d36c77f..134e61953 100644 --- a/doc/pub/week42/html/._week42-bs006.html +++ b/doc/pub/week42/html/._week42-bs006.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -484,7 +491,7 @@ MathJax.Hub.Config({
  • 15
  • 16
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs007.html b/doc/pub/week42/html/._week42-bs007.html index d0e16119b..aadf5059d 100644 --- a/doc/pub/week42/html/._week42-bs007.html +++ b/doc/pub/week42/html/._week42-bs007.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -503,7 +510,7 @@ $$
  • 16
  • 17
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs008.html b/doc/pub/week42/html/._week42-bs008.html index 0f2c12c8d..200298551 100644 --- a/doc/pub/week42/html/._week42-bs008.html +++ b/doc/pub/week42/html/._week42-bs008.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -488,7 +495,7 @@ MathJax.Hub.Config({
  • 17
  • 18
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs009.html b/doc/pub/week42/html/._week42-bs009.html index e676d244c..7dc90011b 100644 --- a/doc/pub/week42/html/._week42-bs009.html +++ b/doc/pub/week42/html/._week42-bs009.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -509,7 +516,7 @@ $$
  • 18
  • 19
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs010.html b/doc/pub/week42/html/._week42-bs010.html index 0409da91c..e1512f11d 100644 --- a/doc/pub/week42/html/._week42-bs010.html +++ b/doc/pub/week42/html/._week42-bs010.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -506,7 +513,7 @@ $$
  • 19
  • 20
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs011.html b/doc/pub/week42/html/._week42-bs011.html index 1f3594913..72f64635e 100644 --- a/doc/pub/week42/html/._week42-bs011.html +++ b/doc/pub/week42/html/._week42-bs011.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -490,7 +497,7 @@ MathJax.Hub.Config({
  • 20
  • 21
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs012.html b/doc/pub/week42/html/._week42-bs012.html index 834a84a1f..c419ade16 100644 --- a/doc/pub/week42/html/._week42-bs012.html +++ b/doc/pub/week42/html/._week42-bs012.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -504,7 +511,7 @@ $$
  • 21
  • 22
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs013.html b/doc/pub/week42/html/._week42-bs013.html index a6f1b0282..1b76b1762 100644 --- a/doc/pub/week42/html/._week42-bs013.html +++ b/doc/pub/week42/html/._week42-bs013.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -497,7 +504,7 @@ is referenced as the exploding gradient problem.
  • 22
  • 23
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs014.html b/doc/pub/week42/html/._week42-bs014.html index ec140681c..ef02a49d8 100644 --- a/doc/pub/week42/html/._week42-bs014.html +++ b/doc/pub/week42/html/._week42-bs014.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -501,7 +508,7 @@ $$
  • 23
  • 24
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs015.html b/doc/pub/week42/html/._week42-bs015.html index 0f433440e..07663628e 100644 --- a/doc/pub/week42/html/._week42-bs015.html +++ b/doc/pub/week42/html/._week42-bs015.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -578,7 +585,7 @@ eta = 0.124
  • 25
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs016.html b/doc/pub/week42/html/._week42-bs016.html index ddd2039f8..44d62dfd1 100644 --- a/doc/pub/week42/html/._week42-bs016.html +++ b/doc/pub/week42/html/._week42-bs016.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -502,7 +509,7 @@ $$
  • 25
  • 26
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs017.html b/doc/pub/week42/html/._week42-bs017.html index 6284d7bb7..b2f87e56e 100644 --- a/doc/pub/week42/html/._week42-bs017.html +++ b/doc/pub/week42/html/._week42-bs017.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -490,7 +497,7 @@ MathJax.Hub.Config({
  • 26
  • 27
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs018.html b/doc/pub/week42/html/._week42-bs018.html index 2449657cd..03567cd6e 100644 --- a/doc/pub/week42/html/._week42-bs018.html +++ b/doc/pub/week42/html/._week42-bs018.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -497,7 +504,7 @@ $$
  • 27
  • 28
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs019.html b/doc/pub/week42/html/._week42-bs019.html index 0eac6400e..c2bc1a2d8 100644 --- a/doc/pub/week42/html/._week42-bs019.html +++ b/doc/pub/week42/html/._week42-bs019.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -497,7 +504,7 @@ $$
  • 28
  • 29
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs020.html b/doc/pub/week42/html/._week42-bs020.html index a84ac648e..c9846baec 100644 --- a/doc/pub/week42/html/._week42-bs020.html +++ b/doc/pub/week42/html/._week42-bs020.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -495,7 +502,7 @@ $$
  • 29
  • 30
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs021.html b/doc/pub/week42/html/._week42-bs021.html index a69de5a4e..d3ab874da 100644 --- a/doc/pub/week42/html/._week42-bs021.html +++ b/doc/pub/week42/html/._week42-bs021.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -506,7 +513,7 @@ $$
  • 30
  • 31
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs022.html b/doc/pub/week42/html/._week42-bs022.html index 55e51689c..5d2782ec4 100644 --- a/doc/pub/week42/html/._week42-bs022.html +++ b/doc/pub/week42/html/._week42-bs022.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -501,7 +508,7 @@ $$
  • 31
  • 32
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs023.html b/doc/pub/week42/html/._week42-bs023.html index f9ef2d351..b54fdcd75 100644 --- a/doc/pub/week42/html/._week42-bs023.html +++ b/doc/pub/week42/html/._week42-bs023.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -499,7 +506,7 @@ $$
  • 32
  • 33
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs024.html b/doc/pub/week42/html/._week42-bs024.html index bd5af2a55..af2c49271 100644 --- a/doc/pub/week42/html/._week42-bs024.html +++ b/doc/pub/week42/html/._week42-bs024.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -500,7 +507,7 @@ $$
  • 33
  • 34
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs025.html b/doc/pub/week42/html/._week42-bs025.html index a7f4b90a2..f0a2e0401 100644 --- a/doc/pub/week42/html/._week42-bs025.html +++ b/doc/pub/week42/html/._week42-bs025.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -496,7 +503,7 @@ $$
  • 34
  • 35
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs026.html b/doc/pub/week42/html/._week42-bs026.html index ecded6b7e..79e07a6cc 100644 --- a/doc/pub/week42/html/._week42-bs026.html +++ b/doc/pub/week42/html/._week42-bs026.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -508,7 +515,7 @@ $$
  • 35
  • 36
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs027.html b/doc/pub/week42/html/._week42-bs027.html index d080537fa..87cc9972a 100644 --- a/doc/pub/week42/html/._week42-bs027.html +++ b/doc/pub/week42/html/._week42-bs027.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -502,7 +509,7 @@ all inputs \( \boldsymbol{x} \) are given by \( \boldsymbol{\tilde{y}}_i \).
  • 36
  • 37
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs028.html b/doc/pub/week42/html/._week42-bs028.html index 9ff9b2a49..9feba3653 100644 --- a/doc/pub/week42/html/._week42-bs028.html +++ b/doc/pub/week42/html/._week42-bs028.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,11 +464,11 @@ MathJax.Hub.Config({

     

     

     

    -

    Layout of a neural network with three hidden layers

    +

    Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))



    -

    +



    @@ -490,7 +497,7 @@ MathJax.Hub.Config({
  • 37
  • 38
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs029.html b/doc/pub/week42/html/._week42-bs029.html index 494b7a613..9886f5380 100644 --- a/doc/pub/week42/html/._week42-bs029.html +++ b/doc/pub/week42/html/._week42-bs029.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -507,7 +514,7 @@ $$
  • 38
  • 39
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs030.html b/doc/pub/week42/html/._week42-bs030.html index eee5502ff..26a71dc2b 100644 --- a/doc/pub/week42/html/._week42-bs030.html +++ b/doc/pub/week42/html/._week42-bs030.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -496,7 +503,7 @@ $$
  • 39
  • 40
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs031.html b/doc/pub/week42/html/._week42-bs031.html index cc7596e1e..38241757c 100644 --- a/doc/pub/week42/html/._week42-bs031.html +++ b/doc/pub/week42/html/._week42-bs031.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,23 +464,13 @@ MathJax.Hub.Config({

     

     

     

    -

    Derivatives and the chain rule

    - -

    From the definition of the input variable to the activation function, that is \( z_j^l \) we have

    -$$ -\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1}, -$$ - -

    and

    -$$ -\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l. -$$ - -

    With our definition of the activation function we have that (note that this function depends only on \( z_j^l \))

    -$$ -\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=\sigma(z_j^l)(1-\sigma(z_j^l)). -$$ +

    Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)

    +

    +
    +

    +
    +

    @@ -500,7 +497,7 @@ $$

  • 40
  • 41
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs032.html b/doc/pub/week42/html/._week42-bs032.html index 9b2b119e0..8d1802082 100644 --- a/doc/pub/week42/html/._week42-bs032.html +++ b/doc/pub/week42/html/._week42-bs032.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,24 +464,21 @@ MathJax.Hub.Config({

     

     

     

    -

    Derivative of the cost function

    +

    Derivatives and the chain rule

    -

    With these definitions we can now compute the derivative of the cost function in terms of the weights.

    - -

    Let us specialize to the output layer \( l=L \). Our cost function is

    +

    From the definition of the input variable to the activation function, that is \( z_j^l \) we have

    $$ -{\cal C}(\boldsymbol{\Theta}^L) = \frac{1}{2}\sum_{i=1}^n\left(y_i - \tilde{y}_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - y_i\right)^2, +\frac{\partial z_j^l}{\partial w_{ij}^l} = a_i^{l-1}, $$ -

    The derivative of this function with respect to the weights is

    - +

    and

    $$ -\frac{\partial{\cal C}(\boldsymbol{\Theta}^L)}{\partial w_{ij}^L} = \left(a_j^L - y_j\right)\frac{\partial a_j^L}{\partial w_{ij}^{L}}, +\frac{\partial z_j^l}{\partial a_i^{l-1}} = w_{ji}^l. $$ -

    The last partial derivative can easily be computed and reads (by applying the chain rule)

    +

    With our definition of the activation function we have that (note that this function depends only on \( z_j^l \))

    $$ -\frac{\partial a_j^L}{\partial w_{ij}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{ij}^{L}}=a_j^L(1-a_j^L)a_i^{L-1}. +\frac{\partial a_j^l}{\partial z_j^{l}} = a_j^l(1-a_j^l)=\sigma(z_j^l)(1-\sigma(z_j^l)). $$ @@ -503,7 +507,7 @@ $$
  • 41
  • 42
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs033.html b/doc/pub/week42/html/._week42-bs033.html index aa2a0fbbb..ff1904224 100644 --- a/doc/pub/week42/html/._week42-bs033.html +++ b/doc/pub/week42/html/._week42-bs033.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,21 +464,24 @@ MathJax.Hub.Config({

     

     

     

    -

    The back propagation equations for a neural network

    +

    Derivative of the cost function

    -

    We have thus

    +

    With these definitions we can now compute the derivative of the cost function in terms of the weights.

    + +

    Let us specialize to the output layer \( l=L \). Our cost function is

    $$ -\frac{\partial{\cal C}((\boldsymbol{\Theta}^L)}{\partial w_{ij}^L} = \left(a_j^L - y_j\right)a_j^L(1-a_j^L)a_i^{L-1}, +{\cal C}(\boldsymbol{\Theta}^L) = \frac{1}{2}\sum_{i=1}^n\left(y_i - \tilde{y}_i\right)^2=\frac{1}{2}\sum_{i=1}^n\left(a_i^L - y_i\right)^2, $$ -

    Defining

    +

    The derivative of this function with respect to the weights is

    + $$ -\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - y_j\right) = \sigma'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}, +\frac{\partial{\cal C}(\boldsymbol{\Theta}^L)}{\partial w_{ij}^L} = \left(a_j^L - y_j\right)\frac{\partial a_j^L}{\partial w_{ij}^{L}}, $$ -

    and using the Hadamard product of two vectors we can write this as

    +

    The last partial derivative can easily be computed and reads (by applying the chain rule)

    $$ -\boldsymbol{\delta}^L = \sigma'(\boldsymbol{z}^L)\circ\frac{\partial {\cal C}}{\partial (\boldsymbol{a}^L)}. +\frac{\partial a_j^L}{\partial w_{ij}^{L}} = \frac{\partial a_j^L}{\partial z_{j}^{L}}\frac{\partial z_j^L}{\partial w_{ij}^{L}}=a_j^L(1-a_j^L)a_i^{L-1}. $$ @@ -500,7 +510,7 @@ $$
  • 42
  • 43
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs034.html b/doc/pub/week42/html/._week42-bs034.html index 61ee18d9c..4b22fd5c8 100644 --- a/doc/pub/week42/html/._week42-bs034.html +++ b/doc/pub/week42/html/._week42-bs034.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,16 +464,23 @@ MathJax.Hub.Config({

     

     

     

    -

    Analyzing the last results

    +

    The back propagation equations for a neural network

    + +

    We have thus

    +$$ +\frac{\partial{\cal C}((\boldsymbol{\Theta}^L)}{\partial w_{ij}^L} = \left(a_j^L - y_j\right)a_j^L(1-a_j^L)a_i^{L-1}, +$$ + +

    Defining

    +$$ +\delta_j^L = a_j^L(1-a_j^L)\left(a_j^L - y_j\right) = \sigma'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}, +$$ + +

    and using the Hadamard product of two vectors we can write this as

    +$$ +\boldsymbol{\delta}^L = \sigma'(\boldsymbol{z}^L)\circ\frac{\partial {\cal C}}{\partial (\boldsymbol{a}^L)}. +$$ -

    This is an important expression. The second term on the right handside -measures how fast the cost function is changing as a function of the $j$th -output activation. If, for example, the cost function doesn't depend -much on a particular output node \( j \), then \( \delta_j^L \) will be small, -which is what we would expect. The first term on the right, measures -how fast the activation function \( f \) is changing at a given activation -value \( z_j^L \). -

    @@ -493,7 +507,7 @@ value \( z_j^L \).

  • 43
  • 44
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs035.html b/doc/pub/week42/html/._week42-bs035.html index 9125a580e..a6693827a 100644 --- a/doc/pub/week42/html/._week42-bs035.html +++ b/doc/pub/week42/html/._week42-bs035.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,27 +464,17 @@ MathJax.Hub.Config({

     

     

     

    -

    More considerations

    +

    Analyzing the last results

    -

    Notice that everything in the above equations is easily computed. In -particular, we compute \( z_j^L \) while computing the behaviour of the -network, and it is only a small additional overhead to compute -\( \sigma'(z^L_j) \). The exact form of the derivative with respect to the -output depends on the form of the cost function. -However, provided the cost function is known there should be little -trouble in calculating +

    This is an important expression. The second term on the right handside +measures how fast the cost function is changing as a function of the $j$th +output activation. If, for example, the cost function doesn't depend +much on a particular output node \( j \), then \( \delta_j^L \) will be small, +which is what we would expect. The first term on the right, measures +how fast the activation function \( f \) is changing at a given activation +value \( z_j^L \).

    -$$ -\frac{\partial {\cal C}}{\partial (a_j^L)} -$$ - -

    With the definition of \( \delta_j^L \) we have a more compact definition of the derivative of the cost function in terms of the weights, namely

    -$$ -\frac{\partial{\cal C}}{\partial w_{ij}^L} = \delta_j^La_i^{L-1}. -$$ - -

    diff --git a/doc/pub/week42/html/._week42-bs036.html b/doc/pub/week42/html/._week42-bs036.html index 94383fcbd..4c0068c55 100644 --- a/doc/pub/week42/html/._week42-bs036.html +++ b/doc/pub/week42/html/._week42-bs036.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,20 +464,26 @@ MathJax.Hub.Config({

     

     

     

    -

    Derivatives in terms of \( z_j^L \)

    +

    More considerations

    -

    It is also easy to see that our previous equation can be written as

    +

    Notice that everything in the above equations is easily computed. In +particular, we compute \( z_j^L \) while computing the behaviour of the +network, and it is only a small additional overhead to compute +\( \sigma'(z^L_j) \). The exact form of the derivative with respect to the +output depends on the form of the cost function. +However, provided the cost function is known there should be little +trouble in calculating +

    $$ -\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L}, +\frac{\partial {\cal C}}{\partial (a_j^L)} $$ -

    which can also be interpreted as the partial derivative of the cost function with respect to the biases \( b_j^L \), namely

    +

    With the definition of \( \delta_j^L \) we have a more compact definition of the derivative of the cost function in terms of the weights, namely

    $$ -\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L}, +\frac{\partial{\cal C}}{\partial w_{ij}^L} = \delta_j^La_i^{L-1}. $$ -

    That is, the error \( \delta_j^L \) is exactly equal to the rate of change of the cost function as a function of the bias.

    @@ -497,7 +510,7 @@ $$

  • 45
  • 46
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs037.html b/doc/pub/week42/html/._week42-bs037.html index ab9b6e9aa..187c57639 100644 --- a/doc/pub/week42/html/._week42-bs037.html +++ b/doc/pub/week42/html/._week42-bs037.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,34 +464,20 @@ MathJax.Hub.Config({

     

     

     

    -

    Bringing it together

    +

    Derivatives in terms of \( z_j^L \)

    -

    We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are

    +

    It is also easy to see that our previous equation can be written as

    $$ -\begin{equation} -\frac{\partial{\cal C}(\boldsymbol{W^L})}{\partial w_{ij}^L} = \delta_j^La_i^{L-1}, -\tag{1} -\end{equation} +\delta_j^L =\frac{\partial {\cal C}}{\partial z_j^L}= \frac{\partial {\cal C}}{\partial a_j^L}\frac{\partial a_j^L}{\partial z_j^L}, $$ -

    and

    +

    which can also be interpreted as the partial derivative of the cost function with respect to the biases \( b_j^L \), namely

    $$ -\begin{equation} -\delta_j^L = \sigma'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}, -\tag{2} -\end{equation} -$$ - -

    and

    - -$$ -\begin{equation} -\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}, -\tag{3} -\end{equation} +\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}\frac{\partial b_j^L}{\partial z_j^L}=\frac{\partial {\cal C}}{\partial b_j^L}, $$ +

    That is, the error \( \delta_j^L \) is exactly equal to the rate of change of the cost function as a function of the bias.

    @@ -511,7 +504,7 @@ $$

  • 46
  • 47
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs038.html b/doc/pub/week42/html/._week42-bs038.html index 1ba4bbe23..e31f59398 100644 --- a/doc/pub/week42/html/._week42-bs038.html +++ b/doc/pub/week42/html/._week42-bs038.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,14 +464,34 @@ MathJax.Hub.Config({

     

     

     

    -

    Final back propagating equation

    +

    Bringing it together

    + +

    We have now three equations that are essential for the computations of the derivatives of the cost function at the output layer. These equations are needed to start the algorithm and they are

    -

    We have that (replacing \( L \) with a general layer \( l \))

    $$ -\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}. +\begin{equation} +\frac{\partial{\cal C}(\boldsymbol{W^L})}{\partial w_{ij}^L} = \delta_j^La_i^{L-1}, +\tag{1} +\end{equation} +$$ + +

    and

    +$$ +\begin{equation} +\delta_j^L = \sigma'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}, +\tag{2} +\end{equation} +$$ + +

    and

    + +$$ +\begin{equation} +\delta_j^L = \frac{\partial {\cal C}}{\partial b_j^L}, +\tag{3} +\end{equation} $$ -

    We want to express this in terms of the equations for layer \( l+1 \).

    @@ -491,7 +518,7 @@ $$

  • 47
  • 48
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs039.html b/doc/pub/week42/html/._week42-bs039.html index f91b8d751..4e00e8c88 100644 --- a/doc/pub/week42/html/._week42-bs039.html +++ b/doc/pub/week42/html/._week42-bs039.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,26 +464,14 @@ MathJax.Hub.Config({

     

     

     

    -

    Using the chain rule and summing over all \( k \) entries

    +

    Final back propagating equation

    -

    We obtain

    +

    We have that (replacing \( L \) with a general layer \( l \))

    $$ -\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}, +\delta_j^l =\frac{\partial {\cal C}}{\partial z_j^l}. $$ -

    and recalling that

    -$$ -z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1}, -$$ - -

    with \( M_l \) being the number of nodes in layer \( l \), we obtain

    -$$ -\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l), -$$ - -

    This is our final equation.

    - -

    We are now ready to set up the algorithm for back propagation and learning the weights and biases.

    +

    We want to express this in terms of the equations for layer \( l+1 \).

    @@ -503,7 +498,7 @@ $$

  • 48
  • 49
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs040.html b/doc/pub/week42/html/._week42-bs040.html index b38eb44f2..1d13e5439 100644 --- a/doc/pub/week42/html/._week42-bs040.html +++ b/doc/pub/week42/html/._week42-bs040.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,22 +464,26 @@ MathJax.Hub.Config({

     

     

     

    -

    Setting up the back propagation algorithm

    +

    Using the chain rule and summing over all \( k \) entries

    -

    The four equations provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.

    +

    We obtain

    +$$ +\delta_j^l =\sum_k \frac{\partial {\cal C}}{\partial z_k^{l+1}}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}=\sum_k \delta_k^{l+1}\frac{\partial z_k^{l+1}}{\partial z_j^{l}}, +$$ -

    First, we set up the input data \( \boldsymbol{x} \) and the activations -\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and -the pertinent outputs \( \boldsymbol{a}^1 \). -

    +

    and recalling that

    +$$ +z_j^{l+1} = \sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1}, +$$ -

    Secondly, we perform then the feed forward till we reach the output -layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the -activation function and the pertinent outputs \( \boldsymbol{a}^l \) for -\( l=1,2,3,\dots,L \). -

    +

    with \( M_l \) being the number of nodes in layer \( l \), we obtain

    +$$ +\delta_j^l =\sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l), +$$ -

    Notation: The first hidden layer has \( l=1 \) as label and the final output layer has \( l=L \).

    +

    This is our final equation.

    + +

    We are now ready to set up the algorithm for back propagation and learning the weights and biases.

    @@ -499,7 +510,7 @@ activation function and the pertinent outputs \( \boldsymbol{a}^l \) for

  • 49
  • 50
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs041.html b/doc/pub/week42/html/._week42-bs041.html index 0ab64d8a6..19c1dd6ad 100644 --- a/doc/pub/week42/html/._week42-bs041.html +++ b/doc/pub/week42/html/._week42-bs041.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,18 +464,22 @@ MathJax.Hub.Config({

     

     

     

    -

    Setting up the back propagation algorithm, part 2

    +

    Setting up the back propagation algorithm

    -

    Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all

    -$$ -\delta_j^L = \sigma'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}. -$$ +

    The four equations provide us with a way of computing the gradient of the cost function. Let us write this out in the form of an algorithm.

    -

    Then we compute the back propagate error for each \( l=L-1,L-2,\dots,1 \) as

    -$$ -\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l). -$$ +

    First, we set up the input data \( \boldsymbol{x} \) and the activations +\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and +the pertinent outputs \( \boldsymbol{a}^1 \). +

    +

    Secondly, we perform then the feed forward till we reach the output +layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the +activation function and the pertinent outputs \( \boldsymbol{a}^l \) for +\( l=1,2,3,\dots,L \). +

    + +

    Notation: The first hidden layer has \( l=1 \) as label and the final output layer has \( l=L \).

    @@ -495,7 +506,7 @@ $$

  • 50
  • 51
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs042.html b/doc/pub/week42/html/._week42-bs042.html index bf659b445..bf46208ac 100644 --- a/doc/pub/week42/html/._week42-bs042.html +++ b/doc/pub/week42/html/._week42-bs042.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,23 +464,18 @@ MathJax.Hub.Config({

     

     

     

    -

    Setting up the Back propagation algorithm, part 3

    - -

    Finally, we update the weights and the biases using gradient descent -for each \( l=L-1,L-2,\dots,1 \) (the first hidden layer) and update the weights and biases -according to the rules -

    +

    Setting up the back propagation algorithm, part 2

    +

    Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all

    $$ -w_{ij}^l\leftarrow = w_{ij}^l- \eta \delta_j^la_i^{l-1}, +\delta_j^L = \sigma'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}. $$ - +

    Then we compute the back propagate error for each \( l=L-1,L-2,\dots,1 \) as

    $$ -b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l, +\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l). $$ -

    with \( \eta \) being the learning rate.

    @@ -500,7 +502,7 @@ $$

  • 51
  • 52
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs043.html b/doc/pub/week42/html/._week42-bs043.html index c2618d03d..164217d4e 100644 --- a/doc/pub/week42/html/._week42-bs043.html +++ b/doc/pub/week42/html/._week42-bs043.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,14 +464,13 @@ MathJax.Hub.Config({

     

     

     

    -

    Updating the gradients

    +

    Setting up the Back propagation algorithm, part 3

    -

    With the back propagate error for each \( l=L-1,L-2,\dots,1 \) as

    -$$ -\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l), -$$ +

    Finally, we update the weights and the biases using gradient descent +for each \( l=L-1,L-2,\dots,1 \) (the first hidden layer) and update the weights and biases +according to the rules +

    -

    we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,1 \) and update the weights and biases according to the rules

    $$ w_{ij}^l\leftarrow = w_{ij}^l- \eta \delta_j^la_i^{l-1}, $$ @@ -474,6 +480,7 @@ $$ b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l, $$ +

    with \( \eta \) being the learning rate.

    @@ -500,7 +507,7 @@ $$

  • 52
  • 53
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs044.html b/doc/pub/week42/html/._week42-bs044.html index 9ba7f7680..27689cfbc 100644 --- a/doc/pub/week42/html/._week42-bs044.html +++ b/doc/pub/week42/html/._week42-bs044.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,20 +464,24 @@ MathJax.Hub.Config({

     

     

     

    -

    Activation functions

    +

    Updating the gradients

    + +

    With the back propagate error for each \( l=L-1,L-2,\dots,1 \) as

    +$$ +\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l), +$$ + +

    we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,1 \) and update the weights and biases according to the rules

    +$$ +w_{ij}^l\leftarrow = w_{ij}^l- \eta \delta_j^la_i^{l-1}, +$$ + + +$$ +b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l, +$$ -

    A property that characterizes a neural network, other than its -connectivity, is the choice of activation function(s). As described -in, the following restrictions are imposed on an activation function -for a FFNN to fulfill the universal approximation theorem -

    -

    diff --git a/doc/pub/week42/html/._week42-bs045.html b/doc/pub/week42/html/._week42-bs045.html index af21f45b1..c7013865a 100644 --- a/doc/pub/week42/html/._week42-bs045.html +++ b/doc/pub/week42/html/._week42-bs045.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,29 +464,20 @@ MathJax.Hub.Config({

     

     

     

    -

    Activation functions, Logistic and Hyperbolic ones

    +

    Activation functions

    -

    The second requirement excludes all linear functions. Furthermore, in -a MLP with only linear activation functions, each layer simply -performs a linear transformation of its inputs. +

    A property that characterizes a neural network, other than its +connectivity, is the choice of activation function(s). As described +in, the following restrictions are imposed on an activation function +for a FFNN to fulfill the universal approximation theorem

    -

    Regardless of the number of layers, the output of the NN will be -nothing but a linear function of the inputs. Thus we need to introduce -some kind of non-linearity to the NN to be able to fit non-linear -functions Typical examples are the logistic Sigmoid -

    - -$$ - \sigma(x) = \frac{1}{1 + e^{-x}}, -$$ - -

    and the hyperbolic tangent function

    -$$ - \sigma(x) = \tanh(x) -$$ - - +

    diff --git a/doc/pub/week42/html/._week42-bs046.html b/doc/pub/week42/html/._week42-bs046.html index a4a93ab63..b42bf51ce 100644 --- a/doc/pub/week42/html/._week42-bs046.html +++ b/doc/pub/week42/html/._week42-bs046.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,107 +464,27 @@ MathJax.Hub.Config({

     

     

     

    -

    Relevance

    +

    Activation functions, Logistic and Hyperbolic ones

    -

    The sigmoid function are more biologically plausible because the -output of inactive neurons are zero. Such activation function are -called one-sided. However, it has been shown that the hyperbolic -tangent performs better than the sigmoid for training MLPs. has -become the most popular for deep neural networks +

    The second requirement excludes all linear functions. Furthermore, in +a MLP with only linear activation functions, each layer simply +performs a linear transformation of its inputs.

    +

    Regardless of the number of layers, the output of the NN will be +nothing but a linear function of the inputs. Thus we need to introduce +some kind of non-linearity to the NN to be able to fit non-linear +functions Typical examples are the logistic Sigmoid +

    - -
    -
    -
    -
    -
    - 
    """The sigmoid function (or the logistic curve) is a 
-function that takes any real number, z, and outputs a number (0,1).
-It is useful in neural networks for assigning weights on a relative scale.
-The value z is the weighted sum of parameters involved in the learning algorithm."""
+$$
+ \sigma(x) = \frac{1}{1 + e^{-x}},
+$$
 
-import numpy
-import matplotlib.pyplot as plt
-import math as mt
-
-z = numpy.arange(-5, 5, .1)
-sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z)))
-sigma = sigma_fn(z)
-
-fig = plt.figure()
-ax = fig.add_subplot(111)
-ax.plot(z, sigma)
-ax.set_ylim([-0.1, 1.1])
-ax.set_xlim([-5,5])
-ax.grid(True)
-ax.set_xlabel('z')
-ax.set_title('sigmoid function')
-
-plt.show()
-
-"""Step Function"""
-z = numpy.arange(-5, 5, .02)
-step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0)
-step = step_fn(z)
-
-fig = plt.figure()
-ax = fig.add_subplot(111)
-ax.plot(z, step)
-ax.set_ylim([-0.5, 1.5])
-ax.set_xlim([-5,5])
-ax.grid(True)
-ax.set_xlabel('z')
-ax.set_title('step function')
-
-plt.show()
-
-"""Sine Function"""
-z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1)
-t = numpy.sin(z)
-
-fig = plt.figure()
-ax = fig.add_subplot(111)
-ax.plot(z, t)
-ax.set_ylim([-1.0, 1.0])
-ax.set_xlim([-2*mt.pi,2*mt.pi])
-ax.grid(True)
-ax.set_xlabel('z')
-ax.set_title('sine function')
-
-plt.show()
-
-"""Plots a graph of the squashing function used by a rectified linear
-unit"""
-z = numpy.arange(-2, 2, .1)
-zero = numpy.zeros(len(z))
-y = numpy.max([zero, z], axis=0)
-
-fig = plt.figure()
-ax = fig.add_subplot(111)
-ax.plot(z, y)
-ax.set_ylim([-2.0, 2.0])
-ax.set_xlim([-2.0, 2.0])
-ax.grid(True)
-ax.set_xlabel('z')
-ax.set_title('Rectified linear unit')
-
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    and the hyperbolic tangent function

    +$$ + \sigma(x) = \tanh(x) +$$

    @@ -585,7 +512,7 @@ plt.show()

  • 55
  • 56
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs047.html b/doc/pub/week42/html/._week42-bs047.html index 53d66b288..8870bc557 100644 --- a/doc/pub/week42/html/._week42-bs047.html +++ b/doc/pub/week42/html/._week42-bs047.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,30 +464,109 @@ MathJax.Hub.Config({

     

     

     

    -

    Fine-tuning neural network hyperparameters

    +

    Relevance

    -

    The flexibility of neural networks is also one of their main -drawbacks: there are many hyperparameters to tweak. Not only can you -use any imaginable network topology (how neurons/nodes are -interconnected), but even in a simple FFNN you can change the number -of layers, the number of neurons per layer, the type of activation -function to use in each layer, the weight initialization logic, the -stochastic gradient optmized and much more. How do you know what -combination of hyperparameters is the best for your task? +

    The sigmoid function are more biologically plausible because the +output of inactive neurons are zero. Such activation function are +called one-sided. However, it has been shown that the hyperbolic +tangent performs better than the sigmoid for training MLPs. has +become the most popular for deep neural networks

    - -

    However,since there are many hyperparameters to tune, and since -training a neural network on a large dataset takes a lot of time, you -will only be able to explore a tiny part of the hyperparameter space. -

    - + +
    +
    +
    +
    +
    + 
    """The sigmoid function (or the logistic curve) is a 
+function that takes any real number, z, and outputs a number (0,1).
+It is useful in neural networks for assigning weights on a relative scale.
+The value z is the weighted sum of parameters involved in the learning algorithm."""
+
+import numpy
+import matplotlib.pyplot as plt
+import math as mt
+
+z = numpy.arange(-5, 5, .1)
+sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z)))
+sigma = sigma_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, sigma)
+ax.set_ylim([-0.1, 1.1])
+ax.set_xlim([-5,5])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('sigmoid function')
+
+plt.show()
+
+"""Step Function"""
+z = numpy.arange(-5, 5, .02)
+step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0)
+step = step_fn(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, step)
+ax.set_ylim([-0.5, 1.5])
+ax.set_xlim([-5,5])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('step function')
+
+plt.show()
+
+"""Sine Function"""
+z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1)
+t = numpy.sin(z)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, t)
+ax.set_ylim([-1.0, 1.0])
+ax.set_xlim([-2*mt.pi,2*mt.pi])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('sine function')
+
+plt.show()
+
+"""Plots a graph of the squashing function used by a rectified linear
+unit"""
+z = numpy.arange(-2, 2, .1)
+zero = numpy.zeros(len(z))
+y = numpy.max([zero, z], axis=0)
+
+fig = plt.figure()
+ax = fig.add_subplot(111)
+ax.plot(z, y)
+ax.set_ylim([-2.0, 2.0])
+ax.set_xlim([-2.0, 2.0])
+ax.grid(True)
+ax.set_xlabel('z')
+ax.set_title('Rectified linear unit')
+
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +

    diff --git a/doc/pub/week42/html/._week42-bs048.html b/doc/pub/week42/html/._week42-bs048.html index 42bfd6c60..1cd245fcc 100644 --- a/doc/pub/week42/html/._week42-bs048.html +++ b/doc/pub/week42/html/._week42-bs048.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,26 +464,30 @@ MathJax.Hub.Config({

     

     

     

    -

    Hidden layers

    +

    Fine-tuning neural network hyperparameters

    -

    For many problems you can start with just one or two hidden layers and -it will work just fine. For the MNIST data set you ca easily get a -high accuracy using just one hidden layer with a few hundred neurons. -You can reach for this data set above 98% accuracy using two hidden -layers with the same total amount of neurons, in roughly the same -amount of training time. +

    The flexibility of neural networks is also one of their main +drawbacks: there are many hyperparameters to tweak. Not only can you +use any imaginable network topology (how neurons/nodes are +interconnected), but even in a simple FFNN you can change the number +of layers, the number of neurons per layer, the type of activation +function to use in each layer, the weight initialization logic, the +stochastic gradient optmized and much more. How do you know what +combination of hyperparameters is the best for your task?

    -

    For more complex problems, you can gradually ramp up the number of -hidden layers, until you start overfitting the training set. Very -complex tasks, such as large image classification or speech -recognition, typically require networks with dozens of layers and they -need a huge amount of training data. However, you will rarely have to -train such networks from scratch: it is much more common to reuse -parts of a pretrained state-of-the-art network that performs a similar -task. +

    +

    However,since there are many hyperparameters to tune, and since +training a neural network on a large dataset takes a lot of time, you +will only be able to explore a tiny part of the hyperparameter space.

    +

    diff --git a/doc/pub/week42/html/._week42-bs049.html b/doc/pub/week42/html/._week42-bs049.html index 744a0b520..d459c5de5 100644 --- a/doc/pub/week42/html/._week42-bs049.html +++ b/doc/pub/week42/html/._week42-bs049.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,21 +463,25 @@ MathJax.Hub.Config({

     

     

     

    - -

    Vanishing gradients

    + +

    Hidden layers

    -

    The Back propagation algorithm we derived above works by going from -the output layer to the input layer, propagating the error gradient on -the way. Once the algorithm has computed the gradient of the cost -function with regards to each parameter in the network, it uses these -gradients to update each parameter with a Gradient Descent (GD) step. +

    For many problems you can start with just one or two hidden layers and +it will work just fine. For the MNIST data set you ca easily get a +high accuracy using just one hidden layer with a few hundred neurons. +You can reach for this data set above 98% accuracy using two hidden +layers with the same total amount of neurons, in roughly the same +amount of training time.

    -

    Unfortunately for us, the gradients often get smaller and smaller as -the algorithm progresses down to the first hidden layers. As a result, -the GD update leaves the lower layer connection weights virtually -unchanged, and training never converges to a good solution. This is -known in the literature as the vanishing gradients problem. +

    For more complex problems, you can gradually ramp up the number of +hidden layers, until you start overfitting the training set. Very +complex tasks, such as large image classification or speech +recognition, typically require networks with dozens of layers and they +need a huge amount of training data. However, you will rarely have to +train such networks from scratch: it is much more common to reuse +parts of a pretrained state-of-the-art network that performs a similar +task.

    @@ -498,7 +509,7 @@ known in the literature as the vanishing gradients problem.

  • 58
  • 59
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs050.html b/doc/pub/week42/html/._week42-bs050.html index 6007bf9a2..5de2e0dfd 100644 --- a/doc/pub/week42/html/._week42-bs050.html +++ b/doc/pub/week42/html/._week42-bs050.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,16 +463,21 @@ MathJax.Hub.Config({

     

     

     

    - -

    Exploding gradients

    + +

    Vanishing gradients

    -

    In other cases, the opposite can happen, namely the the gradients can -grow bigger and bigger. The result is that many of the layers get -large updates of the weights the algorithm diverges. This is the -exploding gradients problem, which is mostly encountered in -recurrent neural networks. More generally, deep neural networks suffer -from unstable gradients, different layers may learn at widely -different speeds +

    The Back propagation algorithm we derived above works by going from +the output layer to the input layer, propagating the error gradient on +the way. Once the algorithm has computed the gradient of the cost +function with regards to each parameter in the network, it uses these +gradients to update each parameter with a Gradient Descent (GD) step. +

    + +

    Unfortunately for us, the gradients often get smaller and smaller as +the algorithm progresses down to the first hidden layers. As a result, +the GD update leaves the lower layer connection weights virtually +unchanged, and training never converges to a good solution. This is +known in the literature as the vanishing gradients problem.

    @@ -493,7 +505,7 @@ different speeds

  • 59
  • 60
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs051.html b/doc/pub/week42/html/._week42-bs051.html index 931762fec..fac0e5ad8 100644 --- a/doc/pub/week42/html/._week42-bs051.html +++ b/doc/pub/week42/html/._week42-bs051.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,22 +463,16 @@ MathJax.Hub.Config({

     

     

     

    - -

    Is the Logistic activation function (Sigmoid) our choice?

    + +

    Exploding gradients

    -

    Although this unfortunate behavior has been empirically observed for -quite a while (it was one of the reasons why deep neural networks were -mostly abandoned for a long time), it is only around 2010 that -significant progress was made in understanding it. -

    - -

    A paper titled Understanding the Difficulty of Training Deep -Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio found that -the problems with the popular logistic -sigmoid activation function and the weight initialization technique -that was most popular at the time, namely random initialization using -a normal distribution with a mean of 0 and a standard deviation of -1. +

    In other cases, the opposite can happen, namely the the gradients can +grow bigger and bigger. The result is that many of the layers get +large updates of the weights the algorithm diverges. This is the +exploding gradients problem, which is mostly encountered in +recurrent neural networks. More generally, deep neural networks suffer +from unstable gradients, different layers may learn at widely +different speeds

    @@ -499,7 +500,7 @@ a normal distribution with a mean of 0 and a standard deviation of

  • 60
  • 61
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs052.html b/doc/pub/week42/html/._week42-bs052.html index b5c954d2b..6b5768aaa 100644 --- a/doc/pub/week42/html/._week42-bs052.html +++ b/doc/pub/week42/html/._week42-bs052.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,17 +463,22 @@ MathJax.Hub.Config({

     

     

     

    - -

    Logistic function as the root of problems

    + +

    Is the Logistic activation function (Sigmoid) our choice?

    -

    They showed that with this activation function and this -initialization scheme, the variance of the outputs of each layer is -much greater than the variance of its inputs. Going forward in the -network, the variance keeps increasing after each layer until the -activation function saturates at the top layers. This is actually made -worse by the fact that the logistic function has a mean of 0.5, not 0 -(the hyperbolic tangent function has a mean of 0 and behaves slightly -better than the logistic function in deep networks). +

    Although this unfortunate behavior has been empirically observed for +quite a while (it was one of the reasons why deep neural networks were +mostly abandoned for a long time), it is only around 2010 that +significant progress was made in understanding it. +

    + +

    A paper titled Understanding the Difficulty of Training Deep +Feedforward Neural Networks by Xavier Glorot and Yoshua Bengio found that +the problems with the popular logistic +sigmoid activation function and the weight initialization technique +that was most popular at the time, namely random initialization using +a normal distribution with a mean of 0 and a standard deviation of +1.

    @@ -494,7 +506,7 @@ better than the logistic function in deep networks).

  • 61
  • 62
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs053.html b/doc/pub/week42/html/._week42-bs053.html index 33aea1f27..39cea7f49 100644 --- a/doc/pub/week42/html/._week42-bs053.html +++ b/doc/pub/week42/html/._week42-bs053.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,26 +464,16 @@ MathJax.Hub.Config({

     

     

     

    -

    The derivative of the Logistic funtion

    +

    Logistic function as the root of problems

    -

    Looking at the logistic activation function, when inputs become large -(negative or positive), the function saturates at 0 or 1, with a -derivative extremely close to 0. Thus when backpropagation kicks in, -it has virtually no gradient to propagate back through the network, -and what little gradient exists keeps getting diluted as -backpropagation progresses down through the top layers, so there is -really nothing left for the lower layers. -

    - -

    In their paper, Glorot and Bengio propose a way to significantly -alleviate this problem. We need the signal to flow properly in both -directions: in the forward direction when making predictions, and in -the reverse direction when backpropagating gradients. We don’t want -the signal to die out, nor do we want it to explode and saturate. For -the signal to flow properly, the authors argue that we need the -variance of the outputs of each layer to be equal to the variance of -its inputs, and we also need the gradients to have equal variance -before and after flowing through a layer in the reverse direction. +

    They showed that with this activation function and this +initialization scheme, the variance of the outputs of each layer is +much greater than the variance of its inputs. Going forward in the +network, the variance keeps increasing after each layer until the +activation function saturates at the top layers. This is actually made +worse by the fact that the logistic function has a mean of 0.5, not 0 +(the hyperbolic tangent function has a mean of 0 and behaves slightly +better than the logistic function in deep networks).

    @@ -504,7 +501,7 @@ before and after flowing through a layer in the reverse direction.

  • 62
  • 63
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs054.html b/doc/pub/week42/html/._week42-bs054.html index 7f4ad52c1..87d43678f 100644 --- a/doc/pub/week42/html/._week42-bs054.html +++ b/doc/pub/week42/html/._week42-bs054.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,17 +464,26 @@ MathJax.Hub.Config({

     

     

     

    -

    Insights from the paper by Glorot and Bengio

    +

    The derivative of the Logistic funtion

    -

    One of the insights in the 2010 paper by Glorot and Bengio was that -the vanishing/exploding gradients problems were in part due to a poor -choice of activation function. Until then most people had assumed that -if Nature had chosen to use roughly sigmoid activation functions in -biological neurons, they must be an excellent choice. But it turns out -that other activation functions behave much better in deep neural -networks, in particular the ReLU activation function, mostly because -it does not saturate for positive values (and also because it is quite -fast to compute). +

    Looking at the logistic activation function, when inputs become large +(negative or positive), the function saturates at 0 or 1, with a +derivative extremely close to 0. Thus when backpropagation kicks in, +it has virtually no gradient to propagate back through the network, +and what little gradient exists keeps getting diluted as +backpropagation progresses down through the top layers, so there is +really nothing left for the lower layers. +

    + +

    In their paper, Glorot and Bengio propose a way to significantly +alleviate this problem. We need the signal to flow properly in both +directions: in the forward direction when making predictions, and in +the reverse direction when backpropagating gradients. We don’t want +the signal to die out, nor do we want it to explode and saturate. For +the signal to flow properly, the authors argue that we need the +variance of the outputs of each layer to be equal to the variance of +its inputs, and we also need the gradients to have equal variance +before and after flowing through a layer in the reverse direction.

    @@ -495,7 +511,7 @@ fast to compute).

  • 63
  • 64
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs055.html b/doc/pub/week42/html/._week42-bs055.html index 2f7f2bdcb..d597e0008 100644 --- a/doc/pub/week42/html/._week42-bs055.html +++ b/doc/pub/week42/html/._week42-bs055.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,19 +464,17 @@ MathJax.Hub.Config({

     

     

     

    -

    The RELU function family

    +

    Insights from the paper by Glorot and Bengio

    -

    The ReLU activation function suffers from a problem known as the dying -ReLUs: during training, some neurons effectively die, meaning they -stop outputting anything other than 0. -

    - -

    In some cases, you may find that half of your network’s neurons are -dead, especially if you used a large learning rate. During training, -if a neuron’s weights get updated such that the weighted sum of the -neuron’s inputs is negative, it will start outputting 0. When this -happen, the neuron is unlikely to come back to life since the gradient -of the ReLU function is 0 when its input is negative. +

    One of the insights in the 2010 paper by Glorot and Bengio was that +the vanishing/exploding gradients problems were in part due to a poor +choice of activation function. Until then most people had assumed that +if Nature had chosen to use roughly sigmoid activation functions in +biological neurons, they must be an excellent choice. But it turns out +that other activation functions behave much better in deep neural +networks, in particular the ReLU activation function, mostly because +it does not saturate for positive values (and also because it is quite +fast to compute).

    @@ -497,7 +502,7 @@ of the ReLU function is 0 when its input is negative.

  • 64
  • 65
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs056.html b/doc/pub/week42/html/._week42-bs056.html index a485f496a..d73931d2e 100644 --- a/doc/pub/week42/html/._week42-bs056.html +++ b/doc/pub/week42/html/._week42-bs056.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,17 +464,20 @@ MathJax.Hub.Config({

     

     

     

    -

    ELU function

    +

    The RELU function family

    -

    To solve this problem, nowadays practitioners use a variant of the -ReLU function, such as the leaky ReLU discussed above or the so-called -exponential linear unit (ELU) function +

    The ReLU activation function suffers from a problem known as the dying +ReLUs: during training, some neurons effectively die, meaning they +stop outputting anything other than 0.

    -$$ -ELU(z) = \left\{\begin{array}{cc} \alpha\left( \exp{(z)}-1\right) & z < 0,\\ z & z \ge 0.\end{array}\right. -$$ - +

    In some cases, you may find that half of your network’s neurons are +dead, especially if you used a large learning rate. During training, +if a neuron’s weights get updated such that the weighted sum of the +neuron’s inputs is negative, it will start outputting 0. When this +happen, the neuron is unlikely to come back to life since the gradient +of the ReLU function is 0 when its input is negative. +

    @@ -494,7 +504,7 @@ $$

  • 65
  • 66
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs057.html b/doc/pub/week42/html/._week42-bs057.html index d382d4721..ac4c2b367 100644 --- a/doc/pub/week42/html/._week42-bs057.html +++ b/doc/pub/week42/html/._week42-bs057.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,21 +464,17 @@ MathJax.Hub.Config({

     

     

     

    -

    Which activation function should we use?

    +

    ELU function

    -

    In general it seems that the ELU activation function is better than -the leaky ReLU function (and its variants), which is better than -ReLU. ReLU performs better than \( \tanh \) which in turn performs better -than the logistic function. +

    To solve this problem, nowadays practitioners use a variant of the +ReLU function, such as the leaky ReLU discussed above or the so-called +exponential linear unit (ELU) function

    -

    If runtime performance is an issue, then you may opt for the leaky -ReLU function over the ELU function If you don’t want to tweak yet -another hyperparameter, you may just use the default \( \alpha \) of -\( 0.01 \) for the leaky ReLU, and \( 1 \) for ELU. If you have spare time and -computing power, you can use cross-validation or bootstrap to evaluate -other activation functions. -

    +$$ +ELU(z) = \left\{\begin{array}{cc} \alpha\left( \exp{(z)}-1\right) & z < 0,\\ z & z \ge 0.\end{array}\right. +$$ +

    @@ -498,7 +501,7 @@ other activation functions.

  • 66
  • 67
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs058.html b/doc/pub/week42/html/._week42-bs058.html index 32bfb5c9e..caa6470a6 100644 --- a/doc/pub/week42/html/._week42-bs058.html +++ b/doc/pub/week42/html/._week42-bs058.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,22 +464,22 @@ MathJax.Hub.Config({

     

     

     

    -

    More on activation functions, output layers

    +

    Which activation function should we use?

    -

    In most cases you can use the ReLU activation function in the hidden -layers (or one of its variants). +

    In general it seems that the ELU activation function is better than +the leaky ReLU function (and its variants), which is better than +ReLU. ReLU performs better than \( \tanh \) which in turn performs better +than the logistic function.

    -

    It is a bit faster to compute than other activation functions, and the -gradient descent optimization does in general not get stuck. +

    If runtime performance is an issue, then you may opt for the leaky +ReLU function over the ELU function If you don’t want to tweak yet +another hyperparameter, you may just use the default \( \alpha \) of +\( 0.01 \) for the leaky ReLU, and \( 1 \) for ELU. If you have spare time and +computing power, you can use cross-validation or bootstrap to evaluate +other activation functions.

    -For the output layer: - -
      -
    • For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).
      • -
    • For regression tasks, you can simply use no activation function at all.
      • -

      @@ -498,7 +505,7 @@ gradient descent optimization does in general not get stuck.
    • 67
    • 68
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs059.html b/doc/pub/week42/html/._week42-bs059.html index d21745a7a..b82e0f61d 100644 --- a/doc/pub/week42/html/._week42-bs059.html +++ b/doc/pub/week42/html/._week42-bs059.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,26 +464,22 @@ MathJax.Hub.Config({

     

     

     

    -

    Batch Normalization

    +

    More on activation functions, output layers

    -

    Batch Normalization aims to address the vanishing/exploding gradients -problems, and more generally the problem that the distribution of each -layer’s inputs changes during training, as the parameters of the -previous layers change. +

    In most cases you can use the ReLU activation function in the hidden +layers (or one of its variants).

    -

    The technique consists of adding an operation in the model just before -the activation function of each layer, simply zero-centering and -normalizing the inputs, then scaling and shifting the result using two -new parameters per layer (one for scaling, the other for shifting). In -other words, this operation lets the model learn the optimal scale and -mean of the inputs for each layer. In order to zero-center and -normalize the inputs, the algorithm needs to estimate the inputs’ mean -and standard deviation. It does so by evaluating the mean and standard -deviation of the inputs over the current mini-batch, from this the -name batch normalization. +

    It is a bit faster to compute than other activation functions, and the +gradient descent optimization does in general not get stuck.

    +For the output layer: + +
      +
    • For classification the softmax activation function is generally a good choice for classification tasks (when the classes are mutually exclusive).
      • +
    • For regression tasks, you can simply use no activation function at all.
      • +

      @@ -502,7 +505,7 @@ name batch normalization.
    • 68
    • 69
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs060.html b/doc/pub/week42/html/._week42-bs060.html index a5815a79e..d7284fa32 100644 --- a/doc/pub/week42/html/._week42-bs060.html +++ b/doc/pub/week42/html/._week42-bs060.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,18 +464,24 @@ MathJax.Hub.Config({

     

     

     

    -

    Dropout

    +

    Batch Normalization

    -

    It is a fairly simple algorithm: at every training step, every neuron -(including the input neurons but excluding the output neurons) has a -probability \( p \) of being temporarily dropped out, meaning it will be -entirely ignored during this training step, but it may be active -during the next step. +

    Batch Normalization aims to address the vanishing/exploding gradients +problems, and more generally the problem that the distribution of each +layer’s inputs changes during training, as the parameters of the +previous layers change.

    -

    The hyperparameter \( p \) is called the dropout rate, and it is typically -set to 50%. After training, the neurons are not dropped anymore. It -is viewed as one of the most popular regularization techniques. +

    The technique consists of adding an operation in the model just before +the activation function of each layer, simply zero-centering and +normalizing the inputs, then scaling and shifting the result using two +new parameters per layer (one for scaling, the other for shifting). In +other words, this operation lets the model learn the optimal scale and +mean of the inputs for each layer. In order to zero-center and +normalize the inputs, the algorithm needs to estimate the inputs’ mean +and standard deviation. It does so by evaluating the mean and standard +deviation of the inputs over the current mini-batch, from this the +name batch normalization.

    @@ -496,7 +509,7 @@ is viewed as one of the most popular regularization techniques.

  • 69
  • 70
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs061.html b/doc/pub/week42/html/._week42-bs061.html index 260bc5182..f5835f44d 100644 --- a/doc/pub/week42/html/._week42-bs061.html +++ b/doc/pub/week42/html/._week42-bs061.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,18 +464,18 @@ MathJax.Hub.Config({

     

     

     

    -

    Gradient Clipping

    +

    Dropout

    -

    A popular technique to lessen the exploding gradients problem is to -simply clip the gradients during backpropagation so that they never -exceed some threshold (this is mostly useful for recurrent neural -networks). +

    It is a fairly simple algorithm: at every training step, every neuron +(including the input neurons but excluding the output neurons) has a +probability \( p \) of being temporarily dropped out, meaning it will be +entirely ignored during this training step, but it may be active +during the next step.

    -

    This technique is called Gradient Clipping.

    - -

    In general however, Batch -Normalization is preferred. +

    The hyperparameter \( p \) is called the dropout rate, and it is typically +set to 50%. After training, the neurons are not dropped anymore. It +is viewed as one of the most popular regularization techniques.

    @@ -496,7 +503,7 @@ Normalization is preferred.

  • 70
  • 71
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs062.html b/doc/pub/week42/html/._week42-bs062.html index 352febb6e..71d83f4db 100644 --- a/doc/pub/week42/html/._week42-bs062.html +++ b/doc/pub/week42/html/._week42-bs062.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,25 +463,21 @@ MathJax.Hub.Config({

     

     

     

    - -

    A top-down perspective on Neural networks

    + +

    Gradient Clipping

    -

    The first thing we would like to do is divide the data into two or -three parts. A training set, a validation or dev (development) set, -and a test set. The test set is the data on which we want to make -predictions. The dev set is a subset of the training data we use to -check how well we are doing out-of-sample, after training the model on -the training dataset. We use the validation error as a proxy for the -test error in order to make tweaks to our model. It is crucial that we -do not use any of the test data to train the algorithm. This is a -cardinal sin in ML. Then: +

    A popular technique to lessen the exploding gradients problem is to +simply clip the gradients during backpropagation so that they never +exceed some threshold (this is mostly useful for recurrent neural +networks). +

    + +

    This technique is called Gradient Clipping.

    + +

    In general however, Batch +Normalization is preferred.

    -
      -
    1. Estimate optimal error rate
      2. -
    2. Minimize underfitting (bias) on training data set.
      3. -
    3. Make sure you are not overfitting.
      4. -

      @@ -500,7 +503,7 @@ cardinal sin in ML. Then:
    • 71
    • 72
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs063.html b/doc/pub/week42/html/._week42-bs063.html index 076540ce0..c2070555b 100644 --- a/doc/pub/week42/html/._week42-bs063.html +++ b/doc/pub/week42/html/._week42-bs063.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,30 +463,25 @@ MathJax.Hub.Config({

     

     

     

    - -

    More top-down perspectives

    + +

    A top-down perspective on Neural networks

    -

    If the validation and test sets are drawn from the same distributions, -then a good performance on the validation set should lead to similarly -good performance on the test set. -

    - -

    However, sometimes -the training data and test data differ in subtle ways because, for -example, they are collected using slightly different methods, or -because it is cheaper to collect data in one way versus another. In -this case, there can be a mismatch between the training and test -data. This can lead to the neural network overfitting these small -differences between the test and training sets, and a poor performance -on the test set despite having a good performance on the validation -set. To rectify this, Andrew Ng suggests making two validation or dev -sets, one constructed from the training data and one constructed from -the test data. The difference between the performance of the algorithm -on these two validation sets quantifies the train-test mismatch. This -can serve as another important diagnostic when using DNNs for -supervised learning. +

    The first thing we would like to do is divide the data into two or +three parts. A training set, a validation or dev (development) set, +and a test set. The test set is the data on which we want to make +predictions. The dev set is a subset of the training data we use to +check how well we are doing out-of-sample, after training the model on +the training dataset. We use the validation error as a proxy for the +test error in order to make tweaks to our model. It is crucial that we +do not use any of the test data to train the algorithm. This is a +cardinal sin in ML. Then:

    +
      +
    1. Estimate optimal error rate
      2. +
    2. Minimize underfitting (bias) on training data set.
      3. +
    3. Make sure you are not overfitting.
      4. +

    diff --git a/doc/pub/week42/html/._week42-bs064.html b/doc/pub/week42/html/._week42-bs064.html index d8df2b23a..037a0c524 100644 --- a/doc/pub/week42/html/._week42-bs064.html +++ b/doc/pub/week42/html/._week42-bs064.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,15 +464,27 @@ MathJax.Hub.Config({

     

     

     

    -

    Limitations of supervised learning with deep networks

    +

    More top-down perspectives

    -

    Like all statistical methods, supervised learning using neural -networks has important limitations. This is especially important when -one seeks to apply these methods, especially to physics problems. Like -all tools, DNNs are not a universal solution. Often, the same or -better performance on a task can be achieved by using a few -hand-engineered features (or even a collection of random -features). +

    If the validation and test sets are drawn from the same distributions, +then a good performance on the validation set should lead to similarly +good performance on the test set. +

    + +

    However, sometimes +the training data and test data differ in subtle ways because, for +example, they are collected using slightly different methods, or +because it is cheaper to collect data in one way versus another. In +this case, there can be a mismatch between the training and test +data. This can lead to the neural network overfitting these small +differences between the test and training sets, and a poor performance +on the test set despite having a good performance on the validation +set. To rectify this, Andrew Ng suggests making two validation or dev +sets, one constructed from the training data and one constructed from +the test data. The difference between the performance of the algorithm +on these two validation sets quantifies the train-test mismatch. This +can serve as another important diagnostic when using DNNs for +supervised learning.

    @@ -493,7 +512,7 @@ features).

  • 73
  • 74
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs065.html b/doc/pub/week42/html/._week42-bs065.html index 4c3c80ca4..931dc4358 100644 --- a/doc/pub/week42/html/._week42-bs065.html +++ b/doc/pub/week42/html/._week42-bs065.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,14 +464,17 @@ MathJax.Hub.Config({

     

     

     

    -

    Limitations of NNs

    +

    Limitations of supervised learning with deep networks

    -

    Here we list some of the important limitations of supervised neural network based models.

    +

    Like all statistical methods, supervised learning using neural +networks has important limitations. This is especially important when +one seeks to apply these methods, especially to physics problems. Like +all tools, DNNs are not a universal solution. Often, the same or +better performance on a task can be achieved by using a few +hand-engineered features (or even a collection of random +features). +

    -
      -
    • Need labeled data. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).
      • -
    • Supervised neural networks are extremely data intensive. DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.
      • -

    diff --git a/doc/pub/week42/html/._week42-bs066.html b/doc/pub/week42/html/._week42-bs066.html index a3bba4859..e0d93fa69 100644 --- a/doc/pub/week42/html/._week42-bs066.html +++ b/doc/pub/week42/html/._week42-bs066.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,10 +464,13 @@ MathJax.Hub.Config({

     

     

     

    -

    Homogeneous data

    +

    Limitations of NNs

    + +

    Here we list some of the important limitations of supervised neural network based models.

      -
    • Homogeneous data. Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e. some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.
      • +
    • Need labeled data. All supervised learning methods, DNNs for supervised learning require labeled data. Often, labeled data is harder to acquire than unlabeled data (e.g. one must pay for human experts to label images).
      • +
    • Supervised neural networks are extremely data intensive. DNNs are data hungry. They perform best when data is plentiful. This is doubly so for supervised methods where the data must also be labeled. The utility of DNNs is extremely limited if data is hard to acquire or the datasets are small (hundreds to a few thousand samples). In this case, the performance of other methods that utilize hand-engineered features can exceed that of DNNs.

    @@ -487,7 +497,7 @@ MathJax.Hub.Config({

  • 75
  • 76
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs067.html b/doc/pub/week42/html/._week42-bs067.html index 866bc68ec..136a538b7 100644 --- a/doc/pub/week42/html/._week42-bs067.html +++ b/doc/pub/week42/html/._week42-bs067.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,13 +464,11 @@ MathJax.Hub.Config({

     

     

     

    -

    More limitations

    +

    Homogeneous data

      -
    • Many problems are not about prediction. In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a wrong model. The model might or might not be useful for understanding the underlying science.
      • +
    • Homogeneous data. Almost all DNNs deal with homogeneous data of one type. It is very hard to design architectures that mix and match data types (i.e. some continuous variables, some discrete variables, some time series). In applications beyond images, video, and language, this is often what is required. In contrast, ensemble models like random forests or gradient-boosted trees have no difficulty handling mixed data types.
    -

    Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.

    -

    diff --git a/doc/pub/week42/html/._week42-bs068.html b/doc/pub/week42/html/._week42-bs068.html index fd594f43f..c46fd7545 100644 --- a/doc/pub/week42/html/._week42-bs068.html +++ b/doc/pub/week42/html/._week42-bs068.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,74 +464,12 @@ MathJax.Hub.Config({

     

     

     

    -

    Setting up the back-propagation algorithm

    +

    More limitations

    -

    Let us write this out in the form of an algorithm.

    - -
    -
    - -

    First, we set up the input data \( \boldsymbol{x} \) and the activations -\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and -the pertinent outputs \( \boldsymbol{a}^1 \). -

    -
    -
    - - -
    -
    - -

    Secondly, we perform then the feed forward till we reach the output -layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the -activation function and the pertinent outputs \( \boldsymbol{a}^l \) for -\( l=2,3,\dots,L \). -

    -
    -
    - - -
    -
    - -

    Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all

    -$$ -\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}. -$$ -
    -
    - - -
    -
    - -

    Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as

    -$$ -\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l). -$$ -
    -
    - - -
    -
    - -

    Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules

    -$$ -w_{ij}^l\leftarrow = w_{ij}^l- \eta \delta_j^la_i^{l-1}, -$$ - - -$$ -b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l, -$$ -
    -
    - - -

    The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods. -Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training. -

    +
      +
    • Many problems are not about prediction. In natural science we are often interested in learning something about the underlying distribution that generates the data. In this case, it is often difficult to cast these ideas in a supervised learning setting. While the problems are related, it is possible to make good predictions with a wrong model. The model might or might not be useful for understanding the underlying science.
      • +
    +

    Some of these remarks are particular to DNNs, others are shared by all supervised learning methods. This motivates the use of unsupervised methods which in part circumvent these problems.

    @@ -551,7 +496,7 @@ Here it is convenient to use stochastic gradient descent (see the examples below

  • 77
  • 78
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs069.html b/doc/pub/week42/html/._week42-bs069.html index c097eb7ae..53bbf9cb8 100644 --- a/doc/pub/week42/html/._week42-bs069.html +++ b/doc/pub/week42/html/._week42-bs069.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,43 +463,76 @@ MathJax.Hub.Config({

     

     

     

    - -

    Setting up a Multi-layer perceptron model for classification

    + +

    Setting up the back-propagation algorithm

    -

    We are now gong to develop an example based on the MNIST data -base. This is a classification problem and we need to use our -cross-entropy function we discussed in connection with logistic -regression. The cross-entropy defines our cost function for the -classificaton problems with neural networks. -

    +

    Let us write this out in the form of an algorithm.

    -

    In binary classification with two classes \( (0, 1) \) we define the -logistic/sigmoid function as the probability that a particular input -is in class \( 0 \) or \( 1 \). This is possible because the logistic -function takes any input from the real numbers and inputs a number -between 0 and 1, and can therefore be interpreted as a probability. It -also has other nice properties, such as a derivative that is simple to -calculate. +

    +
    + +

    First, we set up the input data \( \boldsymbol{x} \) and the activations +\( \boldsymbol{z}_1 \) of the input layer and compute the activation function and +the pertinent outputs \( \boldsymbol{a}^1 \).

    +
    +
    -

    For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \) -is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \) -represents our activation values \( z \). We have + +

    +
    + +

    Secondly, we perform then the feed forward till we reach the output +layer and compute all \( \boldsymbol{z}_l \) of the input layer and compute the +activation function and the pertinent outputs \( \boldsymbol{a}^l \) for +\( l=2,3,\dots,L \).

    +
    +
    + + +
    +
    + +

    Thereafter we compute the ouput error \( \boldsymbol{\delta}^L \) by computing all

    $$ -P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp{(- \boldsymbol{x}})} , +\delta_j^L = f'(z_j^L)\frac{\partial {\cal C}}{\partial (a_j^L)}. +$$ +
    +
    + + +
    +
    + +

    Then we compute the back propagate error for each \( l=L-1,L-2,\dots,2 \) as

    +$$ +\delta_j^l = \sum_k \delta_k^{l+1}w_{kj}^{l+1}\sigma'(z_j^l). +$$ +
    +
    + + +
    +
    + +

    Finally, we update the weights and the biases using gradient descent for each \( l=L-1,L-2,\dots,2 \) and update the weights and biases according to the rules

    +$$ +w_{ij}^l\leftarrow = w_{ij}^l- \eta \delta_j^la_i^{l-1}, $$ -

    and

    -$$ -P(y = 1 \mid \boldsymbol{x}, \boldsymbol{\theta}) = 1 - P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) , -$$ -

    where \( y \in \{0, 1\} \) and \( \boldsymbol{\theta} \) represents the weights and biases -of our network. +$$ +b_j^l \leftarrow b_j^l-\eta \frac{\partial {\cal C}}{\partial b_j^l}=b_j^l-\eta \delta_j^l, +$$ +

    +
    + + +

    The parameter \( \eta \) is the learning parameter discussed in connection with the gradient descent methods. +Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training.

    -

    diff --git a/doc/pub/week42/html/._week42-bs070.html b/doc/pub/week42/html/._week42-bs070.html index 3356c508b..6fbaa97c1 100644 --- a/doc/pub/week42/html/._week42-bs070.html +++ b/doc/pub/week42/html/._week42-bs070.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,57 +463,42 @@ MathJax.Hub.Config({

     

     

     

    - -

    Defining the cost function

    + +

    Setting up a Multi-layer perceptron model for classification

    -

    Our cost function is given as (see the Logistic regression lectures)

    -$$ -\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n -y_i \ln[P(y_i = 0)] + (1 - y_i) \ln [1 - P(y_i = 0)] = \sum_{i=1}^n \mathcal{L}_i(\boldsymbol{\theta}) . -$$ - -

    This last equality means that we can interpret our cost function as a sum over the loss function -for each point in the dataset \( \mathcal{L}_i(\boldsymbol{\theta}) \). -The negative sign is just so that we can think about our algorithm as minimizing a positive number, rather -than maximizing a negative number. +

    We are now gong to develop an example based on the MNIST data +base. This is a classification problem and we need to use our +cross-entropy function we discussed in connection with logistic +regression. The cross-entropy defines our cost function for the +classificaton problems with neural networks.

    -

    In multiclass classification it is common to treat each integer label as a so called one-hot vector:

    - -

    \( y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) , \) and

    - -\( y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) , \) - -

    i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset (numbers from \( 0 \) to \( 9 \))..

    - -

    If \( \boldsymbol{x}_i \) is the \( i \)-th input (image), \( y_{ic} \) refers to the \( c \)-th component of the \( i \)-th -output vector \( \boldsymbol{y}_i \). -The probability of \( \boldsymbol{x}_i \) being in class \( c \) will be given by the softmax function: +

    In binary classification with two classes \( (0, 1) \) we define the +logistic/sigmoid function as the probability that a particular input +is in class \( 0 \) or \( 1 \). This is possible because the logistic +function takes any input from the real numbers and inputs a number +between 0 and 1, and can therefore be interpreted as a probability. It +also has other nice properties, such as a derivative that is simple to +calculate.

    +

    For an input \( \boldsymbol{a} \) from the hidden layer, the probability that the input \( \boldsymbol{x} \) +is in class 0 or 1 is just. We let \( \theta \) represent the unknown weights and biases to be adjusted by our equations). The variable \( x \) +represents our activation values \( z \). We have +

    $$ -P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) = \frac{\exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_c)}} -{\sum_{c'=0}^{C-1} \exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_{c'})}} , +P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) = \frac{1}{1 + \exp{(- \boldsymbol{x}})} , $$ -

    which reduces to the logistic function in the binary case. -The likelihood of this \( C \)-class classifier -is now given as: +

    and

    +$$ +P(y = 1 \mid \boldsymbol{x}, \boldsymbol{\theta}) = 1 - P(y = 0 \mid \boldsymbol{x}, \boldsymbol{\theta}) , +$$ + +

    where \( y \in \{0, 1\} \) and \( \boldsymbol{\theta} \) represents the weights and biases +of our network.

    -$$ -P(\mathcal{D} \mid \boldsymbol{\theta}) = \prod_{i=1}^n \prod_{c=0}^{C-1} [P(y_{ic} = 1)]^{y_{ic}} . -$$ - -

    Again we take the negative log-likelihood to define our cost function:

    - -$$ -\mathcal{C}(\boldsymbol{\theta}) = - \log{P(\mathcal{D} \mid \boldsymbol{\theta})}. -$$ - -

    See the logistic regression lectures for a full definition of the cost function.

    - -

    The back propagation equations need now only a small change, namely the definition of a new cost function. We are thus ready to use the same equations as before!

    @@ -533,7 +525,7 @@ $$

  • 79
  • 80
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs071.html b/doc/pub/week42/html/._week42-bs071.html index 6762e9eb5..98c4f00c4 100644 --- a/doc/pub/week42/html/._week42-bs071.html +++ b/doc/pub/week42/html/._week42-bs071.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,50 +464,56 @@ MathJax.Hub.Config({

     

     

     

    -

    Example: binary classification problem

    +

    Defining the cost function

    -

    As an example of the above, relevant for project 2 as well, let us consider a binary class. As discussed in our logistic regression lectures, we defined a cost function in terms of the parameters \( \beta \) as

    +

    Our cost function is given as (see the Logistic regression lectures)

    $$ -\mathcal{C}(\boldsymbol{\beta}) = - \sum_{i=1}^n \left(y_i\log{p(y_i \vert x_i,\boldsymbol{\beta})}+(1-y_i)\log{1-p(y_i \vert x_i,\boldsymbol{\beta})}\right), +\mathcal{C}(\boldsymbol{\theta}) = - \ln P(\mathcal{D} \mid \boldsymbol{\theta}) = - \sum_{i=1}^n +y_i \ln[P(y_i = 0)] + (1 - y_i) \ln [1 - P(y_i = 0)] = \sum_{i=1}^n \mathcal{L}_i(\boldsymbol{\theta}) . $$ -

    where we had defined the logistic (sigmoid) function

    -$$ -p(y_i =1\vert x_i,\boldsymbol{\beta})=\frac{\exp{(\beta_0+\beta_1 x_i)}}{1+\exp{(\beta_0+\beta_1 x_i)}}, -$$ - -

    and

    -$$ -p(y_i =0\vert x_i,\boldsymbol{\beta})=1-p(y_i =1\vert x_i,\boldsymbol{\beta}). -$$ - -

    The parameters \( \boldsymbol{\beta} \) were defined using a minimization method like gradient descent or Newton-Raphson's method.

    - -

    Now we replace \( x_i \) with the activation \( z_i^l \) for a given layer \( l \) and the outputs as \( y_i=a_i^l=f(z_i^l) \), with \( z_i^l \) now being a function of the weights \( w_{ij}^l \) and biases \( b_i^l \). -We have then +

    This last equality means that we can interpret our cost function as a sum over the loss function +for each point in the dataset \( \mathcal{L}_i(\boldsymbol{\theta}) \). +The negative sign is just so that we can think about our algorithm as minimizing a positive number, rather +than maximizing a negative number.

    -$$ -a_i^l = y_i = \frac{\exp{(z_i^l)}}{1+\exp{(z_i^l)}}, -$$ -

    with

    -$$ -z_i^l = \sum_{j}w_{ij}^l a_j^{l-1}+b_i^l, -$$ +

    In multiclass classification it is common to treat each integer label as a so called one-hot vector:

    -

    where the superscript \( l-1 \) indicates that these are the outputs from layer \( l-1 \). -Our cost function at the final layer \( l=L \) is now +

    \( y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) , \) and

    + +\( y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) , \) + +

    i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset (numbers from \( 0 \) to \( 9 \))..

    + +

    If \( \boldsymbol{x}_i \) is the \( i \)-th input (image), \( y_{ic} \) refers to the \( c \)-th component of the \( i \)-th +output vector \( \boldsymbol{y}_i \). +The probability of \( \boldsymbol{x}_i \) being in class \( c \) will be given by the softmax function:

    + $$ -\mathcal{C}(\boldsymbol{W}) = - \sum_{i=1}^n \left(t_i\log{a_i^L}+(1-t_i)\log{(1-a_i^L)}\right), +P(y_{ic} = 1 \mid \boldsymbol{x}_i, \boldsymbol{\theta}) = \frac{\exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_c)}} +{\sum_{c'=0}^{C-1} \exp{((\boldsymbol{a}_i^{hidden})^T \boldsymbol{w}_{c'})}} , $$ -

    where we have defined the targets \( t_i \). The derivatives of the cost function with respect to the output \( a_i^L \) are then easily calculated and we get

    +

    which reduces to the logistic function in the binary case. +The likelihood of this \( C \)-class classifier +is now given as: +

    + $$ -\frac{\partial \mathcal{C}(\boldsymbol{W})}{\partial a_i^L} = \frac{a_i^L-t_i}{a_i^L(1-a_i^L)}. +P(\mathcal{D} \mid \boldsymbol{\theta}) = \prod_{i=1}^n \prod_{c=0}^{C-1} [P(y_{ic} = 1)]^{y_{ic}} . $$ -

    In case we use another activation function than the logistic one, we need to evaluate other derivatives.

    +

    Again we take the negative log-likelihood to define our cost function:

    + +$$ +\mathcal{C}(\boldsymbol{\theta}) = - \log{P(\mathcal{D} \mid \boldsymbol{\theta})}. +$$ + +

    See the logistic regression lectures for a full definition of the cost function.

    + +

    The back propagation equations need now only a small change, namely the definition of a new cost function. We are thus ready to use the same equations as before!

    @@ -527,7 +540,7 @@ $$

  • 80
  • 81
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs072.html b/doc/pub/week42/html/._week42-bs072.html index 03661dc1a..891270c5a 100644 --- a/doc/pub/week42/html/._week42-bs072.html +++ b/doc/pub/week42/html/._week42-bs072.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,24 +464,50 @@ MathJax.Hub.Config({

     

     

     

    -

    The Softmax function

    -

    In case we employ the more general case given by the Softmax equation, we need to evaluate the derivative of the activation function with respect to the activation \( z_i^l \), that is we need

    +

    Example: binary classification problem

    + +

    As an example of the above, relevant for project 2 as well, let us consider a binary class. As discussed in our logistic regression lectures, we defined a cost function in terms of the parameters \( \beta \) as

    $$ -\frac{\partial f(z_i^l)}{\partial w_{jk}^l} = -\frac{\partial f(z_i^l)}{\partial z_j^l} \frac{\partial z_j^l}{\partial w_{jk}^l}= \frac{\partial f(z_i^l)}{\partial z_j^l}a_k^{l-1}. +\mathcal{C}(\boldsymbol{\beta}) = - \sum_{i=1}^n \left(y_i\log{p(y_i \vert x_i,\boldsymbol{\beta})}+(1-y_i)\log{1-p(y_i \vert x_i,\boldsymbol{\beta})}\right), $$ -

    For the Softmax function we have

    +

    where we had defined the logistic (sigmoid) function

    $$ -f(z_i^l) = \frac{\exp{(z_i^l)}}{\sum_{m=1}^K\exp{(z_m^l)}}. +p(y_i =1\vert x_i,\boldsymbol{\beta})=\frac{\exp{(\beta_0+\beta_1 x_i)}}{1+\exp{(\beta_0+\beta_1 x_i)}}, $$ -

    Its derivative with respect to \( z_j^l \) gives

    +

    and

    $$ -\frac{\partial f(z_i^l)}{\partial z_j^l}= f(z_i^l)\left(\delta_{ij}-f(z_j^l)\right), +p(y_i =0\vert x_i,\boldsymbol{\beta})=1-p(y_i =1\vert x_i,\boldsymbol{\beta}). $$ -

    which in case of the simply binary model reduces to having \( i=j \).

    +

    The parameters \( \boldsymbol{\beta} \) were defined using a minimization method like gradient descent or Newton-Raphson's method.

    + +

    Now we replace \( x_i \) with the activation \( z_i^l \) for a given layer \( l \) and the outputs as \( y_i=a_i^l=f(z_i^l) \), with \( z_i^l \) now being a function of the weights \( w_{ij}^l \) and biases \( b_i^l \). +We have then +

    +$$ +a_i^l = y_i = \frac{\exp{(z_i^l)}}{1+\exp{(z_i^l)}}, +$$ + +

    with

    +$$ +z_i^l = \sum_{j}w_{ij}^l a_j^{l-1}+b_i^l, +$$ + +

    where the superscript \( l-1 \) indicates that these are the outputs from layer \( l-1 \). +Our cost function at the final layer \( l=L \) is now +

    +$$ +\mathcal{C}(\boldsymbol{W}) = - \sum_{i=1}^n \left(t_i\log{a_i^L}+(1-t_i)\log{(1-a_i^L)}\right), +$$ + +

    where we have defined the targets \( t_i \). The derivatives of the cost function with respect to the output \( a_i^L \) are then easily calculated and we get

    +$$ +\frac{\partial \mathcal{C}(\boldsymbol{W})}{\partial a_i^L} = \frac{a_i^L-t_i}{a_i^L(1-a_i^L)}. +$$ + +

    In case we use another activation function than the logistic one, we need to evaluate other derivatives.

    @@ -501,7 +534,7 @@ $$

  • 81
  • 82
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs073.html b/doc/pub/week42/html/._week42-bs073.html index 2d2da4bb2..4e80380e8 100644 --- a/doc/pub/week42/html/._week42-bs073.html +++ b/doc/pub/week42/html/._week42-bs073.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,19 +463,26 @@ MathJax.Hub.Config({

     

     

     

    - -

    Developing a code for doing neural networks with back propagation

    + +

    The Softmax function

    +

    In case we employ the more general case given by the Softmax equation, we need to evaluate the derivative of the activation function with respect to the activation \( z_i^l \), that is we need

    +$$ +\frac{\partial f(z_i^l)}{\partial w_{jk}^l} = +\frac{\partial f(z_i^l)}{\partial z_j^l} \frac{\partial z_j^l}{\partial w_{jk}^l}= \frac{\partial f(z_i^l)}{\partial z_j^l}a_k^{l-1}. +$$ -

    One can identify a set of key steps when using neural networks to solve supervised learning problems:

    +

    For the Softmax function we have

    +$$ +f(z_i^l) = \frac{\exp{(z_i^l)}}{\sum_{m=1}^K\exp{(z_m^l)}}. +$$ + +

    Its derivative with respect to \( z_j^l \) gives

    +$$ +\frac{\partial f(z_i^l)}{\partial z_j^l}= f(z_i^l)\left(\delta_{ij}-f(z_j^l)\right), +$$ + +

    which in case of the simply binary model reduces to having \( i=j \).

    -
      -
    1. Collect and pre-process data
      2. -
    2. Define model and architecture
      3. -
    3. Choose cost function and optimizer
      4. -
    4. Train the model
      5. -
    5. Evaluate model performance on test data
      6. -
    6. Adjust hyperparameters (if necessary, network architecture)
      7. -

    diff --git a/doc/pub/week42/html/._week42-bs074.html b/doc/pub/week42/html/._week42-bs074.html index 499922228..0ad12f712 100644 --- a/doc/pub/week42/html/._week42-bs074.html +++ b/doc/pub/week42/html/._week42-bs074.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,118 +463,19 @@ MathJax.Hub.Config({

     

     

     

    - -

    Collect and pre-process data

    - -

    Here we will be using the MNIST dataset, which is readily available through the scikit-learn -package. You may also find it for example here. -The MNIST (Modified National Institute of Standards and Technology) database is a large database -of handwritten digits that is commonly used for training various image processing systems. -The MNIST dataset consists of 70 000 images of size \( 28\times 28 \) pixels, each labeled from 0 to 9. -The scikit-learn dataset we will use consists of a selection of 1797 images of size \( 8\times 8 \) collected and processed from this database. -

    - -

    To feed data into a feed-forward neural network we need to represent -the inputs as a design/feature matrix \( X = (n_{inputs}, n_{features}) \). Each -row represents an input, in this case a handwritten digit, and -each column represents a feature, in this case a pixel. The -correct answers, also known as labels or targets are -represented as a 1D array of integers -\( Y = (n_{inputs}) = (5, 3, 1, 8,...) \). -

    - -

    As an example, say we want to build a neural network using supervised learning to predict Body-Mass Index (BMI) from -measurements of height (in m) -and weight (in kg). If we have measurements of 5 people the design/feature matrix could be for example: -

    - -

    $$ X = \begin{bmatrix} -1.85 & 81\\ -1.71 & 65\\ -1.95 & 103\\ -1.55 & 42\\ -1.63 & 56 -\end{bmatrix} ,$$ -

    - -

    and the targets would be:

    - -

    $$ Y = (23.7, 22.2, 27.1, 17.5, 21.1) $$

    - -

    Since each input image is a 2D matrix, we need to flatten the image -(i.e. "unravel" the 2D matrix into a 1D array) to turn the data into a -design/feature matrix. This means we lose all spatial information in the -image, such as locality and translational invariance. More complicated -architectures such as Convolutional Neural Networks can take advantage -of such information, and are most commonly applied when analyzing -images. -

    - - - -
    -
    -
    -
    -
    - 
    # import necessary packages
-import numpy as np
-import matplotlib.pyplot as plt
-from sklearn import datasets
-
-
-# ensure the same random numbers appear every time
-np.random.seed(0)
-
-# display images in notebook
-%matplotlib inline
-plt.rcParams['figure.figsize'] = (12,12)
-
-
-# download MNIST dataset
-digits = datasets.load_digits()
-
-# define inputs and labels
-inputs = digits.images
-labels = digits.target
-
-print("inputs = (n_inputs, pixel_width, pixel_height) = " + str(inputs.shape))
-print("labels = (n_inputs) = " + str(labels.shape))
-
-
-# flatten the image
-# the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64
-n_inputs = len(inputs)
-inputs = inputs.reshape(n_inputs, -1)
-print("X = (n_inputs, n_features) = " + str(inputs.shape))
-
-
-# choose some random images to display
-indices = np.arange(n_inputs)
-random_indices = np.random.choice(indices, size=5)
-
-for i, image in enumerate(digits.images[random_indices]):
-    plt.subplot(1, 5, i+1)
-    plt.axis('off')
-    plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
-    plt.title("Label: %d" % digits.target[random_indices[i]])
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    + +

    Developing a code for doing neural networks with back propagation

    +

    One can identify a set of key steps when using neural networks to solve supervised learning problems:

    +
      +
    1. Collect and pre-process data
      2. +
    2. Define model and architecture
      3. +
    3. Choose cost function and optimizer
      4. +
    4. Train the model
      5. +
    5. Evaluate model performance on test data
      6. +
    6. Adjust hyperparameters (if necessary, network architecture)
      7. +

    diff --git a/doc/pub/week42/html/._week42-bs075.html b/doc/pub/week42/html/._week42-bs075.html index 1288b7b87..2d8b05efd 100644 --- a/doc/pub/week42/html/._week42-bs075.html +++ b/doc/pub/week42/html/._week42-bs075.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,17 +464,50 @@ MathJax.Hub.Config({

     

     

     

    -

    Train and test datasets

    +

    Collect and pre-process data

    -

    Performing analysis before partitioning the dataset is a major error, that can lead to incorrect conclusions.

    +

    Here we will be using the MNIST dataset, which is readily available through the scikit-learn +package. You may also find it for example here. +The MNIST (Modified National Institute of Standards and Technology) database is a large database +of handwritten digits that is commonly used for training various image processing systems. +The MNIST dataset consists of 70 000 images of size \( 28\times 28 \) pixels, each labeled from 0 to 9. +The scikit-learn dataset we will use consists of a selection of 1797 images of size \( 8\times 8 \) collected and processed from this database. +

    -

    We will reserve \( 80 \% \) of our dataset for training and \( 20 \% \) for testing.

    +

    To feed data into a feed-forward neural network we need to represent +the inputs as a design/feature matrix \( X = (n_{inputs}, n_{features}) \). Each +row represents an input, in this case a handwritten digit, and +each column represents a feature, in this case a pixel. The +correct answers, also known as labels or targets are +represented as a 1D array of integers +\( Y = (n_{inputs}) = (5, 3, 1, 8,...) \). +

    -

    It is important that the train and test datasets are drawn randomly from our dataset, to ensure -no bias in the sampling. -Say you are taking measurements of weather data to predict the weather in the coming 5 days. -You don't want to train your model on measurements taken from the hours 00.00 to 12.00, and then test it on data -collected from 12.00 to 24.00. +

    As an example, say we want to build a neural network using supervised learning to predict Body-Mass Index (BMI) from +measurements of height (in m) +and weight (in kg). If we have measurements of 5 people the design/feature matrix could be for example: +

    + +

    $$ X = \begin{bmatrix} +1.85 & 81\\ +1.71 & 65\\ +1.95 & 103\\ +1.55 & 42\\ +1.63 & 56 +\end{bmatrix} ,$$ +

    + +

    and the targets would be:

    + +

    $$ Y = (23.7, 22.2, 27.1, 17.5, 21.1) $$

    + +

    Since each input image is a 2D matrix, we need to flatten the image +(i.e. "unravel" the 2D matrix into a 1D array) to turn the data into a +design/feature matrix. This means we lose all spatial information in the +image, such as locality and translational invariance. More complicated +architectures such as Convolutional Neural Networks can take advantage +of such information, and are most commonly applied when analyzing +images.

    @@ -477,33 +517,48 @@ collected from 12.00 to 24.00.
    - 
    from sklearn.model_selection import train_test_split
+  
    # import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn import datasets
 
-# one-liner from scikit-learn library
-train_size = 0.8
-test_size = 1 - train_size
-X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
-                                                    test_size=test_size)
 
-# equivalently in numpy
-def train_test_split_numpy(inputs, labels, train_size, test_size):
-    n_inputs = len(inputs)
-    inputs_shuffled = inputs.copy()
-    labels_shuffled = labels.copy()
-    
-    np.random.shuffle(inputs_shuffled)
-    np.random.shuffle(labels_shuffled)
-    
-    train_end = int(n_inputs*train_size)
-    X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]
-    Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]
-    
-    return X_train, X_test, Y_train, Y_test
+# ensure the same random numbers appear every time
+np.random.seed(0)
 
-#X_train, X_test, Y_train, Y_test = train_test_split_numpy(inputs, labels, train_size, test_size)
+# display images in notebook
+%matplotlib inline
+plt.rcParams['figure.figsize'] = (12,12)
 
-print("Number of training images: " + str(len(X_train)))
-print("Number of test images: " + str(len(X_test)))
+
+# download MNIST dataset
+digits = datasets.load_digits()
+
+# define inputs and labels
+inputs = digits.images
+labels = digits.target
+
+print("inputs = (n_inputs, pixel_width, pixel_height) = " + str(inputs.shape))
+print("labels = (n_inputs) = " + str(labels.shape))
+
+
+# flatten the image
+# the value -1 means dimension is inferred from the remaining dimensions: 8x8 = 64
+n_inputs = len(inputs)
+inputs = inputs.reshape(n_inputs, -1)
+print("X = (n_inputs, n_features) = " + str(inputs.shape))
+
+
+# choose some random images to display
+indices = np.arange(n_inputs)
+random_indices = np.random.choice(indices, size=5)
+
+for i, image in enumerate(digits.images[random_indices]):
+    plt.subplot(1, 5, i+1)
+    plt.axis('off')
+    plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
+    plt.title("Label: %d" % digits.target[random_indices[i]])
+plt.show()
    @@ -545,7 +600,7 @@ X_train, X_test, Y_train, Y_test = train_tes
  • 84
  • 85
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs076.html b/doc/pub/week42/html/._week42-bs076.html index 65ea8d04d..bfbcf171a 100644 --- a/doc/pub/week42/html/._week42-bs076.html +++ b/doc/pub/week42/html/._week42-bs076.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,46 +464,68 @@ MathJax.Hub.Config({

     

     

     

    -

    Define model and architecture

    +

    Train and test datasets

    -

    Our simple feed-forward neural network will consist of an input layer, a single hidden layer and an output layer. The activation \( y \) of each neuron is a weighted sum of inputs, passed through an activation function. In case of the simple perceptron model we have

    +

    Performing analysis before partitioning the dataset is a major error, that can lead to incorrect conclusions.

    -

    $$ z = \sum_{i=1}^n w_i a_i ,$$

    +

    We will reserve \( 80 \% \) of our dataset for training and \( 20 \% \) for testing.

    -

    $$ y = f(z) ,$$

    - -

    where \( f \) is the activation function, \( a_i \) represents input from neuron \( i \) in the preceding layer -and \( w_i \) is the weight to input \( i \). -The activation of the neurons in the input layer is just the features (e.g. a pixel value). +

    It is important that the train and test datasets are drawn randomly from our dataset, to ensure +no bias in the sampling. +Say you are taking measurements of weather data to predict the weather in the coming 5 days. +You don't want to train your model on measurements taken from the hours 00.00 to 12.00, and then test it on data +collected from 12.00 to 24.00.

    -

    The simplest activation function for a neuron is the Heaviside function:

    -

    $$ f(z) = -\begin{cases} -1, & z > 0\\ -0, & \text{otherwise} -\end{cases} -$$ -

    + +
    +
    +
    +
    +
    + 
    from sklearn.model_selection import train_test_split
 
-
    A feed-forward neural network with this activation is known as a perceptron.  
-For a binary classifier (i.e. two classes, 0 or 1, dog or not-dog) we can also use this in our output layer.  
-This activation can be generalized to \( k \) classes (using e.g. the one-against-all strategy), 
-and we call these architectures multiclass perceptrons.  
-
    
+# one-liner from scikit-learn library
+train_size = 0.8
+test_size = 1 - train_size
+X_train, X_test, Y_train, Y_test = train_test_split(inputs, labels, train_size=train_size,
+                                                    test_size=test_size)
 
-
    However, it is now common to use the terms Single Layer Perceptron (SLP) (1 hidden layer) and  
-Multilayer Perceptron (MLP) (2 or more hidden layers) to refer to feed-forward neural networks with any activation function.  
-
    
+# equivalently in numpy
+def train_test_split_numpy(inputs, labels, train_size, test_size):
+    n_inputs = len(inputs)
+    inputs_shuffled = inputs.copy()
+    labels_shuffled = labels.copy()
+    
+    np.random.shuffle(inputs_shuffled)
+    np.random.shuffle(labels_shuffled)
+    
+    train_end = int(n_inputs*train_size)
+    X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]
+    Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]
+    
+    return X_train, X_test, Y_train, Y_test
 
-
    Typical choices for activation functions include the sigmoid function, hyperbolic tangent, and Rectified Linear Unit (ReLU).  
-We will be using the sigmoid function \( \sigma(x) \):  
-
    
+#X_train, X_test, Y_train, Y_test = train_test_split_numpy(inputs, labels, train_size, test_size)
 
-
    $$ f(x) = \sigma(x) = \frac{1}{1 + e^{-x}} ,$$
    
+print("Number of training images: " + str(len(X_train)))
+print("Number of test images: " + str(len(X_test)))
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    which is inspired by probability theory (see logistic regression) and was most commonly used until about 2011. See the discussion below concerning other activation functions.

    @@ -523,7 +552,7 @@ We will be using the sigmoid function \( \sigma(x) \):

  • 85
  • 86
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs077.html b/doc/pub/week42/html/._week42-bs077.html index 661d5e2c9..13644fe15 100644 --- a/doc/pub/week42/html/._week42-bs077.html +++ b/doc/pub/week42/html/._week42-bs077.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,47 +463,48 @@ MathJax.Hub.Config({

     

     

     

    - -

    Layers

    + +

    Define model and architecture

    -
      -
    • Input
      • -
    -

    Since each input image has 8x8 = 64 pixels or features, we have an input layer of 64 neurons.

    +

    Our simple feed-forward neural network will consist of an input layer, a single hidden layer and an output layer. The activation \( y \) of each neuron is a weighted sum of inputs, passed through an activation function. In case of the simple perceptron model we have

    -
      -
    • Hidden layer
      • -
    -

    We will use 50 neurons in the hidden layer receiving input from the neurons in the input layer. -Since each neuron in the hidden layer is connected to the 64 inputs we have 64x50 = 3200 weights to the hidden layer. +

    $$ z = \sum_{i=1}^n w_i a_i ,$$

    + +

    $$ y = f(z) ,$$

    + +

    where \( f \) is the activation function, \( a_i \) represents input from neuron \( i \) in the preceding layer +and \( w_i \) is the weight to input \( i \). +The activation of the neurons in the input layer is just the features (e.g. a pixel value).

    -
      -
    • Output
      • -
    -

    If we were building a binary classifier, it would be sufficient with a single neuron in the output layer, -which could output 0 or 1 according to the Heaviside function. This would be an example of a hard classifier, meaning it outputs the class of the input directly. However, if we are dealing with noisy data it is often beneficial to use a soft classifier, which outputs the probability of being in class 0 or 1. +

    The simplest activation function for a neuron is the Heaviside function:

    + +

    $$ f(z) = +\begin{cases} +1, & z > 0\\ +0, & \text{otherwise} +\end{cases} +$$

    -

    For a soft binary classifier, we could use a single neuron and interpret the output as either being the probability of being in class 0 or the probability of being in class 1. Alternatively we could use 2 neurons, and interpret each neuron as the probability of being in each class.

    - -

    Since we are doing multiclass classification, with 10 categories, it is natural to use 10 neurons in the output layer. We number the neurons \( j = 0,1,...,9 \). The activation of each output neuron \( j \) will be according to the softmax function:

    - -

    $$ P(\text{class \( j \)} \mid \text{input \( \boldsymbol{a} \)}) = \frac{\exp{(\boldsymbol{a}^T \boldsymbol{w}_j)}} -{\sum_{c=0}^{9} \exp{(\boldsymbol{a}^T \boldsymbol{w}_c)}} ,$$ +

    A feed-forward neural network with this activation is known as a perceptron. +For a binary classifier (i.e. two classes, 0 or 1, dog or not-dog) we can also use this in our output layer. +This activation can be generalized to \( k \) classes (using e.g. the one-against-all strategy), +and we call these architectures multiclass perceptrons.

    -

    i.e. each neuron \( j \) outputs the probability of being in class \( j \) given an input from the hidden layer \( \boldsymbol{a} \), with \( \boldsymbol{w}_j \) the weights of neuron \( j \) to the inputs. -The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1. -The exponent is just the weighted sum of inputs as before: +

    However, it is now common to use the terms Single Layer Perceptron (SLP) (1 hidden layer) and +Multilayer Perceptron (MLP) (2 or more hidden layers) to refer to feed-forward neural networks with any activation function.

    -

    $$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$

    - -

    Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500 -weights to the output layer. +

    Typical choices for activation functions include the sigmoid function, hyperbolic tangent, and Rectified Linear Unit (ReLU). +We will be using the sigmoid function \( \sigma(x) \):

    +

    $$ f(x) = \sigma(x) = \frac{1}{1 + e^{-x}} ,$$

    + +

    which is inspired by probability theory (see logistic regression) and was most commonly used until about 2011. See the discussion below concerning other activation functions.

    +

      @@ -522,7 +530,7 @@ weights to the output layer.
    • 86
    • 87
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs078.html b/doc/pub/week42/html/._week42-bs078.html index c7264104f..22d0a88ad 100644 --- a/doc/pub/week42/html/._week42-bs078.html +++ b/doc/pub/week42/html/._week42-bs078.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,56 +464,45 @@ MathJax.Hub.Config({

     

     

     

    -

    Weights and biases

    +

    Layers

    -

    Typically weights are initialized with small values distributed around zero, drawn from a uniform -or normal distribution. Setting all weights to zero means all neurons give the same output, making the network useless. +

      +
    • Input
      • +
    +

    Since each input image has 8x8 = 64 pixels or features, we have an input layer of 64 neurons.

    + +
      +
    • Hidden layer
      • +
    +

    We will use 50 neurons in the hidden layer receiving input from the neurons in the input layer. +Since each neuron in the hidden layer is connected to the 64 inputs we have 64x50 = 3200 weights to the hidden layer.

    -

    Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range -of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \): +

      +
    • Output
      • +
    +

    If we were building a binary classifier, it would be sufficient with a single neuron in the output layer, +which could output 0 or 1 according to the Heaviside function. This would be an example of a hard classifier, meaning it outputs the class of the input directly. However, if we are dealing with noisy data it is often beneficial to use a soft classifier, which outputs the probability of being in class 0 or 1.

    -

    $$ z_j = \sum_{i=1}^n w_ {ij} a_i + b_j.$$

    +

    For a soft binary classifier, we could use a single neuron and interpret the output as either being the probability of being in class 0 or the probability of being in class 1. Alternatively we could use 2 neurons, and interpret each neuron as the probability of being in each class.

    -

    The bias weights \( \boldsymbol{b} \) are often initialized to zero, but a small value like \( 0.01 \) ensures all neurons have some output which can be backpropagated in the first training cycle.

    +

    Since we are doing multiclass classification, with 10 categories, it is natural to use 10 neurons in the output layer. We number the neurons \( j = 0,1,...,9 \). The activation of each output neuron \( j \) will be according to the softmax function:

    - -
    -
    -
    -
    -
    - 
    # building our neural network
+
    $$ P(\text{class \( j \)} \mid \text{input \( \boldsymbol{a} \)}) = \frac{\exp{(\boldsymbol{a}^T \boldsymbol{w}_j)}}
+{\sum_{c=0}^{9} \exp{(\boldsymbol{a}^T \boldsymbol{w}_c)}} ,$$  
+
    
 
-n_inputs, n_features = X_train.shape
-n_hidden_neurons = 50
-n_categories = 10
+
    i.e. each neuron \( j \) outputs the probability of being in class \( j \) given an input from the hidden layer \( \boldsymbol{a} \), with \( \boldsymbol{w}_j \) the weights of neuron \( j \) to the inputs.  
+The denominator is a normalization factor to ensure the outputs (probabilities) sum up to 1.  
+The exponent is just the weighted sum of inputs as before:  
+
    
 
-# we make the weights normally distributed using numpy.random.randn
-
-# weights and bias in the hidden layer
-hidden_weights = np.random.randn(n_features, n_hidden_neurons)
-hidden_bias = np.zeros(n_hidden_neurons) + 0.01
-
-# weights and bias in the output layer
-output_weights = np.random.randn(n_hidden_neurons, n_categories)
-output_bias = np.zeros(n_categories) + 0.01
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    $$ z_j = \sum_{i=1}^n w_ {ij} a_i+b_j.$$

    +

    Since each neuron in the output layer is connected to the 50 inputs from the hidden layer we have 50x10 = 500 +weights to the output layer. +

    @@ -533,7 +529,7 @@ output_bias = np87

  • 88
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs079.html b/doc/pub/week42/html/._week42-bs079.html index beb29d2f5..54871322e 100644 --- a/doc/pub/week42/html/._week42-bs079.html +++ b/doc/pub/week42/html/._week42-bs079.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,29 +463,58 @@ MathJax.Hub.Config({

     

     

     

    - -

    Feed-forward pass

    + +

    Weights and biases

    -

    Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories. -For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \): +

    Typically weights are initialized with small values distributed around zero, drawn from a uniform +or normal distribution. Setting all weights to zero means all neurons give the same output, making the network useless.

    -

    $$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$

    - -

    this is then passed through our activation function

    - -

    $$ a_{j}^{l} = f(z_{j}^{l}) .$$

    - -

    We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:

    - -

    $$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$

    - -

    Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:

    - -

    $$ a_{j}^{L} = \frac{\exp{(z_j^{L})}} -{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$ +

    Adding a bias value to the weighted sum of inputs allows the neural network to represent a greater range +of values. Without it, any input with the value 0 will be mapped to zero (before being passed through the activation). The bias unit has an output of 1, and a weight to each neuron \( j \), \( b_j \):

    +

    $$ z_j = \sum_{i=1}^n w_ {ij} a_i + b_j.$$

    + +

    The bias weights \( \boldsymbol{b} \) are often initialized to zero, but a small value like \( 0.01 \) ensures all neurons have some output which can be backpropagated in the first training cycle.

    + + +
    +
    +
    +
    +
    + 
    # building our neural network
+
+n_inputs, n_features = X_train.shape
+n_hidden_neurons = 50
+n_categories = 10
+
+# we make the weights normally distributed using numpy.random.randn
+
+# weights and bias in the hidden layer
+hidden_weights = np.random.randn(n_features, n_hidden_neurons)
+hidden_bias = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+output_weights = np.random.randn(n_hidden_neurons, n_categories)
+output_bias = np.zeros(n_categories) + 0.01
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +

      @@ -504,7 +540,7 @@ For each input image we calculate a weighted sum of input features (pixel values
    • 88
    • 89
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs080.html b/doc/pub/week42/html/._week42-bs080.html index d6899189d..e9ea00896 100644 --- a/doc/pub/week42/html/._week42-bs080.html +++ b/doc/pub/week42/html/._week42-bs080.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,96 +463,29 @@ MathJax.Hub.Config({

     

     

     

    - -

    Matrix multiplications

    + +

    Feed-forward pass

    -

    Since our data has the dimensions \( X = (n_{inputs}, n_{features}) \) and our weights to the hidden -layer have the dimensions -\( W_{hidden} = (n_{features}, n_{hidden}) \), -we can easily feed the network all our training data in one go by taking the matrix product +

    Denote \( F \) the number of features, \( H \) the number of hidden neurons and \( C \) the number of categories. +For each input image we calculate a weighted sum of input features (pixel values) to each neuron \( j \) in the hidden layer \( l \):

    -

    $$ X W^{h} = (n_{inputs}, n_{hidden}),$$

    +

    $$ z_{j}^{l} = \sum_{i=1}^{F} w_{ij}^{l} x_i + b_{j}^{l},$$

    -

    and obtain a matrix that holds the weighted sum of inputs to the hidden layer -for each input image and each hidden neuron. -We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \): +

    this is then passed through our activation function

    + +

    $$ a_{j}^{l} = f(z_{j}^{l}) .$$

    + +

    We calculate a weighted sum of inputs (activations in the hidden layer) to each neuron \( j \) in the output layer:

    + +

    $$ z_{j}^{L} = \sum_{i=1}^{H} w_{ij}^{L} a_{i}^{l} + b_{j}^{L}.$$

    + +

    Finally we calculate the output of neuron \( j \) in the output layer using the softmax function:

    + +

    $$ a_{j}^{L} = \frac{\exp{(z_j^{L})}} +{\sum_{c=0}^{C-1} \exp{(z_c^{L})}} .$$

    -

    $$ \boldsymbol{z}^{l} = \boldsymbol{X} \boldsymbol{W}^{l} + \boldsymbol{b}^{l} ,$$

    - -

    meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image. -This is then passed through the activation: -

    - -

    $$ \boldsymbol{a}^{l} = f(\boldsymbol{z}^l) .$$

    - -

    This is fed to the output layer:

    - -

    $$ \boldsymbol{z}^{L} = \boldsymbol{a}^{L} \boldsymbol{W}^{L} + \boldsymbol{b}^{L} .$$

    - -

    Finally we receive our output values for each image and each category by passing it through the softmax function:

    - -

    $$ output = softmax (\boldsymbol{z}^{L}) = (n_{inputs}, n_{categories}) .$$

    - - - -
    -
    -
    -
    -
    - 
    # setup the feed-forward pass, subscript h = hidden layer
-
-def sigmoid(x):
-    return 1/(1 + np.exp(-x))
-
-def feed_forward(X):
-    # weighted sum of inputs to the hidden layer
-    z_h = np.matmul(X, hidden_weights) + hidden_bias
-    # activation in the hidden layer
-    a_h = sigmoid(z_h)
-    
-    # weighted sum of inputs to the output layer
-    z_o = np.matmul(a_h, output_weights) + output_bias
-    # softmax output
-    # axis 0 holds each input and axis 1 the probabilities of each category
-    exp_term = np.exp(z_o)
-    probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-    
-    return probabilities
-
-probabilities = feed_forward(X_train)
-print("probabilities = (n_inputs, n_categories) = " + str(probabilities.shape))
-print("probability that image 0 is in category 0,1,2,...,9 = \n" + str(probabilities[0]))
-print("probabilities sum up to: " + str(probabilities[0].sum()))
-print()
-
-# we obtain a prediction by taking the class with the highest likelihood
-def predict(X):
-    probabilities = feed_forward(X)
-    return np.argmax(probabilities, axis=1)
-
-predictions = predict(X_train)
-print("predictions = (n_inputs) = " + str(predictions.shape))
-print("prediction for image 0: " + str(predictions[0]))
-print("correct label for image 0: " + str(Y_train[0]))
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

      @@ -571,7 +511,7 @@ predictions = predict(X_train)
    • 89
    • 90
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs081.html b/doc/pub/week42/html/._week42-bs081.html index 722d2a04c..77374d728 100644 --- a/doc/pub/week42/html/._week42-bs081.html +++ b/doc/pub/week42/html/._week42-bs081.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,33 +463,95 @@ MathJax.Hub.Config({

     

     

     

    - -

    Choose cost function and optimizer

    + +

    Matrix multiplications

    -

    To measure how well our neural network is doing we need to introduce a cost function. -We will call the function that gives the error of a single sample output the loss function, and the function -that gives the total error of our network across all samples the cost function. -A typical choice for multiclass classification is the cross-entropy loss, also known as the negative log likelihood. +

    Since our data has the dimensions \( X = (n_{inputs}, n_{features}) \) and our weights to the hidden +layer have the dimensions +\( W_{hidden} = (n_{features}, n_{hidden}) \), +we can easily feed the network all our training data in one go by taking the matrix product

    -

    In multiclass classification it is common to treat each integer label as a so called one-hot vector:

    +

    $$ X W^{h} = (n_{inputs}, n_{hidden}),$$

    -

    $$ y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,$$

    - -

    $$ y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,$$

    - -

    i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset.

    - -

    Let \( y_{ic} \) denote the \( c \)-th component of the \( i \)-th one-hot vector. -We define the cost function \( \mathcal{C} \) as a sum over the cross-entropy loss for each point \( \boldsymbol{x}_i \) in the dataset. +

    and obtain a matrix that holds the weighted sum of inputs to the hidden layer +for each input image and each hidden neuron. +We also add the bias to obtain a matrix of weighted sums to the hidden layer \( Z^{h} \):

    -

    In the one-hot representation only one of the terms in the loss function is non-zero, namely the -probability of the correct category \( c' \) -(i.e. the category \( c' \) such that \( y_{ic'} = 1 \)). This means that the cross entropy loss only punishes you for how wrong -you got the correct label. The probability of category \( c \) is given by the softmax function. The vector \( \boldsymbol{\theta} \) represents the parameters of our network, i.e. all the weights and biases. +

    $$ \boldsymbol{z}^{l} = \boldsymbol{X} \boldsymbol{W}^{l} + \boldsymbol{b}^{l} ,$$

    + +

    meaning the same bias (1D array with size equal number of hidden neurons) is added to each input image. +This is then passed through the activation:

    +

    $$ \boldsymbol{a}^{l} = f(\boldsymbol{z}^l) .$$

    + +

    This is fed to the output layer:

    + +

    $$ \boldsymbol{z}^{L} = \boldsymbol{a}^{L} \boldsymbol{W}^{L} + \boldsymbol{b}^{L} .$$

    + +

    Finally we receive our output values for each image and each category by passing it through the softmax function:

    + +

    $$ output = softmax (\boldsymbol{z}^{L}) = (n_{inputs}, n_{categories}) .$$

    + + + +
    +
    +
    +
    +
    + 
    # setup the feed-forward pass, subscript h = hidden layer
+
+def sigmoid(x):
+    return 1/(1 + np.exp(-x))
+
+def feed_forward(X):
+    # weighted sum of inputs to the hidden layer
+    z_h = np.matmul(X, hidden_weights) + hidden_bias
+    # activation in the hidden layer
+    a_h = sigmoid(z_h)
+    
+    # weighted sum of inputs to the output layer
+    z_o = np.matmul(a_h, output_weights) + output_bias
+    # softmax output
+    # axis 0 holds each input and axis 1 the probabilities of each category
+    exp_term = np.exp(z_o)
+    probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+    
+    return probabilities
+
+probabilities = feed_forward(X_train)
+print("probabilities = (n_inputs, n_categories) = " + str(probabilities.shape))
+print("probability that image 0 is in category 0,1,2,...,9 = \n" + str(probabilities[0]))
+print("probabilities sum up to: " + str(probabilities[0].sum()))
+print()
+
+# we obtain a prediction by taking the class with the highest likelihood
+def predict(X):
+    probabilities = feed_forward(X)
+    return np.argmax(probabilities, axis=1)
+
+predictions = predict(X_train)
+print("predictions = (n_inputs) = " + str(predictions.shape))
+print("prediction for image 0: " + str(predictions[0]))
+print("correct label for image 0: " + str(Y_train[0]))
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    @@ -509,7 +578,7 @@ you got the correct label. The probability of category \( c \) is given by the s

  • 90
  • 91
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs082.html b/doc/pub/week42/html/._week42-bs082.html index 4c89bad1b..3fb3af7d0 100644 --- a/doc/pub/week42/html/._week42-bs082.html +++ b/doc/pub/week42/html/._week42-bs082.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,39 +464,32 @@ MathJax.Hub.Config({

     

     

     

    -

    Optimizing the cost function

    +

    Choose cost function and optimizer

    -

    The network is trained by finding the weights and biases that minimize the cost function. One of the most widely used classes of methods is gradient descent and its generalizations. The idea behind gradient descent -is simply to adjust the weights in the direction where the gradient of the cost function is large and negative. This ensures we flow toward a local minimum of the cost function. -Each parameter \( \theta \) is iteratively adjusted according to the rule +

    To measure how well our neural network is doing we need to introduce a cost function. +We will call the function that gives the error of a single sample output the loss function, and the function +that gives the total error of our network across all samples the cost function. +A typical choice for multiclass classification is the cross-entropy loss, also known as the negative log likelihood.

    -

    $$ \theta_{i+1} = \theta_i - \eta \nabla \mathcal{C}(\theta_i) ,$$

    +

    In multiclass classification it is common to treat each integer label as a so called one-hot vector:

    -

    where \( \eta \) is known as the learning rate, which controls how big a step we take towards the minimum. -This update can be repeated for any number of iterations, or until we are satisfied with the result. +

    $$ y = 5 \quad \rightarrow \quad \boldsymbol{y} = (0, 0, 0, 0, 0, 1, 0, 0, 0, 0) ,$$

    + +

    $$ y = 1 \quad \rightarrow \quad \boldsymbol{y} = (0, 1, 0, 0, 0, 0, 0, 0, 0, 0) ,$$

    + +

    i.e. a binary bit string of length \( C \), where \( C = 10 \) is the number of classes in the MNIST dataset.

    + +

    Let \( y_{ic} \) denote the \( c \)-th component of the \( i \)-th one-hot vector. +We define the cost function \( \mathcal{C} \) as a sum over the cross-entropy loss for each point \( \boldsymbol{x}_i \) in the dataset.

    -

    A simple and effective improvement is a variant called Batch Gradient Descent. -Instead of calculating the gradient on the whole dataset, we calculate an approximation of the gradient -on a subset of the data called a minibatch. -If there are \( N \) data points and we have a minibatch size of \( M \), the total number of batches -is \( N/M \). -We denote each minibatch \( B_k \), with \( k = 1, 2,...,N/M \). The gradient then becomes: +

    In the one-hot representation only one of the terms in the loss function is non-zero, namely the +probability of the correct category \( c' \) +(i.e. the category \( c' \) such that \( y_{ic'} = 1 \)). This means that the cross entropy loss only punishes you for how wrong +you got the correct label. The probability of category \( c \) is given by the softmax function. The vector \( \boldsymbol{\theta} \) represents the parameters of our network, i.e. all the weights and biases.

    -

    $$ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad -\frac{1}{M} \sum_{i \in B_k} \nabla \mathcal{L}_i(\theta) ,$$ -

    - -

    i.e. instead of averaging the loss over the entire dataset, we average over a minibatch.

    - -

    This has two important benefits:

    -
      -
    1. Introducing stochasticity decreases the chance that the algorithm becomes stuck in a local minima.
      2. -
    2. It significantly speeds up the calculation, since we do not have to use the entire dataset to calculate the gradient.
      3. -
    -

    The various optmization methods, with codes and algorithms, are discussed in our lectures on Gradient descent approaches.

    @@ -516,7 +516,7 @@ We denote each minibatch \( B_k \), with \( k = 1, 2,...,N/M \). The gradient th

  • 91
  • 92
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs083.html b/doc/pub/week42/html/._week42-bs083.html index ce9dca8a4..6475ce415 100644 --- a/doc/pub/week42/html/._week42-bs083.html +++ b/doc/pub/week42/html/._week42-bs083.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,36 +463,41 @@ MathJax.Hub.Config({

     

     

     

    - -

    Regularization

    + +

    Optimizing the cost function

    -

    It is common to add an extra term to the cost function, proportional -to the size of the weights. This is equivalent to constraining the -size of the weights, so that they do not grow out of control. -Constraining the size of the weights means that the weights cannot -grow arbitrarily large to fit the training data, and in this way -reduces overfitting. +

    The network is trained by finding the weights and biases that minimize the cost function. One of the most widely used classes of methods is gradient descent and its generalizations. The idea behind gradient descent +is simply to adjust the weights in the direction where the gradient of the cost function is large and negative. This ensures we flow toward a local minimum of the cost function. +Each parameter \( \theta \) is iteratively adjusted according to the rule

    -

    We will measure the size of the weights using the so called L2-norm, meaning our cost function becomes:

    +

    $$ \theta_{i+1} = \theta_i - \eta \nabla \mathcal{C}(\theta_i) ,$$

    -

    $$ \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \mathcal{L}_i(\theta) \quad \rightarrow \quad -\frac{1}{N} \sum_{i=1}^N \mathcal{L}_i(\theta) + \lambda \lvert \lvert \boldsymbol{w} \rvert \rvert_2^2 -= \frac{1}{N} \sum_{i=1}^N \mathcal{L}(\theta) + \lambda \sum_{ij} w_{ij}^2,$$ +

    where \( \eta \) is known as the learning rate, which controls how big a step we take towards the minimum. +This update can be repeated for any number of iterations, or until we are satisfied with the result.

    -

    i.e. we sum up all the weights squared. The factor \( \lambda \) is known as a regularization parameter.

    - -

    In order to train the model, we need to calculate the derivative of -the cost function with respect to every bias and weight in the -network. In total our network has \( (64 + 1)\times 50=3250 \) weights in -the hidden layer and \( (50 + 1)\times 10=510 \) weights to the output -layer (\( +1 \) for the bias), and the gradient must be calculated for -every parameter. We use the backpropagation algorithm discussed -above. This is a clever use of the chain rule that allows us to -calculate the gradient efficently. +

    A simple and effective improvement is a variant called Batch Gradient Descent. +Instead of calculating the gradient on the whole dataset, we calculate an approximation of the gradient +on a subset of the data called a minibatch. +If there are \( N \) data points and we have a minibatch size of \( M \), the total number of batches +is \( N/M \). +We denote each minibatch \( B_k \), with \( k = 1, 2,...,N/M \). The gradient then becomes:

    +

    $$ \nabla \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \nabla \mathcal{L}_i(\theta) \quad \rightarrow \quad +\frac{1}{M} \sum_{i \in B_k} \nabla \mathcal{L}_i(\theta) ,$$ +

    + +

    i.e. instead of averaging the loss over the entire dataset, we average over a minibatch.

    + +

    This has two important benefits:

    +
      +
    1. Introducing stochasticity decreases the chance that the algorithm becomes stuck in a local minima.
      2. +
    2. It significantly speeds up the calculation, since we do not have to use the entire dataset to calculate the gradient.
      3. +
    +

    The various optmization methods, with codes and algorithms, are discussed in our lectures on Gradient descent approaches.

    +

      @@ -511,7 +523,7 @@ calculate the gradient efficently.
    • 92
    • 93
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs084.html b/doc/pub/week42/html/._week42-bs084.html index 9eb737007..9795d92ba 100644 --- a/doc/pub/week42/html/._week42-bs084.html +++ b/doc/pub/week42/html/._week42-bs084.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -456,133 +463,36 @@ MathJax.Hub.Config({

     

     

     

    - -

    Matrix multiplication

    + +

    Regularization

    -

    To more efficently train our network these equations are implemented using matrix operations. -The error in the output layer is calculated simply as, with \( \boldsymbol{t} \) being our targets, +

    It is common to add an extra term to the cost function, proportional +to the size of the weights. This is equivalent to constraining the +size of the weights, so that they do not grow out of control. +Constraining the size of the weights means that the weights cannot +grow arbitrarily large to fit the training data, and in this way +reduces overfitting.

    -

    $$ \delta_L = \boldsymbol{t} - \boldsymbol{y} = (n_{inputs}, n_{categories}) .$$

    +

    We will measure the size of the weights using the so called L2-norm, meaning our cost function becomes:

    -

    The gradient for the output weights is calculated as

    - -

    $$ \nabla W_{L} = \boldsymbol{a}^T \delta_L = (n_{hidden}, n_{categories}) ,$$

    - -

    where \( \boldsymbol{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input. -Since we are going backwards we have to transpose the activation matrix. +

    $$ \mathcal{C}(\theta) = \frac{1}{N} \sum_{i=1}^N \mathcal{L}_i(\theta) \quad \rightarrow \quad +\frac{1}{N} \sum_{i=1}^N \mathcal{L}_i(\theta) + \lambda \lvert \lvert \boldsymbol{w} \rvert \rvert_2^2 += \frac{1}{N} \sum_{i=1}^N \mathcal{L}(\theta) + \lambda \sum_{ij} w_{ij}^2,$$

    -

    The gradient with respect to the output bias is then

    +

    i.e. we sum up all the weights squared. The factor \( \lambda \) is known as a regularization parameter.

    -

    $$ \nabla \boldsymbol{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$

    - -

    The error in the hidden layer is

    - -

    $$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$

    - -

    where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean -that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes -the Hadamard product, meaning element-wise multiplication. +

    In order to train the model, we need to calculate the derivative of +the cost function with respect to every bias and weight in the +network. In total our network has \( (64 + 1)\times 50=3250 \) weights in +the hidden layer and \( (50 + 1)\times 10=510 \) weights to the output +layer (\( +1 \) for the bias), and the gradient must be calculated for +every parameter. We use the backpropagation algorithm discussed +above. This is a clever use of the chain rule that allows us to +calculate the gradient efficently.

    -

    This again gives us the gradients in the hidden layer:

    - -

    $$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$

    - -

    $$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$

    - - - -
    -
    -
    -
    -
    - 
    # to categorical turns our integer vector into a onehot representation
-from sklearn.metrics import accuracy_score
-
-# one-hot in numpy
-def to_categorical_numpy(integer_vector):
-    n_inputs = len(integer_vector)
-    n_categories = np.max(integer_vector) + 1
-    onehot_vector = np.zeros((n_inputs, n_categories))
-    onehot_vector[range(n_inputs), integer_vector] = 1
-    
-    return onehot_vector
-
-#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test)
-Y_train_onehot, Y_test_onehot = to_categorical_numpy(Y_train), to_categorical_numpy(Y_test)
-
-def feed_forward_train(X):
-    # weighted sum of inputs to the hidden layer
-    z_h = np.matmul(X, hidden_weights) + hidden_bias
-    # activation in the hidden layer
-    a_h = sigmoid(z_h)
-    
-    # weighted sum of inputs to the output layer
-    z_o = np.matmul(a_h, output_weights) + output_bias
-    # softmax output
-    # axis 0 holds each input and axis 1 the probabilities of each category
-    exp_term = np.exp(z_o)
-    probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-    
-    # for backpropagation need activations in hidden and output layers
-    return a_h, probabilities
-
-def backpropagation(X, Y):
-    a_h, probabilities = feed_forward_train(X)
-    
-    # error in the output layer
-    error_output = probabilities - Y
-    # error in the hidden layer
-    error_hidden = np.matmul(error_output, output_weights.T) * a_h * (1 - a_h)
-    
-    # gradients for the output layer
-    output_weights_gradient = np.matmul(a_h.T, error_output)
-    output_bias_gradient = np.sum(error_output, axis=0)
-    
-    # gradient for the hidden layer
-    hidden_weights_gradient = np.matmul(X.T, error_hidden)
-    hidden_bias_gradient = np.sum(error_hidden, axis=0)
-
-    return output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
-
-print("Old accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
-
-eta = 0.01
-lmbd = 0.01
-for i in range(1000):
-    # calculate gradients
-    dWo, dBo, dWh, dBh = backpropagation(X_train, Y_train_onehot)
-    
-    # regularization term gradients
-    dWo += lmbd * output_weights
-    dWh += lmbd * hidden_weights
-    
-    # update weights and biases
-    output_weights -= eta * dWo
-    output_bias -= eta * dBo
-    hidden_weights -= eta * dWh
-    hidden_bias -= eta * dBh
-
-print("New accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -

    diff --git a/doc/pub/week42/html/._week42-bs085.html b/doc/pub/week42/html/._week42-bs085.html index ef81256e4..0f69cd444 100644 --- a/doc/pub/week42/html/._week42-bs085.html +++ b/doc/pub/week42/html/._week42-bs085.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,22 +464,132 @@ MathJax.Hub.Config({

     

     

     

    -

    Improving performance

    +

    Matrix multiplication

    -

    As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image. -In order to obtain a network that does something useful, we will have to do a bit more work. +

    To more efficently train our network these equations are implemented using matrix operations. +The error in the output layer is calculated simply as, with \( \boldsymbol{t} \) being our targets,

    -

    The choice of hyperparameters such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a grid-search is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates \( \eta = 10^{-6}, 10^{-5},...,10^{-1} \) with different regularization parameters \( \lambda = 10^{-6},...,10^{-0} \).

    +

    $$ \delta_L = \boldsymbol{t} - \boldsymbol{y} = (n_{inputs}, n_{categories}) .$$

    -

    Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an iteration, and a full training period -going through the entire dataset (\( n/M \) batches) an epoch. +

    The gradient for the output weights is calculated as

    + +

    $$ \nabla W_{L} = \boldsymbol{a}^T \delta_L = (n_{hidden}, n_{categories}) ,$$

    + +

    where \( \boldsymbol{a} = (n_{inputs}, n_{hidden}) \). This simply means that we are summing up the gradients for each input. +Since we are going backwards we have to transpose the activation matrix.

    -

    If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers. -Andrew Ng goes through some of these considerations in this video. You can find a summary of the video here. +

    The gradient with respect to the output bias is then

    + +

    $$ \nabla \boldsymbol{b}_{L} = \sum_{i=1}^{n_{inputs}} \delta_L = (n_{categories}) .$$

    + +

    The error in the hidden layer is

    + +

    $$ \Delta_h = \delta_L W_{L}^T \circ f'(z_{h}) = \delta_L W_{L}^T \circ a_{h} \circ (1 - a_{h}) = (n_{inputs}, n_{hidden}) ,$$

    + +

    where \( f'(a_{h}) \) is the derivative of the activation in the hidden layer. The matrix products mean +that we are summing up the products for each neuron in the output layer. The symbol \( \circ \) denotes +the Hadamard product, meaning element-wise multiplication.

    +

    This again gives us the gradients in the hidden layer:

    + +

    $$ \nabla W_{h} = X^T \delta_h = (n_{features}, n_{hidden}) ,$$

    + +

    $$ \nabla b_{h} = \sum_{i=1}^{n_{inputs}} \delta_h = (n_{hidden}) .$$

    + + + +
    +
    +
    +
    +
    + 
    # to categorical turns our integer vector into a onehot representation
+from sklearn.metrics import accuracy_score
+
+# one-hot in numpy
+def to_categorical_numpy(integer_vector):
+    n_inputs = len(integer_vector)
+    n_categories = np.max(integer_vector) + 1
+    onehot_vector = np.zeros((n_inputs, n_categories))
+    onehot_vector[range(n_inputs), integer_vector] = 1
+    
+    return onehot_vector
+
+#Y_train_onehot, Y_test_onehot = to_categorical(Y_train), to_categorical(Y_test)
+Y_train_onehot, Y_test_onehot = to_categorical_numpy(Y_train), to_categorical_numpy(Y_test)
+
+def feed_forward_train(X):
+    # weighted sum of inputs to the hidden layer
+    z_h = np.matmul(X, hidden_weights) + hidden_bias
+    # activation in the hidden layer
+    a_h = sigmoid(z_h)
+    
+    # weighted sum of inputs to the output layer
+    z_o = np.matmul(a_h, output_weights) + output_bias
+    # softmax output
+    # axis 0 holds each input and axis 1 the probabilities of each category
+    exp_term = np.exp(z_o)
+    probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+    
+    # for backpropagation need activations in hidden and output layers
+    return a_h, probabilities
+
+def backpropagation(X, Y):
+    a_h, probabilities = feed_forward_train(X)
+    
+    # error in the output layer
+    error_output = probabilities - Y
+    # error in the hidden layer
+    error_hidden = np.matmul(error_output, output_weights.T) * a_h * (1 - a_h)
+    
+    # gradients for the output layer
+    output_weights_gradient = np.matmul(a_h.T, error_output)
+    output_bias_gradient = np.sum(error_output, axis=0)
+    
+    # gradient for the hidden layer
+    hidden_weights_gradient = np.matmul(X.T, error_hidden)
+    hidden_bias_gradient = np.sum(error_hidden, axis=0)
+
+    return output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
+
+print("Old accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
+
+eta = 0.01
+lmbd = 0.01
+for i in range(1000):
+    # calculate gradients
+    dWo, dBo, dWh, dBh = backpropagation(X_train, Y_train_onehot)
+    
+    # regularization term gradients
+    dWo += lmbd * output_weights
+    dWh += lmbd * hidden_weights
+    
+    # update weights and biases
+    output_weights -= eta * dWo
+    output_bias -= eta * dBo
+    hidden_weights -= eta * dWh
+    hidden_bias -= eta * dBh
+
+print("New accuracy on training data: " + str(accuracy_score(predict(X_train), Y_train)))
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +

      @@ -498,7 +615,7 @@ Andrew Ng goes through some of these considerations in this 94
    • 95
    • ...
      • -
    • 99
      • +
    • 100
    • »
    diff --git a/doc/pub/week42/html/._week42-bs086.html b/doc/pub/week42/html/._week42-bs086.html index 3b0d16fa2..0bfab2a0c 100644 --- a/doc/pub/week42/html/._week42-bs086.html +++ b/doc/pub/week42/html/._week42-bs086.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,133 +464,21 @@ MathJax.Hub.Config({

     

     

     

    -

    Full object-oriented implementation

    +

    Improving performance

    -

    It is very natural to think of the network as an object, with specific instances of the network -being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below. +

    As we can see the network does not seem to be learning at all. It seems to be just guessing the label for each image. +In order to obtain a network that does something useful, we will have to do a bit more work.

    +

    The choice of hyperparameters such as learning rate and regularization parameter is hugely influential for the performance of the network. Typically a grid-search is performed, wherein we test different hyperparameters separated by orders of magnitude. For example we could test the learning rates \( \eta = 10^{-6}, 10^{-5},...,10^{-1} \) with different regularization parameters \( \lambda = 10^{-6},...,10^{-0} \).

    - -
    -
    -
    -
    -
    - 
    class NeuralNetwork:
-    def __init__(
-            self,
-            X_data,
-            Y_data,
-            n_hidden_neurons=50,
-            n_categories=10,
-            epochs=10,
-            batch_size=100,
-            eta=0.1,
-            lmbd=0.0):
-
-        self.X_data_full = X_data
-        self.Y_data_full = Y_data
-
-        self.n_inputs = X_data.shape[0]
-        self.n_features = X_data.shape[1]
-        self.n_hidden_neurons = n_hidden_neurons
-        self.n_categories = n_categories
-
-        self.epochs = epochs
-        self.batch_size = batch_size
-        self.iterations = self.n_inputs // self.batch_size
-        self.eta = eta
-        self.lmbd = lmbd
-
-        self.create_biases_and_weights()
-
-    def create_biases_and_weights(self):
-        self.hidden_weights = np.random.randn(self.n_features, self.n_hidden_neurons)
-        self.hidden_bias = np.zeros(self.n_hidden_neurons) + 0.01
-
-        self.output_weights = np.random.randn(self.n_hidden_neurons, self.n_categories)
-        self.output_bias = np.zeros(self.n_categories) + 0.01
-
-    def feed_forward(self):
-        # feed-forward for training
-        self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias
-        self.a_h = sigmoid(self.z_h)
-
-        self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias
-
-        exp_term = np.exp(self.z_o)
-        self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-
-    def feed_forward_out(self, X):
-        # feed-forward for output
-        z_h = np.matmul(X, self.hidden_weights) + self.hidden_bias
-        a_h = sigmoid(z_h)
-
-        z_o = np.matmul(a_h, self.output_weights) + self.output_bias
-        
-        exp_term = np.exp(z_o)
-        probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
-        return probabilities
-
-    def backpropagation(self):
-        error_output = self.probabilities - self.Y_data
-        error_hidden = np.matmul(error_output, self.output_weights.T) * self.a_h * (1 - self.a_h)
-
-        self.output_weights_gradient = np.matmul(self.a_h.T, error_output)
-        self.output_bias_gradient = np.sum(error_output, axis=0)
-
-        self.hidden_weights_gradient = np.matmul(self.X_data.T, error_hidden)
-        self.hidden_bias_gradient = np.sum(error_hidden, axis=0)
-
-        if self.lmbd > 0.0:
-            self.output_weights_gradient += self.lmbd * self.output_weights
-            self.hidden_weights_gradient += self.lmbd * self.hidden_weights
-
-        self.output_weights -= self.eta * self.output_weights_gradient
-        self.output_bias -= self.eta * self.output_bias_gradient
-        self.hidden_weights -= self.eta * self.hidden_weights_gradient
-        self.hidden_bias -= self.eta * self.hidden_bias_gradient
-
-    def predict(self, X):
-        probabilities = self.feed_forward_out(X)
-        return np.argmax(probabilities, axis=1)
-
-    def predict_probabilities(self, X):
-        probabilities = self.feed_forward_out(X)
-        return probabilities
-
-    def train(self):
-        data_indices = np.arange(self.n_inputs)
-
-        for i in range(self.epochs):
-            for j in range(self.iterations):
-                # pick datapoints with replacement
-                chosen_datapoints = np.random.choice(
-                    data_indices, size=self.batch_size, replace=False
-                )
-
-                # minibatch training data
-                self.X_data = self.X_data_full[chosen_datapoints]
-                self.Y_data = self.Y_data_full[chosen_datapoints]
-
-                self.feed_forward()
-                self.backpropagation()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    Next, we haven't implemented minibatching yet, which introduces stochasticity and is though to act as an important regularizer on the weights. We call a feed-forward + backward pass with a minibatch an iteration, and a full training period +going through the entire dataset (\( n/M \) batches) an epoch. +

    +

    If this does not improve network performance, you may want to consider altering the network architecture, adding more neurons or hidden layers. +Andrew Ng goes through some of these considerations in this video. You can find a summary of the video here. +

    @@ -610,7 +505,7 @@ being realizations of this object with different hyperparameters. An implementat

  • 95
  • 96
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs087.html b/doc/pub/week42/html/._week42-bs087.html index d09923748..11e3be9c4 100644 --- a/doc/pub/week42/html/._week42-bs087.html +++ b/doc/pub/week42/html/._week42-bs087.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,17 +464,12 @@ MathJax.Hub.Config({

     

     

     

    -

    Evaluate model performance on test data

    +

    Full object-oriented implementation

    -

    To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data. -We measure the performance of the network using the accuracy score. -The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of \( 1 \). +

    It is very natural to think of the network as an object, with specific instances of the network +being realizations of this object with different hyperparameters. An implementation using Python classes provides a clean structure and interface, and the full implementation of our neural network is given below.

    -

    $$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\tilde{y}_i = y_i)}{n} ,$$

    - -

    where \( I \) is the indicator function, \( 1 \) if \( \tilde{y}_i = y_i \) and \( 0 \) otherwise.

    -
    @@ -475,22 +477,105 @@ The accuracy is as you would expect just the number of images correctly labeled
    - 
    epochs = 100
-batch_size = 100
+  
    class NeuralNetwork:
+    def __init__(
+            self,
+            X_data,
+            Y_data,
+            n_hidden_neurons=50,
+            n_categories=10,
+            epochs=10,
+            batch_size=100,
+            eta=0.1,
+            lmbd=0.0):
 
-dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
-                    n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
-dnn.train()
-test_predict = dnn.predict(X_test)
+        self.X_data_full = X_data
+        self.Y_data_full = Y_data
 
-# accuracy score from scikit library
-print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
+        self.n_inputs = X_data.shape[0]
+        self.n_features = X_data.shape[1]
+        self.n_hidden_neurons = n_hidden_neurons
+        self.n_categories = n_categories
 
-# equivalent in numpy
-def accuracy_score_numpy(Y_test, Y_pred):
-    return np.sum(Y_test == Y_pred) / len(Y_test)
+        self.epochs = epochs
+        self.batch_size = batch_size
+        self.iterations = self.n_inputs // self.batch_size
+        self.eta = eta
+        self.lmbd = lmbd
 
-#print("Accuracy score on test set: ", accuracy_score_numpy(Y_test, test_predict))
+        self.create_biases_and_weights()
+
+    def create_biases_and_weights(self):
+        self.hidden_weights = np.random.randn(self.n_features, self.n_hidden_neurons)
+        self.hidden_bias = np.zeros(self.n_hidden_neurons) + 0.01
+
+        self.output_weights = np.random.randn(self.n_hidden_neurons, self.n_categories)
+        self.output_bias = np.zeros(self.n_categories) + 0.01
+
+    def feed_forward(self):
+        # feed-forward for training
+        self.z_h = np.matmul(self.X_data, self.hidden_weights) + self.hidden_bias
+        self.a_h = sigmoid(self.z_h)
+
+        self.z_o = np.matmul(self.a_h, self.output_weights) + self.output_bias
+
+        exp_term = np.exp(self.z_o)
+        self.probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+
+    def feed_forward_out(self, X):
+        # feed-forward for output
+        z_h = np.matmul(X, self.hidden_weights) + self.hidden_bias
+        a_h = sigmoid(z_h)
+
+        z_o = np.matmul(a_h, self.output_weights) + self.output_bias
+        
+        exp_term = np.exp(z_o)
+        probabilities = exp_term / np.sum(exp_term, axis=1, keepdims=True)
+        return probabilities
+
+    def backpropagation(self):
+        error_output = self.probabilities - self.Y_data
+        error_hidden = np.matmul(error_output, self.output_weights.T) * self.a_h * (1 - self.a_h)
+
+        self.output_weights_gradient = np.matmul(self.a_h.T, error_output)
+        self.output_bias_gradient = np.sum(error_output, axis=0)
+
+        self.hidden_weights_gradient = np.matmul(self.X_data.T, error_hidden)
+        self.hidden_bias_gradient = np.sum(error_hidden, axis=0)
+
+        if self.lmbd > 0.0:
+            self.output_weights_gradient += self.lmbd * self.output_weights
+            self.hidden_weights_gradient += self.lmbd * self.hidden_weights
+
+        self.output_weights -= self.eta * self.output_weights_gradient
+        self.output_bias -= self.eta * self.output_bias_gradient
+        self.hidden_weights -= self.eta * self.hidden_weights_gradient
+        self.hidden_bias -= self.eta * self.hidden_bias_gradient
+
+    def predict(self, X):
+        probabilities = self.feed_forward_out(X)
+        return np.argmax(probabilities, axis=1)
+
+    def predict_probabilities(self, X):
+        probabilities = self.feed_forward_out(X)
+        return probabilities
+
+    def train(self):
+        data_indices = np.arange(self.n_inputs)
+
+        for i in range(self.epochs):
+            for j in range(self.iterations):
+                # pick datapoints with replacement
+                chosen_datapoints = np.random.choice(
+                    data_indices, size=self.batch_size, replace=False
+                )
+
+                # minibatch training data
+                self.X_data = self.X_data_full[chosen_datapoints]
+                self.Y_data = self.Y_data_full[chosen_datapoints]
+
+                self.feed_forward()
+                self.backpropagation()
    @@ -532,7 +617,7 @@ test_predict = dnn96
  • 97
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs088.html b/doc/pub/week42/html/._week42-bs088.html index 587e4a1cf..a0f64a0cc 100644 --- a/doc/pub/week42/html/._week42-bs088.html +++ b/doc/pub/week42/html/._week42-bs088.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,12 +464,17 @@ MathJax.Hub.Config({

     

     

     

    -

    Adjust hyperparameters

    +

    Evaluate model performance on test data

    -

    We now perform a grid search to find the optimal hyperparameters for the network. -Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate). +

    To measure the performance of our network we evaluate how well it does it data it has never seen before, i.e. the test data. +We measure the performance of the network using the accuracy score. +The accuracy is as you would expect just the number of images correctly labeled divided by the total number of images. A perfect classifier will have an accuracy score of \( 1 \).

    +

    $$ \text{Accuracy} = \frac{\sum_{i=1}^n I(\tilde{y}_i = y_i)}{n} ,$$

    + +

    where \( I \) is the indicator function, \( 1 \) if \( \tilde{y}_i = y_i \) and \( 0 \) otherwise.

    +
    @@ -470,26 +482,22 @@ Note that we are only using 1 layer with 50 neurons, and human performance is es
    - 
    eta_vals = np.logspace(-5, 1, 7)
-lmbd_vals = np.logspace(-5, 1, 7)
-# store the models for later use
-DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+  
    epochs = 100
+batch_size = 100
 
-# grid search
-for i, eta in enumerate(eta_vals):
-    for j, lmbd in enumerate(lmbd_vals):
-        dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
-                            n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
-        dnn.train()
-        
-        DNN_numpy[i][j] = dnn
-        
-        test_predict = dnn.predict(X_test)
-        
-        print("Learning rate  = ", eta)
-        print("Lambda = ", lmbd)
-        print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
-        print()
+dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
+                    n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
+dnn.train()
+test_predict = dnn.predict(X_test)
+
+# accuracy score from scikit library
+print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
+
+# equivalent in numpy
+def accuracy_score_numpy(Y_test, Y_pred):
+    return np.sum(Y_test == Y_pred) / len(Y_test)
+
+#print("Accuracy score on test set: ", accuracy_score_numpy(Y_test, test_predict))
    @@ -531,7 +539,7 @@ DNN_numpy = np.
  • 97
  • 98
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs089.html b/doc/pub/week42/html/._week42-bs089.html index 498a32741..2ad0f91c0 100644 --- a/doc/pub/week42/html/._week42-bs089.html +++ b/doc/pub/week42/html/._week42-bs089.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,7 +464,11 @@ MathJax.Hub.Config({

     

     

     

    -

    Visualization

    +

    Adjust hyperparameters

    + +

    We now perform a grid search to find the optimal hyperparameters for the network. +Note that we are only using 1 layer with 50 neurons, and human performance is estimated to be around \( 98\% \) (\( 2\% \) error rate). +

    @@ -466,39 +477,26 @@ MathJax.Hub.Config({
    - 
    # visual representation of grid search
-# uses seaborn heatmap, you can also do this with matplotlib imshow
-import seaborn as sns
+  
    eta_vals = np.logspace(-5, 1, 7)
+lmbd_vals = np.logspace(-5, 1, 7)
+# store the models for later use
+DNN_numpy = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
 
-sns.set()
-
-train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
-test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
-
-for i in range(len(eta_vals)):
-    for j in range(len(lmbd_vals)):
-        dnn = DNN_numpy[i][j]
+# grid search
+for i, eta in enumerate(eta_vals):
+    for j, lmbd in enumerate(lmbd_vals):
+        dnn = NeuralNetwork(X_train, Y_train_onehot, eta=eta, lmbd=lmbd, epochs=epochs, batch_size=batch_size,
+                            n_hidden_neurons=n_hidden_neurons, n_categories=n_categories)
+        dnn.train()
         
-        train_pred = dnn.predict(X_train) 
-        test_pred = dnn.predict(X_test)
-
-        train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
-        test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
-
+        DNN_numpy[i][j] = dnn
         
-fig, ax = plt.subplots(figsize = (10, 10))
-sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
-ax.set_title("Training Accuracy")
-ax.set_ylabel("$\eta$")
-ax.set_xlabel("$\lambda$")
-plt.show()
-
-fig, ax = plt.subplots(figsize = (10, 10))
-sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
-ax.set_title("Test Accuracy")
-ax.set_ylabel("$\eta$")
-ax.set_xlabel("$\lambda$")
-plt.show()
+        test_predict = dnn.predict(X_test)
+        
+        print("Learning rate  = ", eta)
+        print("Lambda = ", lmbd)
+        print("Accuracy score on test set: ", accuracy_score(Y_test, test_predict))
+        print()
    @@ -539,6 +537,8 @@ plt.show()
  • 97
  • 98
  • 99
    • +
  • ...
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs090.html b/doc/pub/week42/html/._week42-bs090.html index 812db220c..c8c345722 100644 --- a/doc/pub/week42/html/._week42-bs090.html +++ b/doc/pub/week42/html/._week42-bs090.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,21 +464,7 @@ MathJax.Hub.Config({

     

     

     

    -

    scikit-learn implementation

    - -

    scikit-learn focuses more -on traditional machine learning methods, such as regression, -clustering, decision trees, etc. As such, it has only two types of -neural networks: Multi Layer Perceptron outputting continuous values, -MPLRegressor, and Multi Layer Perceptron outputting labels, -MLPClassifier. We will see how simple it is to use these classes. -

    - -

    scikit-learn implements a few improvements from our neural network, -such as early stopping, a varying learning rate, different -optimization methods, etc. We would therefore expect a better -performance overall. -

    +

    Visualization

    @@ -480,22 +473,39 @@ performance overall.
    - 
    from sklearn.neural_network import MLPClassifier
-# store models for later use
-DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
+  
    # visual representation of grid search
+# uses seaborn heatmap, you can also do this with matplotlib imshow
+import seaborn as sns
 
-for i, eta in enumerate(eta_vals):
-    for j, lmbd in enumerate(lmbd_vals):
-        dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
-                            alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
-        dnn.fit(X_train, Y_train)
+sns.set()
+
+train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+
+for i in range(len(eta_vals)):
+    for j in range(len(lmbd_vals)):
+        dnn = DNN_numpy[i][j]
         
-        DNN_scikit[i][j] = dnn
+        train_pred = dnn.predict(X_train) 
+        test_pred = dnn.predict(X_test)
+
+        train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
+        test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
+
         
-        print("Learning rate  = ", eta)
-        print("Lambda = ", lmbd)
-        print("Accuracy score on test set: ", dnn.score(X_test, Y_test))
-        print()
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Training Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
    @@ -535,6 +545,7 @@ DNN_scikit = np
  • 97
  • 98
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs091.html b/doc/pub/week42/html/._week42-bs091.html index f3aaad163..914b0b66a 100644 --- a/doc/pub/week42/html/._week42-bs091.html +++ b/doc/pub/week42/html/._week42-bs091.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,7 +464,22 @@ MathJax.Hub.Config({

     

     

     

    -

    Visualization

    +

    scikit-learn implementation

    + +

    scikit-learn focuses more +on traditional machine learning methods, such as regression, +clustering, decision trees, etc. As such, it has only two types of +neural networks: Multi Layer Perceptron outputting continuous values, +MPLRegressor, and Multi Layer Perceptron outputting labels, +MLPClassifier. We will see how simple it is to use these classes. +

    + +

    scikit-learn implements a few improvements from our neural network, +such as early stopping, a varying learning rate, different +optimization methods, etc. We would therefore expect a better +performance overall. +

    +
    @@ -465,40 +487,22 @@ MathJax.Hub.Config({
    - 
    # optional
-# visual representation of grid search
-# uses seaborn heatmap, could probably do this in matplotlib
-import seaborn as sns
+  
    from sklearn.neural_network import MLPClassifier
+# store models for later use
+DNN_scikit = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)
 
-sns.set()
-
-train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
-test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
-
-for i in range(len(eta_vals)):
-    for j in range(len(lmbd_vals)):
-        dnn = DNN_scikit[i][j]
+for i, eta in enumerate(eta_vals):
+    for j, lmbd in enumerate(lmbd_vals):
+        dnn = MLPClassifier(hidden_layer_sizes=(n_hidden_neurons), activation='logistic',
+                            alpha=lmbd, learning_rate_init=eta, max_iter=epochs)
+        dnn.fit(X_train, Y_train)
         
-        train_pred = dnn.predict(X_train) 
-        test_pred = dnn.predict(X_test)
-
-        train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
-        test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
-
+        DNN_scikit[i][j] = dnn
         
-fig, ax = plt.subplots(figsize = (10, 10))
-sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
-ax.set_title("Training Accuracy")
-ax.set_ylabel("$\eta$")
-ax.set_xlabel("$\lambda$")
-plt.show()
-
-fig, ax = plt.subplots(figsize = (10, 10))
-sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
-ax.set_title("Test Accuracy")
-ax.set_ylabel("$\eta$")
-ax.set_xlabel("$\lambda$")
-plt.show()
+        print("Learning rate  = ", eta)
+        print("Lambda = ", lmbd)
+        print("Accuracy score on test set: ", dnn.score(X_test, Y_test))
+        print()
    @@ -537,6 +541,7 @@ plt.show()
  • 97
  • 98
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/._week42-bs092.html b/doc/pub/week42/html/._week42-bs092.html index 178749b7f..6ddd4488a 100644 --- a/doc/pub/week42/html/._week42-bs092.html +++ b/doc/pub/week42/html/._week42-bs092.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -457,17 +464,63 @@ MathJax.Hub.Config({

     

     

     

    -

    Building neural networks in Tensorflow and Keras

    +

    Visualization

    -

    Now we want to build on the experience gained from our neural network implementation in NumPy and scikit-learn -and use it to construct a neural network in Tensorflow. Once we have constructed a neural network in NumPy -and Tensorflow, building one in Keras is really quite trivial, though the performance may suffer. -

    + +
    +
    +
    +
    +
    + 
    # optional
+# visual representation of grid search
+# uses seaborn heatmap, could probably do this in matplotlib
+import seaborn as sns
+
+sns.set()
+
+train_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+test_accuracy = np.zeros((len(eta_vals), len(lmbd_vals)))
+
+for i in range(len(eta_vals)):
+    for j in range(len(lmbd_vals)):
+        dnn = DNN_scikit[i][j]
+        
+        train_pred = dnn.predict(X_train) 
+        test_pred = dnn.predict(X_test)
+
+        train_accuracy[i][j] = accuracy_score(Y_train, train_pred)
+        test_accuracy[i][j] = accuracy_score(Y_test, test_pred)
+
+        
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(train_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Training Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
+fig, ax = plt.subplots(figsize = (10, 10))
+sns.heatmap(test_accuracy, annot=True, ax=ax, cmap="viridis")
+ax.set_title("Test Accuracy")
+ax.set_ylabel("$\eta$")
+ax.set_xlabel("$\lambda$")
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    In our previous example we used only one hidden layer, and in this we will use two. From this it should be quite -clear how to build one using an arbitrary number of hidden layers, using data structures such as Python lists or -NumPy arrays. -

    @@ -490,6 +543,7 @@ NumPy arrays.

  • 97
  • 98
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/week42-bs.html b/doc/pub/week42/html/week42-bs.html index dddce8a8e..c33716b7d 100644 --- a/doc/pub/week42/html/week42-bs.html +++ b/doc/pub/week42/html/week42-bs.html @@ -104,15 +104,21 @@ doconce format html week42.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -369,83 +375,84 @@ MathJax.Hub.Config({
  • Final expressions for the biases of the hidden layer
  • Gradient expressions
  • Setting up the equations for a neural network
    • -
  • Layout of a neural network with three hidden layers
    • +
  • Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))
  • Definitions
  • Inputs to the activation function
    • -
  • Derivatives and the chain rule
    • -
  • Derivative of the cost function
    • -
  • The back propagation equations for a neural network
    • -
  • Analyzing the last results
    • -
  • More considerations
    • -
  • Derivatives in terms of \( z_j^L \)
    • -
  • Bringing it together
    • -
  • Final back propagating equation
    • -
  • Using the chain rule and summing over all \( k \) entries
    • -
  • Setting up the back propagation algorithm
    • -
  • Setting up the back propagation algorithm, part 2
    • -
  • Setting up the Back propagation algorithm, part 3
    • -
  • Updating the gradients
    • -
  •    Activation functions
    • -
  •    Activation functions, Logistic and Hyperbolic ones
    • -
  •    Relevance
    • -
  • Fine-tuning neural network hyperparameters
    • -
  • Hidden layers
    • -
  • Vanishing gradients
    • -
  • Exploding gradients
    • -
  • Is the Logistic activation function (Sigmoid) our choice?
    • -
  • Logistic function as the root of problems
    • -
  • The derivative of the Logistic funtion
    • -
  • Insights from the paper by Glorot and Bengio
    • -
  • The RELU function family
    • -
  • ELU function
    • -
  • Which activation function should we use?
    • -
  • More on activation functions, output layers
    • -
  • Batch Normalization
    • -
  • Dropout
    • -
  • Gradient Clipping
    • -
  • A top-down perspective on Neural networks
    • -
  • More top-down perspectives
    • -
  • Limitations of supervised learning with deep networks
    • -
  • Limitations of NNs
    • -
  • Homogeneous data
    • -
  • More limitations
    • -
  • Setting up the back-propagation algorithm
    • -
  • Setting up a Multi-layer perceptron model for classification
    • -
  • Defining the cost function
    • -
  • Example: binary classification problem
    • -
  • The Softmax function
    • -
  • Developing a code for doing neural networks with back propagation
    • -
  • Collect and pre-process data
    • -
  • Train and test datasets
    • -
  • Define model and architecture
    • -
  • Layers
    • -
  • Weights and biases
    • -
  • Feed-forward pass
    • -
  • Matrix multiplications
    • -
  • Choose cost function and optimizer
    • -
  • Optimizing the cost function
    • -
  • Regularization
    • -
  • Matrix multiplication
    • -
  • Improving performance
    • -
  • Full object-oriented implementation
    • -
  • Evaluate model performance on test data
    • -
  • Adjust hyperparameters
    • -
  • Visualization
    • -
  • scikit-learn implementation
    • -
  • Visualization
    • -
  • Building neural networks in Tensorflow and Keras
    • -
  • Tensorflow
    • -
  • Using Keras
    • -
  • Collect and pre-process data
    • -
  • The Breast Cancer Data, now with Keras
    • -
  • Building a neural network code
    • -
  •    Learning rate methods
    • -
  •    Usage of the above learning rate schedulers
    • -
  •    Cost functions
    • -
  •    Activation functions
    • -
  •    The Neural Network
    • -
  •    Multiclass classification
    • -
  • Testing the XOR gate and other gates
    • +
  • Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)
    • +
  • Derivatives and the chain rule
    • +
  • Derivative of the cost function
    • +
  • The back propagation equations for a neural network
    • +
  • Analyzing the last results
    • +
  • More considerations
    • +
  • Derivatives in terms of \( z_j^L \)
    • +
  • Bringing it together
    • +
  • Final back propagating equation
    • +
  • Using the chain rule and summing over all \( k \) entries
    • +
  • Setting up the back propagation algorithm
    • +
  • Setting up the back propagation algorithm, part 2
    • +
  • Setting up the Back propagation algorithm, part 3
    • +
  • Updating the gradients
    • +
  •    Activation functions
    • +
  •    Activation functions, Logistic and Hyperbolic ones
    • +
  •    Relevance
    • +
  • Fine-tuning neural network hyperparameters
    • +
  • Hidden layers
    • +
  • Vanishing gradients
    • +
  • Exploding gradients
    • +
  • Is the Logistic activation function (Sigmoid) our choice?
    • +
  • Logistic function as the root of problems
    • +
  • The derivative of the Logistic funtion
    • +
  • Insights from the paper by Glorot and Bengio
    • +
  • The RELU function family
    • +
  • ELU function
    • +
  • Which activation function should we use?
    • +
  • More on activation functions, output layers
    • +
  • Batch Normalization
    • +
  • Dropout
    • +
  • Gradient Clipping
    • +
  • A top-down perspective on Neural networks
    • +
  • More top-down perspectives
    • +
  • Limitations of supervised learning with deep networks
    • +
  • Limitations of NNs
    • +
  • Homogeneous data
    • +
  • More limitations
    • +
  • Setting up the back-propagation algorithm
    • +
  • Setting up a Multi-layer perceptron model for classification
    • +
  • Defining the cost function
    • +
  • Example: binary classification problem
    • +
  • The Softmax function
    • +
  • Developing a code for doing neural networks with back propagation
    • +
  • Collect and pre-process data
    • +
  • Train and test datasets
    • +
  • Define model and architecture
    • +
  • Layers
    • +
  • Weights and biases
    • +
  • Feed-forward pass
    • +
  • Matrix multiplications
    • +
  • Choose cost function and optimizer
    • +
  • Optimizing the cost function
    • +
  • Regularization
    • +
  • Matrix multiplication
    • +
  • Improving performance
    • +
  • Full object-oriented implementation
    • +
  • Evaluate model performance on test data
    • +
  • Adjust hyperparameters
    • +
  • Visualization
    • +
  • scikit-learn implementation
    • +
  • Visualization
    • +
  • Building neural networks in Tensorflow and Keras
    • +
  • Tensorflow
    • +
  • Using Keras
    • +
  • Collect and pre-process data
    • +
  • The Breast Cancer Data, now with Keras
    • +
  • Building a neural network code
    • +
  •    Learning rate methods
    • +
  •    Usage of the above learning rate schedulers
    • +
  •    Cost functions
    • +
  •    Activation functions
    • +
  •    The Neural Network
    • +
  •    Multiclass classification
    • +
  • Testing the XOR gate and other gates
    • @@ -500,7 +507,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 99
    • +
  • 100
  • »
    • diff --git a/doc/pub/week42/html/week42-reveal.html b/doc/pub/week42/html/week42-reveal.html index 4ff08534e..0c4c4e0c8 100644 --- a/doc/pub/week42/html/week42-reveal.html +++ b/doc/pub/week42/html/week42-reveal.html @@ -855,11 +855,11 @@ all inputs \( \boldsymbol{x} \) are given by \( \boldsymbol{\tilde{y}}_i \).
    -

    Layout of a neural network with three hidden layers

    +

    Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))



    -

    +



    @@ -911,6 +911,16 @@ $$

     
    +

    +

    Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)

    + +

    +
    +

    +
    +

    +
    +

    Derivatives and the chain rule

    diff --git a/doc/pub/week42/html/week42-solarized.html b/doc/pub/week42/html/week42-solarized.html index 100d78c42..f8e3db8b1 100644 --- a/doc/pub/week42/html/week42-solarized.html +++ b/doc/pub/week42/html/week42-solarized.html @@ -131,15 +131,21 @@ div.toc p,a { 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -936,11 +942,11 @@ all inputs \( \boldsymbol{x} \) are given by \( \boldsymbol{\tilde{y}}_i \).











    -

    Layout of a neural network with three hidden layers

    +

    Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))



    -

    +



    @@ -985,6 +991,15 @@ a_j^l = \sigma(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}. $$ +









    +

    Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)

    + +

    +
    +

    +
    +

    +









    Derivatives and the chain rule

    diff --git a/doc/pub/week42/html/week42.html b/doc/pub/week42/html/week42.html index 483203b5c..5311f40b0 100644 --- a/doc/pub/week42/html/week42.html +++ b/doc/pub/week42/html/week42.html @@ -208,15 +208,21 @@ div.toc p,a { 2, None, 'setting-up-the-equations-for-a-neural-network'), - ('Layout of a neural network with three hidden layers', + ('Layout of a neural network with three hidden layers (last ' + 'later = $l=L=4$, first layer $l=0$)', 2, None, - 'layout-of-a-neural-network-with-three-hidden-layers'), + 'layout-of-a-neural-network-with-three-hidden-layers-last-later-l-l-4-first-layer-l-0'), ('Definitions', 2, None, 'definitions'), ('Inputs to the activation function', 2, None, 'inputs-to-the-activation-function'), + ('Layout of input to first hidden layer $l=1$ from input layer ' + '$l=0$', + 2, + None, + 'layout-of-input-to-first-hidden-layer-l-1-from-input-layer-l-0'), ('Derivatives and the chain rule', 2, None, @@ -1013,11 +1019,11 @@ all inputs \( \boldsymbol{x} \) are given by \( \boldsymbol{\tilde{y}}_i \).











    -

    Layout of a neural network with three hidden layers

    +

    Layout of a neural network with three hidden layers (last later = \( l=L=4 \), first layer \( l=0 \))



    -

    +



    @@ -1062,6 +1068,15 @@ a_j^l = \sigma(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}. $$ +









    +

    Layout of input to first hidden layer \( l=1 \) from input layer \( l=0 \)

    + +

    +
    +

    +
    +

    +









    Derivatives and the chain rule

    diff --git a/doc/pub/week42/ipynb/ipynb-week42-src.tar.gz b/doc/pub/week42/ipynb/ipynb-week42-src.tar.gz index 98f4815985557b58c8d890ad9aa88754242fc675..026cc35e2b845e4dfd6fa246dccf8837cf37cbb3 100644 GIT binary patch delta 77017 zcmV(tK0V@TERYOHa3r|hpC z(ESGe=lyJdkYYI*U74X#rR;C9{i&#OA;sovK#?=z?v)vYudS_r(fFN;hwiaDT`ju> z&<+`)f@WrkuuCiDhWgOWxkAgcL~n7)f`CS+9AVkRhF@;DtoxAsNgHsQnXPDE@x)MP z!{Te(rNP!>m|vfp@e+~hXN_ng<)9f5=q6+q#p4No+8v$y^GI;p++wy{b3{N4KTGre z^>ytaeCE=v!!ClT*#z@|?%$W43{{<<-8a7DlBb6wH^b7gc{$J2d0FfCLaia&wD{Uq zfp3KVD4q1Vo6PfS8J5V-|6-!`oIoZ(;JWYFjNcO@)TxwT#!P4{wl2z--|gDxpTn?2 zKi6x2<~pq%L<73$gb9j~c(FBHYzZW|6@O_EOZEAL=!LJZg^#YRTUTyevLRNYm(M~t z;M(;Vlgiy1Uq!*(k$?&(%?zC@xj5G|D6F`ORwMTrS7TmTYVn`*2x1Q3fe>6}> z5OCFrawO4tIL11aq~e_(Zw+D1Xk+Gw^7xQOCLtJ z>D57-6UvkiZ`mE};26`vXq)$A6I1Q$NYcLjO(L0Zb)37rcWt-ZnMB0sJQ4BA>zo5G zKKBzbrU#%ax|z`zuvbNG^JZt(u$)1E7CI#06v3%&uCmL1wXU)8Hoqp_r-vb7%g7a@ zW}V)b8oLL+L09tA0~^v1vdir>bO~mA5^wuLsSkT`O8V#i0Zs7%i;TlH zodvS8z{RB>QmGy#@gc?dv;CTnD2<9VY!sH!v|=u^K>aN|!nDAzo~1Er(*^vecHw|U#6)S24ON#}i_3Zzo>e>L z=LO?#AI}JJd$jBLcS3$1R^QAXW5575oEogz1ebivh4r&cWs_)NK59!t$r;OIs0PfdyNgtvj(N1uDs!ROqI zQ}3D|D0V>Swbd_eNdS*j-z%JtUeHq0UjH7LTNp!Gq)U0UtfhBep zM+saq2P|V1r&ph!>d8ZvMX0d<6;I&#N|vsEg$T&6wyrxYWu}9kJ0A`O3$ZxHd=+m; z!M=`Tui%;a`@2s8vMaRv$-BEF7ZHX>z$=iZXZ&uvlTIUU24$3RAf=}DfmEiNi4A!a zQ=GzqS7dcSW+cmhF@5ucjr^AWZav7R+;&Y3U0BZ!+|M`L*K)vIUJ;E*-jr1!btkku z93mG9BGtQ1M`zjT9SWPOSJWT0ULL8!^?kUvSgx;+H*9@ZY_8*C8H(?CRL7(#SoyXr zt&N?(k-t#IrK{^uwcWN*N|$JU=JCqNCB(M+$+otC4g)fOA1g6t@U_8RDR1a9+2g2+ zwOD8N{cw4si@|VfW0^+L{S)25IQh{%jhi=}rxV7(gHVIYv1hexnuh>->#2>{RZPm% z8A&!);KD14V@7@>t@3|Z6Mu^!+Ti=Wk(;caWww6UR>8S$0{VsD)4!S|xW)N1$GoA; zw4Nedhj>zdZmRr|k@m09?|*!Us9IK+_+PEywKFICcJszOwbBH#vr&L4UJoOhPL<)H z{c;BHESx?0&AAW`;5vkTu(xy6+RYPLOAmec&g{5Q1K~iIKyzT5&g;o=fsaY%sfvlsu)Ke_xgM9;kI@LDAP3c@5uLkgIPhl0NHyJTc z%Kc*`or0{$u78Ho!^7RKG=&x-+KWuoMz)mtzK_al- zm>LUde;Hq~Tbp`5ev;6XL*XWE$mm#gLsNMt0?9*+)=K+9a^(l7$(B2|peNadAI)sc zoD4sI$mF3p0dGpIy_GQ^nGmroa*bw4&Ymtf;J*-mc6Y$_#OaMkzfuj~^wQ*{w#uc< zHq|U34D(gv?(w({>DZg5$}v4W)}6tnhJ6;F52gQU3F@7rZZav4%V#bkT4WQE^FHt0 zF|5ZKunX+&ELen9EMxwJ3M=aN9b5XOe_z^vKjAqx@Ehm`Y<{B%5XVHnH#3{6Z{^hR zW4h<+*Eb8qod0qacX(6mmLR=FuzU0&`b^(B>->*=w685#rn8N}&4EeqTP{tB6Wn`+ zyRX2^8`6#*yO`n1?U{kv?Shyk(T)RsHuTAg%Vc+*?9=QfMHPCiA4g|vL)X6ER^A$a z?dhzf*ilo1@sYOHv-6fH8?n#1vK7JgtJ}|?I}1SG$c}%%@ao030$gm0@!&zg9{3Q5 zRpvVNTSeM1eYpGlYEkW`I{E=T>+MxKxk-EU5~0acNo|q)Pc?fJ3ty-$+iUX^(nq2% z`K$@+%s(pcf`=X*D=^D=5iJGm1*l zoA<*2J5>BIq@73EXcuQVwsMJQ^~pvQg`O=V;$6qxRS0Q)`&&2D`Ghe2XDs9ZCq4&} z97Z`&;37-s?)I1H#0gIYs}wVMpy^o)k$)#9r0_6Q7~Lga!+2A-=&S}pudX0}D9HfJ z)b*`vzr^1NsyWl{-|LPwfbTkvNusxwt&_W~Zz6xEk_FDS<(ZYVZ5uiln1v00wpBUg zb~>1V>|I17nc3Dj$f6(X;V?y#2w3Z!v+T#r)5k;q(_)nAw6Y7C%}&C4BHfXukVS+Q zzN9S2MNNpJb+<=MsGy;hCr9sLkd0Tj?Ka#phf|tQv0B5QhO<)qzu`EK}vyw4(>A z4b5%!1P2@N8GBPGY<60e)fgmTJ8y+B7Hyj5MejI7MN*u_VRym*QR*9ibeVsU0KaUb zZkpoW)7t&k5B6u};Am~K^;F4Za;p>1DN5)nEmA?M(Xgn|v)hy^nZK&P`*Bk@b)^tL zbQeAzB5se)V$|NmXD2uTpE-nc9XHR;s}=J1JyTJgbR?BL6e;AjFCI`RqQLN}2kQpY z1?Rv2>IsrxkQO;BbM4xHU%GkRvw`Cw5PvsJ9;i|v4<$b{bO;AR9DF^+Jk22UE@=<9 zvn1i&dbJvVezqFyYr9N9dKm7z=bzW$R7a{mQHRFT-Szn7TkON zZc_Vu-yDQ+5D|}ri~)%R!0Rv^*PA)-9^f{AsVghhpY5LS5d_|xlQXB*gUyI_YDArL zwv}T?t5TP1?{OUAc+7mRjgA@;ylgwZKAN28%dVUg=4ttVe-myPc0b=|)m$2Hrw2~7 zr&70_`%ln{ZQHo)YwDn_hTC4aG5zd`F8k zu%cZrzF=8k^VIAg67&OA#rxU5A6LIVTg(KTWU|4=FK^5sl;P<0F=rEq@y#ocEWQKb zBK5PZ-y2td?AmN~L=;Wp^T%7&=IOPvo>*fza<$^>smG@1Al-K>>N!@)Sbra>5Ad(& z5+1!S*)rp)1H^+nKWaK3)L7vdJlWB3DGd4$V2^V$QDg-nGO;vE&ix8{V|hcp5@c`* z-S%$_1&!Tg1}?3k{Hl_Q6q>bgefT+$+gz7`l!iTjtF=&u!M8n4AY9iApZI-(uY{_s z#zd%^x2`+@eRutjh&-IGoJmJHdMaOMGa!A zBL4TxQ&pXiEca?eg0>}IOP-qtbk?2SfJXurizE7u`lWWPAzB9G5eL89QypWQmh1A% zLd!sZm0=jHw;SMRBK|&w(^?_fzwMn@!h#_>y>psh&%FSD$*jVE++Ie{QH@Hsz#`AX z?|a%VV0e_P*vFJyuZjKmSWE3H`*B8oCALGM*}LM|ws5@M2`k?&wsRBP6U-XLlqFm# ztZ&6{$UT|pPZ!`{=^m?kJ)L;^^pC_}RvmSJgsTRA$keAIt#>)l89G0XXnCnq?rLd= zJ_mHw`#5_(r+2*f8lB;#wf_QRlcMgfb%>a~Kc_N==-o=WsV~~}$Gi=AHs2M-OBCN@ z6cAGrjV#sQ-+}IduQ`}ost^5;iuuxF~fEqswxQ${AiS;}U^mNC^8PF}Q)e0)vcV?fPuLwZ+i@NsAzW#2m4!iXR zGUZU#EGA7@60kKOvbYz3eNH5K_}fzDtIyrx2X$ z^QTwB|IlO+CzXIhUf2J1fU+Luu9y*~XJu#N<#o;9p&&xn!@xbM1h(;q+jPSz zYL~#HH<8*!rNg>Dce{gjfp$I$X9->vPF6ij1^dko4mSZ;C+XVetnY>Nzk^ zzBJ)ZzZ+)1DpKTs_$E!ZwEk3oba>E2r^*r~&d?Y+bq&SGb6*wrTEpAG{2!XL`w?NN zQ@yM+^SrEE~(|R;x*i7mC}@-PfYXTd61b8fdUaPOL87r zm)!Vjd?v+=j-UnfzVLoJbC~utzFuB?Gp1FssHm3TJvM!`f99hmG!629CbABiDKRON zSfO(2i5-Z1qQ5gt_ls@lcq_sdRokx@!%Rd}Ld$gbq12_9dus?WO&OBl!<%V8|S;Ju&;|`ZeA0uxr9U1X7|9plqcEiNs zU0>_(e>CNAx_fpgt-^+X3gVjL3KcnQ%c;?me=&A^IT1@1*yOJ;e{qs=H#<@KX0?6k zQGfqt>G6kCmo$DEaN2}kzQr@P|CQIEU!cih*60W;%1$d4*45(21O1|@7wwWb>Ca`Q z7ts219~Hq$%_C2rRA@GC$cH1#evLU%T({nhJ_C(9wiYaR1;vI_j%bLPz zD}Jj&M9p|s^~sEXBGQgSgWaXE*6lP<+hi?5pxILL&Y&;cRerf?GtrVxupAo8oRF?33@FspAhExz$B! zRX`)2LJ0_m|Hy3xk044Lo)G#E;{2<_&@NEP-OJ3!v!Y5(VE0EtHkYj4)N#Z_tPlLV z&5nWF7=a46@ol#xGn_qBvpLHX^wNeEFq&3h$J5paa2v5)rgGR*Qjmy|sJfuT|S3L7RX)e9C@lV{Mlk~22FlmvxQi@ZJMvb$T z1#t8vQ!yUXe?iE@W!RN>Rd?7u{v+*8D>Im03jqaxG!(BDv;WVuh*K#2kLQnE=O>01 zAB@e8ESuP|UtLQzmHhI8`WrGp-5Z{~nY70z5~d{I@!_+W=z&kw2#4pUbHkq=&%YvU zJ+Ru%)1MGWrZE_}zhB!{n*6cPC&zjiPse^bdc7q8L1k@yIH!nWfe9yucDVJwfM_GS zb3g}wvT_1pcc%vJ0*m~Vimsy7N}#7Glq&2~@$XAa%{r!aZYqwav+8*oRA7nB1Y>2F zVc}`tK<5_jrDBJ?b*-78R{G8Mqeq#>BG((*91Ibyz19{p1z@kKeA@cs^Q$a;RK|1K zd6t3<1}``2vgy5^AvBhd{9hmhb2Dj#+I7l*#e1={B@d&JV+H5O)0#c_Od?i;V_lF! zxvG4Me~Ct}Gw5#n&XC=Y-Hn8b)yM!8+7c06?qGDk4sq&zewh-~cXdXz@x5Si@-m7q z3XMuYXTioS7xzDz)_;Qa8T_!Od|kzr;a0LH6Z8uk%)}tf&n>Mm`}xA$x?gChI#&~a zI0@mZYf`f@^*JTRpQ;(;vKhHozqQUk|1$#tqQA6l%dP!=OURPgp2P-KQ*rK64&h1A zx9YOBerLuye`q>0&6-*sT(|Wd5UA303Uns@Z&U?{6C+HQCNdeVi<~^pD=eQsJ5|3x z49x_`y-&L{U^Rh~lKKzf-Lg&9yGG=H%ma#WgukZD1D@vE8=Mo_j3d5^7bBv8fOT)T z|GvEVUrWV`o$Tm}sWOmn!!8FBO>YoRHH_O3hoQciJWhdxS!!NZ7Z1-qSMCdp`l9z+ zZ~i;+nCRDtq9T8Enyc+Fjd&8jij$!VTbtcAW1MuXa3(;l$g4$e*2hy_pW&;2{0sHJ z*!J_YegvL-B)d6X(KDWsKD(nPRCj+QU@0`-wF`K11JkPL;fK9NLkv;|tk&d%FT$fZ zW^|iQL;Y`@V-Ls&#@m2J%3e;_(tb(&&%-ZmTCVOHgaq55f5Qq4iC@u5ml-}Z4*5GN8Ppf-vyx~K zJvseh!bXr!~WWJ*xEI5dR;B2oOq`x&$W$=>L^iDWX2Ylkor5g-CeEZu{_A zt*IJT%u+>R!i)ftJu2{D{p5cv;U%7@O^)lm%VSvc`s>}Fmw(N{_B;penAg9Z1NZHL z>Tb>M65><^<4Ch~vBr{+3>N(E^Zn)Y+cc7YF~_s<00IU4<>!}w_d25fvxY#POBm?- z3^rxo(eQr>wt&v;1n4e$rmc#-mH!g0ogwcFfeUNyM~csheXPLEkz@NrJ(p72{tK0|*o* zgiUs8Qlw#+;+j5x*%-0R;ER%{48#7Ujtkwt ze6Ochi%ddcG;_g}O z?K3~{|0R@QoL<5E1*$wDyX{N=*nxKz0KhhRX=SN$3F9EZ|BlVv%EHar%!S?Fo{P=V z-r|4DcW`ijaPaZ)P&^;``FQ_2a&Z0UGY1DJ7bh<#Hw7mTKL-~-A146&fX3$0yb~)hr?Gm1IP+uQU zjj5)`7CB>zLB-e3l#~p_JW+Gsc{mHdV2enWC9dHpG36*Jh@cu4{+W|xs;v_dkJV%z zQcb>pkKo+2>#;?AV3&H^gpb2J+p9LBeN zb{v&F+u#7LYrim9j(iW~l%VF3pmwFRoJ{r$z#-1YxxE78!0|kfTKtef7)J>oM@i=- z#u5MPw4atzfbi!rqFfxfb3{rB%?<489Nq1IJ4Ff2IGc98@`urbyKK=Ol<+8=aFi@| z6xHTA^vt|+D$mt_ zZ9SQ688#vD?|^pgIMnLtlE(9rzfX#bBL%`;Z&-C^}W7iWC@w!f&!`VqvoDz;cs0@ug3~J_ZzW z2xuju=%@xoMFrAmGj$;t7EIl}Z}uhwNJ|oDvlIL!EYxLGqk}aEi-7_zYdk>WrAOtL zid(X8H?&Wqh~^n?#0Ex3L;x=0fJP5`F~g+K*g>e`$%DSm*A4>re{e8%(B*o63tz`C zD09KZjayjFRxTd6ujP$F3f(7gZa-mDGNwMZS;o6p51gj4k*a z7YTk~P*2fA#?#gY@n7Hye=jA7uUh$SwwPMNI#|Xs7=Q;y0}|M9k70bmzzJ$gM`0b1 zx2J~VjC5K@V^q z%`y+icqTHyHjg3x-{gLDiFT@oUZ4LDF?#LMy z1r~57YQDlQhi6b!P-wfoK3U&}>rCP_5w)|j@_iwW0TzOhBVQ zBaDY#kPilnPC+2e6s)X&7rppe8xfX4m$%s zi(Tii$Vr^KsHw;D*Vfk?3ERz0J`S@Dhoe7RWhne}AnJYV>hu_r$j!~oGvP@l z(Sy!p*o1rn3$vw2$uV51UP&Ucq6#+G4!E7KMES!U?>%MRoSdSIj*Ci4WI0|-QD6u6 zpAp;G{cfyI1ZLs1%HDE2M zm)VMceB&NvwO2gfll1i%sr9p`Ap}~8;KXzYv)M1i$%&F@R=^Pv#jZI1S#|t_54Yj? z$LTohS_rP=c4{c8y?xz6Bm;x;W;7!s!;D#fi!F?kTMS`YuijSL(2(s7rck}@Z1EZi z0ESO&5)v02${SkELw`C(#=+h8%>rWbMTD$;*!G<#1S9NUE)!EE(f5i7srp+p$fw ziJ)yaC})NPxVaf_IwU&)v@2JLYinyqMJ7P$P?+T~;vXTrH2v<28K~%#84-sJ6IK$E z2Fljc;Jt9Xk*mA=$oE$d$9JjSAUJb>*FOBFTy?b=2f>Nf9L!zQG17rru79n1B5KrE~fE4(W-MlI-n7IajC zH@16we|+am{Q-zdOhkt<4{(vV1z{AHT`(1UQDAFW$$5Ekb9%e}(y5`wRlCKc1Q++c zV|>u%0r?0!WEbxtdI}50EgD50;n6GiX0kqFO8^_N)tX^CS)I}%OmhDgeN$dm7BV)I zm1OD-dXX8aM2}lWVLEP?P?{!xG+8gax#3RLOaM|H@Or%2Mgq_b8ck14QB#}X9=Nwu zH5?XtrOqkZMaXjv3oMkn|sOM%^g8_qwPm}9gG znunq$aZozlC!2eI+rTw(Wf~=o0pyH9{L!O|cx}kKMXusV!1wI`8^=qExgp74rL_}8hg~*6FurJx1xDyZX24IM| z0-l7@(sXd>w%3^#4&I4>GFu_4s#y|ism~3FS(NA3nj?OMqj~fhMC%qu=vri40R)#Y zqBH3Tc<@g314yhCUn_@``%Y$Aily>9^queOIKDkVLHax}JdD`s-rC%JTPckqJ|h4< zgG#E5{RV)?((&i0D8cw~%^y6MeSLlVVG&DH4eQ)9GT}%(izL&3?>6~GXGmQ0n5QT< zB{&sM!#V>zu{~b^dX3o*mk>3(urTMC?=_7}h^x%T0o25XG^VJHp-BwkK(x`c)OG8F zqBcv4)|M~IDe{QL*#dsz2WwR1*IM#|4bivvAaYtSF;tstey>(~Oin?OI_aX5Tw#v0 zi{k9WQt~KzV0%7)WNE3rhK^o_83b7A4GnA%d%VWmtRXx4eQKc9Kms#-(MGqt&gXPw|If#$D7 zv>mj0wP{bopX>swKroMh~ z_UYjje{TMN)+U~8B-ve0@)j%ZtZjYD@}v6%DceliN1tuXkOvQo6!sUzfcFEr8Un^e z+yL&XR}}9aeb&(=Htgvn_MPAgqd`KNqZqmV%x^IzfZScqvHJI!J0YSZowmTmA#qyz z;G@wi-WBod0{61Me@?)0@kG)yw%r_|Dd`7gC6nH|4NAtHOTa zy4SJ|UcUtNqBBR@j&Av#dO(x0IQziJ_-Wl z5c!ioe3pjLPYZM$wV{tezJSVyS&JD;HPJYnn}?Cci|Lj-AMXgm7p7XD1zJPOnzobk z@M#dlUk^6f#R}W{H6+~Xyin4CmZQBI2T?0*icx%d+BcygA$AkdEr?g-A<5`7fw(D>6D>Xc@+%TNp&9d<@E6*z`TvlDatjK!1)8_d%|*{j z4OJgkrQ~)zk+@fHE{NfbB1X)8=!lKkA}t6t%XEb}%zy2OgkX^~#!-ud8t~gDhfHgZ zT=9%39A11wJ{AQZrj0wi#BlnR#A$SYv#WAMVuzTdkVSPkdltz-!B_q<{&QO{7fcHl zQ4aJ07=TASarS^P(=rViNuJL%(N>Gi7=^{fGC<&|ciG?|9*06Ft|;>;5?m>Gm9MN+ zZ4%y6qAS=8hUtgR>)sR2#4STX?>WSYsEkn%;>gH|<#isM!=Y!57$0A~5H`wxD~Lz| zCUXkAIi)Lr&6U9oi^(OHeLw(^0EhB^uRX~(^GFfsta){LiO*sPQm{2#QZ&F{T%SdZ zw4ir~7Vp>1{?tH#kL?di_5i2cz3t)&+BC*Htu5Z!yNkU5v|)6<^umEbPrj&ylk=n* zIg(sV)g4-=HLG7P(0oh-k97xsv)7AAQk(hTzd8MQIW8#yXSHGB9o+m$MEc&%BdQ}v zQ7@xN=#2&+gmlzCTvSrWU&NKEK4E=Hcwlsf#+OdNLor3H0 zTl#$E1<2xr$lasaRsXABj2t3v0CL=h<- z=2=-!8~`R?2sE)ilV;j~=)g1Y))wyEXV{CngIV_agTr|+BLU~ban0LRV>ijLWTHdV z`~1&>I7Jc-dbWz#Luo|`vz)qmth@iE&XKwC+LyRy1ysf|B)s-B*Nk^4ow#XqhbyHGokyc7;W72`z1jkgn{jXM7PLa{zXsGl{7|T}VL%@LI8wV=9In;y!?WQ2? zE9)U9DoyNaa&eqImfdeheYjL>FJnw9tGXVT$?L6DjheE%N{h ziLqI zLPNrURDj~}qIq;*0l9g2@x2 zM(1DgKpxIwk>f0#kri;2q>9w6+@ARyAKxy3&hCU645IoNbiJKPwQaHVvIDdXr^{wYcw(BNgO5I#8QA+f#;P(YxUImKM4c3>~ z)f@c)gC4OZzzQee34!@nI*B|7uA292ijEgM7;()7&av)3G=9qQfP|ksc#89doNw8X zJ2L?^@TRQdV~CXjPm31PlIt#(Vo6&ElzFZ~NykioTmrHvE8K@zaXeYL9MJqvh?MhP zd{hB#nZ)*fOW&9W?V8(FQR0aBc{)*B|DJb3V>>>GrDc7?Uld1=te$c@Lf}{r;cXf zSbYJ1hy=)0I{VvO3&5TF0}@=$@?YAXXr-pj42;C10qiA!B{%TIXb#CB%EqqXPGJ|a z9VGzYX9076EMv3)1vpS&&ig_*OW_U=0i*Y3Y2-b&+M=RO#z(D&=y+UmOT%4T(S(Y} zsi>}m0JIg6fF(?0IR9!h&7=PQ0$kf{_+PJooRTR4j*Jexn(om+c=dGc2N=N5|Ks>&7+GPpo8DDLl<4t=IDz&B4}%+j zUf2x2AL;QK@TP4~z`+b>l)U;oTQVuHaf=NNRGO3ITI&cffJEV$VZ2JLzfKtj*9>QS zw)4X@=C$xumc(U@5k{JWO7ObNwNG+CZ=w-EJ8*oYZ6!GjrWc{qY+Mg8@H8 zN@X@2F?pLCJBdjBGDkdG!#SX$t1fc^|cLX2pE^}|)<{ZS$ zU%$FwFm@D7-_)O&#j3wfH^y~?2Omc6NZ&YYMGxSIuwn$@OM$KtuW{7n zW+1%puFF!cG^h$rmHH}Gq*r(dzQ2ZQ9x2ILVa`w~bp8tb7M7G6PK(0oNgu#}4woA| zH;;z?T1J1}3c-?zd}j0hj;nHU(r%R)-&hw8aIigQ{>6cw(6#$0&gSFqFaZkmVjH;o zl#@Uiz_xCanE$16WfRNwWsr~)wN*`Ab6Z@qOBde{Ys191mK>EI^G@Y9UrAC*!bzE9 zTUc0Z$L&|NEgf&xak@n>NWU{?M+8NGy2L@k*mREW8(Q< z26xjy978xd;XW!{I(aEaKl=zYIQ>z?%2oI+_%<>{IpuJwV!GfDh#ks*r~rBnYrG^G zQJ9MurV$DJIT^+D6g$pYu|WhRQG2WDqA=6$ceC`uwzajPpWN}tU4#pMj;pz6J_6d; zjX!K8l8wWdbc+MMDY;&M+VN2`cLhhqqs|V}Zwgl;wi@kIuV(Id{Yulv6u?oHq=`4-Yv;c8Y^aKH&i02$#`Fn_0gMR-tQT?C!wzV*31h)VlMGwi6+O>n4v> zFh`y$eLY&^9I7bYu`f)Ltw_5la0e~IGHj1aeQ9)DQHp?@kgZY7e&dD|NuGsQ^4E^eXUigLOr2Rk zBp&ptm*NrT+l0hQ8nNHme-;7C;gBw0G`>{a4;@lv{isfVD8GyfE%?OHdUm#v#`;Be zmL@dD0ammY0dAN*o#_L}9Xme0y6|nLqrc|XM2WR{&2x^%VW&bP#CKX%Z|bK2Lif)) zCfB`0_9J6p60Z&+_kXQJXu7(|Z>X6ze2LJKb21Q@k>K|;I6Q%7Ro#jPbln?UrJ6wf zrkPt87hUy#o|V0~Pzm=jjjCbu4S?k4Y4dmCPMX$gUIJmoxL?3~sM=&Np*#I{+shb& z-l893A!Zm+Rq`3*CkgK4{yT_ zcwlsz$3!!CFPRD3vQ$@uXmAjHN2@w8v8<&SWV9QkSw#nD7`ZTk59hJL-`P+(KCGqE z1Tc3}3WZ2spW5u88V%E-Qi9pYX4kXtFP6cRzO)_oyjRiMI^!Sd3N#4R_%zD2LTV#C zv%6A%wt|VGO(V9tJISxTs3qZEp&%KSpG{wcjVXnsXtbxpB3s{vwRP3kSds(Gwb+Nu z-nI8jMk>fIAZPSd6TYIMt~AJ6(&qh{BgzmEKr0w(4T!6(DU$z0pc_eN%5DYV6LQS*<7uaOT)rs>Wsma(oIsjE%FFTT;uqN{e(Xt*z7)*-!j=&M!wBR$T?LR>o;N)*wPmXVk=N zv8o5>@pS#6#N4wyB(*o$lDq(}8!9~ADEI806q?_`)2v?)hHt9o_CTvFWh?8eZQTK*8Hg;M4?Jyj#_MKyN2Eke|TxXy$}j#_n*=Y=d262~UcHj`hH1EyN?F-KIx^livgP|gS{chlpxpecFLs~)R{ z`6~nekqdd+u%5bkV-M_k!{x8WD%j`_x|Vyce%DnYC)*wOHxE4ym^zv5QGcde#RFbT@N#DGr# zxkEcI#q5rt!jPhpRmar&A76Ahu~>rC;8!>jcW5@mS?oWv2$onWt~IIY&g1;Lspo7o<6{jgQIa- zhD3K}H8tb_uF|_+IsJzE-j8hXl@KjMI7qnX-v6F|*o=wb@T>`aR`zAjgvexUT3YKP zvYFUx?8hui%lVS|<9%7bMX?lR&sXFLQfV4797ED_V1jN-p$ckrby+Yu3x^k;h;MK9SDT=N1%P{kPIm0_9CqaSi;y?o{Mp2g7PT4F zhQ+$ea4UX*Wzq=g9;H!6<4USxCLhLC4O`J3oQp82j}e}sEQv+h*<ZQ^A+3cYijmW1I6E4t)``<+<#OVtK#?Z_Rg5tf9ms? z?YYkghj0rct=vTwPv;h;MRpbd+hz)P*4n!faCXtui`c$t`YD4z_jaM7s9OBIq(cPD zLT@fH3%E}{@-8h}*{G>J~4u3%BF+F(Em5 z@lC);d_sb^jfKTbX?uHn6~MS=);oQ)cP=4en(*y1&UP&d*K5;Nz4A2jUTrA@v&=C^ zKm9>QMlHC;_O>#+>TA2TJo)y+@^XG}Z*Rj{f!@Q{svjkPVu@%b1eSDQb(y)oz|~Mv zkO1O9^ZWGRoki#uSfEGhZN83vfW`^Z(O8`ldLAdEi#6z*y; zc&Qa=nyiF*>C--JUeg8|6^I>}X?HJT6}ppX)OP`Ja9qyKEf$u{hkGnvIO}hWQb9k( zJ-nnrK=FrvXEBo5Ppv}OGV3m|jqv8}2-Q!7189A=k?_7pcf4(HUney3b$fSkzDsC4 zU79>FASX*P9^Tx%czkoYI_CdqV`O9F<3WMlDn0-7W4S{Al)DWrp{oqt?Hx;imyT!0 zM^Wtd(6SdLo($Iy`G5)ZmyU@x5kZK5hi6bpgGKwzw*LP93h1F9v>jzE zVK7sv!BocEqN%4W90#A{u~4aVQ11s{14u&OJ;Y?9vGJ{J*OXRdJBJVn_t|OGieuL7 zQUBqbF6i{6|If)!k_QvpdaTGdqWX<4QGLaq_rAX~kBV`j%jE8;E4vW;B-7WM=5yEt z2e@v3TwSMWSzN;5XFG&SKFJkHPT(5`=Md$rd1?noZRz40BSvv4*UGBQ;gObpvih|1 z7WFJ0hQ>!FpS#t00`_)VKD9`W-w_rw<$`Ua*lfYZ%84)jAgqu8xGw52O(=0c>MMRB zG|CGH7$Q5OKQ$AkZ2j2X<@YaufmyzU^xgk|ZS`+@`Z0hdpef-*wo>_3xba(slE3Eo z6nN#$VOKJ+;t?3-$*9dAWdh(E3QCN+PdiyBo%&x(gle+bzazHCv+Mmy1#lU-_5*A- zzRd_5MZ9@U#1=ypglKFK{UL}W6WhWWIFxc(Ur#ID*Qed;TC{u7`~pt6*&Jif9lA$< zYEtu|6*6^UJUPZQ*=}t56%v6DAl@Zo3aR-LJA+Zz#BEARWB97O3S`Ql1r~uNM#MPb zLFOT=u~h0pR?_~Z?7oc5W_{m}BrDVlalC|Ej>VqNA1mKTomDdW=zJ?TXW#Z;iv!8 zRk2@32e_uC#ex_t^kw&ou=2~3dWJ)u)bh&=PU~12i1x|JGNv(;cn0#0*?l4*gF(l) zjcfi#e|ep z{;4VlYaz;U)7tvFq37zCev|Ova-wB`)TJ7 zsvOp#?u>AuCW0)g)f1=A?Ydc8C0g3BpTt(p(br4==OVE~jH;eFy6$^_I>qLqmtU?^ zRv0ig&;ov{?yG?@etPikv2W}W2e^kwUbP&G1NzRkv3lug7EV67oE?>l zs%Z2Tex`s^{(z)}ORQBk+?hpNn)494NSSlIdKCacK)%1#eYX*59A=A_cPz^B9xj17 zC#Ci;TYxF_@&(p{jpsvivp2%$e|HG*?>vPax{e&w9ulIXBl^M|LX{LC=c?{K5`b9~ zMBdisizsd)oW$r!IvE-o;3XKeIph_K=5h1736dEpG=h9A7T4^MHwl#oe>!W9C7sYp zu2P>tulRRQ+l;&FlDO#|Y}H~W4wAzmI0(z1i_;n;KuVqrZZq$jA4P$S;G5t^%!Y>L z`U83~&R3!rZf+<@pNF=$ONuo*Xy3kh29ctc#dhq?OT9VVbZa;bxCrLY*ngVSVhmB~ zg7>KLX>sxHx8!;asU9!`fA-zF@1y}g;XGA@NhoiU)6)%9%3-$fQxWm%hBzvk6BRQg z-9}guAn?Zhjqt!D8yj19t(AH|J#8KPnodbLhxRuaqEbz!Z6&6KO%thWoy(F!j>wpp zw5jLT16~_x*6XEeGv?D%=T*l)Rc6DQ_{$R4bGDwi$EYKa7pUk!f1oS}I$o;gD9MM$ z{V42K=SlUIsLxUN-AhaDLHNw=wovi&i|M9YCofsslxYogc4YItvgM&fng%UCrJYcI)e;}3H;@}z;j|Evt zxd;G>qtplx_*Tcbe;aRXo-!|of+2nFi2EhM&~HrUQMlS$=&P8dIV_Yz1YAykrre0$ zKhl|+DstWXo^Hr9*=6PCPW1I$ub5D=OJPx|$n^|)<)V3zLO#D%MN>f?96Q+AaZR@J zqp>E-Va%e;Ak&Fl}FmPif>$`I)x& z>8J2mt&I_~xA(o@kG*kkDn|fJPI<+5n6;BeAT0PCkygP6-JPIAa3{D01`F=) z?wTMW=l}`se=fn@HAt|9|Gm4lx3>1vR9AI%b)WOmhvYfLVk^x`GfQzh5bTc{fRHAGQ~tovG{GG$Jwe|?mS_wffS!CyB9hcc}XJZCOm z%$bsn4S%XsWlG5JC7a7CHuxLQFhS-bhnJK`#2gUhXYE0!H$r8t` zEuA`wyI*rP8^D)UcpKe#rZSO}l4ffZDai~aqCj7sugOOqu0A#CkiGf77SZwI{_X0R zJ{e|Ue;38eM|0i!n)K0qAn9-4h?sB>-OSWJ@gIbbvvWRM{t3fB-|Tz-CLo=VO5y{K zXB*>yF59BZUCjsBJj}9A%f!#K%r2m$|Ige#)i4D&^Q9bXbVaVld-FBZR8%if@<+E} zXoJ)blX+fi>+&DcJwN&`@hU2eAbs@`3y+Pyf2{^CXA71}sS2rOlFhvuAY7KP%hLC9 zQSbgV{@P9W4mz)Lq4W%C+G|-0yu2SS$a=rp*!cJCj_6IIVyt5*wOHdm!CBE@81DxQ z-FLfpX(7>$FFswB?&VdoyR);+*lqC?2%nY|A3*022vTIqphp z`)owMWIbipvzOY5gEr4P>OJhL*fsfBfQw zXpnr>Sq=*k6~ft3)D>3Vo2QKg{-HrI8QsRBu8ki23P?ZC3sKRKwXn8WEtXVN$Rxjy z{GXUXIbF7DpZUdoCiusaR*C#e1^MuAPQ~xvn18Q7OiHCQj~Y5;@^b9d>sdymX>avX$1n&gV-PLj-G8;a{0yQ*6)1=i1e>B@DgpDpd zzkN>j`6_~`c^y|G_NVG!q7pV5LVPN|ysG&`tbb6tu|PS3RUb;r+rK15L> zs+%m#Ax^5IX-vGo%;3AVD@@LNMAZjYZcL@jmQFJvoQz{}Za4^e&u0`$qIZQxae*$WLDy*q=V`J;* zi>-lJ*IvoDZ@=RdNR_qrC5MsFs)gj<5xjx9#VF8mSmR;BeF806G&T2TDsS_A%h3k2jyPE#!KT5nu)lFcDxphc?yjz6@J= z*s~;PXoxg8A6esyW|_&N1YqcnlA4*Uo%?Ohtxh$k4gl8M(kl7y{Euz^4 z#ahw>6aoUMe*TFYX4Tc^&$cdLstFl#?yL_^Yk4aH6=hiTe?5}ij6_toOz@ZG$%iCN zup>j7NV94ZQn_DduM(Ic|3Oe*S~giFrP<7=ZbZ7>5JLWqVzVBq9BGsJH8sRtSzlwA zl#rUYUvs{Gef;)miV&j%Ho#$e?vxPy5UUoO^xQWJVMZy$6h6dSN3OiQ)#DP&Tic!I z#W!h-jEwbBe<9W)?x8&`Kt-j5vEwNoQ(ulUF4|gRWSlzwSnr#sJbYj{Q2BtbHgynw zD6Q#J400AZ^=^^XTk1V_%3>MlzfNADOja)3>v%I(_9|a1qpvyOX*=ZT!0)M^rcJ#LDDwz!^{M)2Ep6@u+=vg#tCX$|0Y zoAR)R{YTc5!-+TB(TH(KIsJ1l-P)m!(Vd3lOX$(?quPcd>DBM3(a`d_ahP5eh-Ybr z+gNNPc&p)1fktd=m~hTqbIdI_gON8^jq7(Ne@xbwHWxyKe4ZwVk>GMUK1d4So3Y5w z?r94B;VybFC;MSs@b*Uh>>y*MP$t&z?3HW9(i>O3#ei!$pEp&(y!!iBD0;6y0q<9z zwJJK9TiFG<&-GPDi`OL2tW|aaIUMnH ze{gHN_Awq-Wy_uT!%Z&4+>{(4v&hKrS-P%bP1?OSJDO)HEyDO=BOX^8SDg(}`nlO~ zzOl0AN13#D^c<1<(|3C1#X&GORUmsbcZ^q z{G*JsPcmTiK#P|H)zSYG`qQ{clSva1e_qWut$GF{`ic-M$cFj%$=>R16N8CI(>Mxm z;9wF(#NU{4lZq9HaeF3N(-&ud`SNAe?_fgaK02%Kr2#<~CU0wOtQc=gai5!6FILL* zUeechZ18@b!h6XqN$B&gfbn{eE^H(cMB1ECFtDJy8NW5>EQAE{OaU0Pn)caSf9#XH zY&U*9(CM(zMt77}=YFVeBkI87x1i$y1?|RXDcAM)waMOWC0_!SVYwA|O7Jmk(Q0($ zxr%@}MqihYN{--9=fl|wf=Z`1dheL@qdTA5*N;n!O0Q4~uXje{O{Jj%~wBGbZwU_h?!eKoLLiViCEINBJuq9UYO7 zqvMC=j{`b5V`fNTMCdk{ zYK>u|jJ}?pV^O|4pLBa=e|Ofz!@dzBJG!W3l7jHpWFP~CMY@hVdzu|bdm3oMsRdhC z|L#2dKYP#mvcfOfS@Jh8^ulj&P@9^P@}w{6#BPfif=H6OP+)ETpJmcazNDEp^bmju zKpVG4V`2*0pKoQBVozI`D%kmslDZ0zuc2;c{c!giR@x$5F+h~}71g>w@u|Zn zxKve8m?WKaiTphuf96zoRu|HbKgtY*f)gfdO9E)Y;4)kUBz29p-SN)~WS~9fI=RmDT`CY#SE7YQjsOpvV+jhmZm+c8X7o$#LN4<3 zqbD@>4a@>9yXYQu{)$#a#OO$={GhiLV+FSe?Wcl_f2?0fJQFEFmVuYqV^ZZlWSu`q zMxW6kKQpX8@FSX@oPIHsI4s>1&=4M7xX-2eDIb)kD`?b0V!1yKFV?BX80g$Cd(8_y7cFb z3%Gk%d*6eefWg>Ym@_SKZhL!s zE_@6LiK}6Y;hzaB-i4~ZXsSEVIdsWS4hGFBigizq0#FLt9ZT~u3Nt6x`zKsA4 zASy3^AqFfhE-tQaDbFd&FBcn>?akK9jec5ZcK|sNr$My4oG%VNsi4Ybao1tNY{Pxt zf23(DQf5VmEI9a(P*7W6u1F|y-$Qp&(pDv4usSXkzAJQ&fWf;ibhfte^6xmJ%dgp3j9iY|p7D&&FY0*|6ra@gZ5y8P)x+ zWf}VnV}^d3&lBQ*4Ckx-dS>REsXcE~e^Zk&b8g`rvQ*-jw)I&R4FBx7qXZQ&Pn<>l zP&g0zTUcJNoInJ6DB##N{9?&QmcC@v=!}O|IB8`cWEWO+gLd0lmNt~o~ zn`oqg;MB-^2CI9I#h%=&*>zNi{+vN<59S}jEK&J?`s1~NL}kY<{n zHvD-B1B{~OVaFZfFvGMSM`wl#`3dOVT_&k76SYlx0KU=w%o}~ezc_2EFIanWw1bd0 z41!F}K@T8^N|)%N3->8Ij;n22e-i~9KEbF4mf{%^GW?1d9&m;NS?c0X@tr%^m@`nB zrp)N-i7<7AFSs(m-mTdoDW(L8>(!|N6u3|UVDj!3TU?$Tj-5xH-Kv?P05fcuf=@_D zXvpt8(U<@w7wj>9)*2#(dR|EWVC8DujcDH7tj^uJ z{h_#OU%k#yIlz;95ShcvnmeiHnF!K)_o)g2)-Kit=VCw;L}w6^Sof!2uAe0D!~i(# zI!JO@8#TL2lcij&JLAlBDm($EFzmzr*gksIX$t{P*lBA=;a@$RqKI+2{ohLp79AK} zhetVHgpC$(R}t*G&>xX1f1onJR$cg44A4iqsp?@R(l*FCQDvsj=F1 z@?RL%&hs5KvOV`m>k$EfcR_J_^*j0!+!!LS=Vp^8TuK?n|?3BXMA}|NWk7`(c{AUU&f*D601U3%ea0;NA#Ek0xwL$uf z2F=YU|Bg8oe;9^f!`%R=Va6GPqY*n%aoO3{0mP{{vp^A!D4>R71p5wVDOhm=&AKeL z-Kc-%rRct^+}(^f z%cU03et*-Q0}d7HiUDK#co7N~P31_!_U6L~9ePee|y)t_?!JI|E-=@`aX(tPNOc!Oa?ZOS4$(Et6vN3YmX|# z_sd(KM1IQ(tM7uN=9E=_4uA4|sR=a{5^>OR1N)<#Eq3y@RTX$hp7kWkePGPlE4hJ3 z^1Uq~**kQH5oq#6|5;T!n3l}@!xZhL+tv4ke^mXpP)51LQiFda771}@C^zzFy%B$y zH8ox943%=?hVFJ1pIYZfhHeQMvdQx__{MSzv23g~OlDm%OwcsY99zgmGwp^<8O0@rtH4OqGw-roH0$HtC&ylN|*M9M#l(dDrYGKcyl*GJs` z8U;Bqae;3V&KS9yV+f-T!DTSR6RZ&ue_AhZuhq=tMX9OLFa-aZmAud!k_D$K-kq|ID_=ETe*7#LB}=4qc~!ebNP^oHA!CER`GYHw`7i+mkq_z-EAXW0SVmnk_Al z0r8B7{Y*G5ZX_8`6C>@cCMl;B%_dP2iTSs0PzdBVMV~k$Nn--~$XNN>DM?nJf6j;1 z@8K*q4v41N|NA%n{E|8Q3IINLZxs-v`g{Xc0C zKjNEF9tJW4q8=U|X%<|ElOK85*Yju%5i5(h1>txCs%b13QDZB*PI9Hn3Jlmtm@cBe zvcSm|7pU(Gm?S*JKbPn*6}=mYeJL z-0wb-qLnME$^jA1K9-hn<t!mSk`j>lwZuzGw&f8Y?V)e;sAiA$vSe#3sVf z7MUpyZFv9!)ad)YJ#a^WHe}VQ9H!@iEx+l70|k{m+G2OrAq@N|fIr{Of(J7>T`R74;&y~i?%^70^l^r-G6oE3N;2Ou6#SitN z7r*p?hfp3f@^ocke|VnU?=&94Mm@b#@?T;05+8Hhwr$U{v2rspxXTrp;_vHU{A~Vw zG)6X1Du1zMQJ7$>aL*cL^5hEENv2?;F!)Y_lp#lVY5N7w*ySOde~?P7hy)~Dvu0o8Ku0N5 zDjQX9HOtaDYUR(;l@f<%5L#(rAf>And!5q9zN>moAN)>BR<3d;q1RXMy?rfg3CK%${<6!8Qulj_MSgj)97lT_x#zdzzxMq< zgWnc9_`~yD#4B6;{P@kYRixza=Ow2M(mJl` z^;vfMVTK0fw}8(gBC6K)B17q$kACU7{=E(4P)T4ysDXoz@osHNU=;y)Gt4U5bp*3z zkPwv1@=RFs-23+xvYy91n!g_~GUQZL5O6;hbymM{5Vl%EW?UYlZ5H&57#u}Zsica~ z^cMorfBof~t3`+HhMuejA-~vRB7POkhR|JlQ~^ywgdlGIc#F1u&04NYQQTuOfD+ac z{mtkklpvB6OiN#ChV3l&gIu)jh=KQ=1|{WuSU7OtnJ!OFBNCluOA~U$$w$8E&k(#*6j8s{%X87+kHf2oOIXWZf9ZEv-u22?*%TdLoKsNNJN z5EiksvYvz}joC$TSNP#$b5Q5T=C$8n7r0t80>gzqW#g(_;u;VZaZCsaL=kQeeY|{a zhG~*o-Gm8Wm@jNxAtz3N4W#_^7D}AdQY%snTg{5r4#O{1{HKqyK+`$7O;o*VJtRH( zf8tkCUr=mMEa<_A-*`(sTvNFlz$IJ z9%Mkym+?d4xlxI{D<6_6vbjZu$88$Bf4xAP8`FMZKKem=dIa%^P30>nwa5p%2duk2 z9iBWNh)lX|l{bkrE?cwb6}VSoRZ$6IL+ODFSGSM=+XHZf|DHSU~}9rVEFiAUz2dsKd@h}07}Wh+uQIM>Q9J*m{sq5eF_ zb^Ev&CxyC)5qrR27WVbdCryDux87Ys_7SBYHp^>*gN^Qk{3QNWavKZQFZ zNE^uwjI$`BnURBKSrfK}()xQkdl!yVC;0BT2zd19Qt)e1aHyxGY0J zu7>jdEtCVmm*U*%C09cg$p)EDcRPY<%1O35V^8h{tsBw1DxFPFe)*kzI zqu5dKrg?PY&xXYQY5AB$95&ES4&98;`W&al+x zF5JqfbV?e9wfbg}(*`pe@o`S3FDJVS@l30%(AWVGlV9FP>S_qPw|TT;t!rZdZd(Qf zQtz73;`4cV3@YC)94Z6kT`(lyzMAdfem)jA8ly8HUVP{CJGOW1aBWqEC!%BR>u;s$mhtXBsM~>)WXcz zM4@WP!30oec66FyP0D;${BDDLVhC#2^jxPHEECZ>nTe=Ma8d`7b{G=7Sc%%rrJ=QT zxd)lRtj|n1F8xO5!}fk5D6f)f{8h|?hjG9QJYAMHx45`Sf1c-RS!zZU^0rZ4;3l_g zSlYf10e(2rxD*61xRqv{&7k1X%5UcOj1$m~oRWh>ay9PUidiYm&_&tuP`-ca;H0JX z8W~80opW_{RZi!zxAFf}K#d1pyT=G^r)G8dv&6zutrCGZ{C%MOGRyJY!<~@J0&xQ3 zb$;lJ4{ek#f5BcoKZSD=GDm;XBr4@v>F&EA)Rk1$y8m_h_zd`RbCKN*jqzDdqE}aw za^m71Z?#>Hegi=}pSDFNCMWs-Wx3F`FU`Dt=&Y&HA&cTVE%I@q=~pGq*wPN0CzN&( zf{%(O$m2-BT3b8>N;D3HbL(J?y$Zb%phKT>c5pZ3f0YzkFXtx;U4jBc6Ch^ju5N!% z9j(IL=x9m92)Pd$ubY~TPm5f~yCTb$E@(!Tec7?l9*zRb@vn?CT&(Rzhe@`EhtZEQ zB&QA1@jYvEWXteaCEOD z3?8-nhj#fZ_RJIl=;sUke{{M8e>K=~e~aHzw(Su;y>Cz+3Y#B%_z?4J&xN1lTIu7* z^qAZSvjr=Vyo0os*35jJiN^E%w$fnd)8RXF&MsNJZd*L<4~8l)|F?A=NwMvu3zuH`p}EZJDQ#neO7b8Y@BdYd$?zG3oS9qj$9>bf#Y*`aDzFMPOni~HtG3Fs~}mGVWSQB zDviS@hD7*iQO;V^uSa7gP4zwZn?z|9tu40lV9d5QumW_A(9GBD^AiX-1~=7J@udgf zjDSXG#&S+!>b~qx{$^lpnoo<wRTtsU+`NX}>2H5cpwD&G|M#l;_w_e^hoKua5m2 zkhX{!9h6okAcCaz6f3QUj??{Tb_13%!`k@*J$E$gZkrKP(8=)^d|9#EAy3}RXC8ey zy22rSFvY>%{5PtQb}cch(ek8B<%TwKMumgMgv4l%fF)BdFadGo)7k{mddR^2nfS1x z;@#Y_^T+2N&sU{4(X%!`f8$@rOs^-ioi=_!pKa}zed)geNRDS&DU-5An@qq1uV@X| zIK#FBV}TDd^!wnh`oZ-MZUSsxM_DsLWGZWP#OaAB5L@#x=w{WYwz{;{M~_Qv^wjLJF&)UB5mBR&bBW+#@U2Ua$R+Wuftznf98}-9I1B%svoduvOxW^S5<^ zC@}3(YrmF67$jQhU}~GXb6;^I6rR~*katG>pJVNpV9gNpbke;lP1LVJaiJxe^6un~K* zMi#Y`M~dwzKCUkt!Nadr?1(ZgSY6aLQhN)xm%x>Abu}%?xvrt36ANY!6Qw|k-DSCb zRH|4diGh1Xc1uc2EeW0W-d4aOsn7yG+4^DsszQ(gy!PEx-#)HX7H_uG&7m<(t!-+G z$#zs$SAQ88e@q@i3yDo}e0dNYjMz3Wpxffgk&sAlR#en563jv2tNR8~6;B!&LLQJ> z4ip|>t;iUeb)eec(ACW@dBcyie2@`D(UU7*67Xv)Y-w)pSH11`y?<0{@wK!xMzUyH z-~|ckOq5m5zKzi3tBrA=W}v$PPcJC@rirt!C{rw%?=qLX+dZQ|7K<}VZl5HV|lir2nT$1 z3c8|e2+LaTqL*Y9Cq*^@W2cKm21<)x7zzmqt#axI18bMu6x60TBp3oD7_C&0mtPDr zGymP~e-X*gpRs#mPyK&wshc!+(vVOE@)FZ5fCIH%pRgWBIPac|C%Lt5<$sC)t?w~e zHETjL@AzT25Yg6V@PWkLZsEJuZ(iVa?VDbf6M2pE0L^6jt1c)5Gr2s=7lndNjKx=~ zlCJm<Qxm^gouFmF8Tt@RjMBRUwsC+S}Xall-9CkP^IGZ?F7z z8%%|?Tb@eMH?CiNgY-1@FI&6tJ(z1;y% ze|U;ccw87hjS@v8P2Jm(35&?_Q%7;>UYed&dX=1_NrV`3kEchdG=S9Gx`R>EAA z`|pD<3VvcW5wxaIW4eTBD^=!Wa z;peDmY42p{G$dxe>Z-_o9mYBU9F0eKejoO7(4+Gsn>_i(?36CxOFY;L9+wSvWTixf)e zI*qY~yF>v1{3CRDTrQEGNl#VnGe__5-+WazVx=0e35c@F!L~)yk6+F-zf@ywf8g2+ zK#YO@RCmt80ZkNbGd+f73H;vV)3jaHgN~fZzaGxV{YwiAf0a1bVe62~|H7wDyvnn= ze>hrgVCP{ifUW4w)zKW?1>NFNdSv&-w&K{>ncig#AqVX-hy@oc-QM0V)pWE#H4mr$ z2M8+6=18ZUL(<&4=dB3~#xxBJf1?`Rl?#@42S~{iir|wKGPCk|#AW>~D6kaYM_vuQ zdRG5v4nk@rprti-NOtNxBBrAQZks4EO)gX37M$oipIz$zrAt&&OhZFw^rh=+zhrED z>xRiM=Z7E}T#y*o-J{jnoonpaD-t@u$jI9TldTP+NtfGZW}~FZF1Fhue-W2o_D#jL zHy>nhs|Brr`W>MzjEW~+_4cU`^i+z%EFaVV|AI^l3;0wV6Ao$SOiE<&pCD=OCJ5=NtmfX4~HPp>dOlc_%3e<$p1^aW@2~r;~(SW|^llcmH2z6o&{0 zFkt_}kD#2>hJeQ~J1Ev8C-_MVMiqoX+oSG^qyFw+!6^chLR=yqe-rTTOrKxrM6C-U z^9ipCk1j)TWH^#FV;00S@wTF}Qs%{T=8xUY>EGCt^_a+fKBtiTPOzpypoA^8iS%=F8aA&3a+`oUz_kh`= z$5~y4J#!IHdBDq4IloISFFhwZZwGhzzJ&Zgo}0ertb53LQBkgnnLK}SaHK3XaJ`A3 zwE*+F>5TQos6XQPTd+AZNnXq_++Hmb;}>#KKHiTm3F_)GeW6l^wS;ZqRXc`4`ev)XP0#K%pi(HG!Zs(ceBk~}c||gBCJ1mhYf=;B z#K1q`Ib-Fq?geQ`N!iW{2K$^aN@>lY_K%LL<7E%I6%RgB_MbYU=%?Wzhc;uVL1^Kx zFTKf_-oQ?cf2O}V@+PQuF+d?`0TWRcC-3GuM&DT7#l*%6fPR$*g-l29k&OB@@rl&A zlt@c8lR?jdrD3o1f^FL9VCp`LiA2PyFW+H@J603gTy$Uu$0CC2zjkz7r|;I-h72u(?a zQ!!^$f6B+7mq1)W+dPk@@!4ZN%^$&XFL7$Z66j!(kttF9=qWBd7j|QHivaO|x-`aE z(JkaR5IiqnH^-7BFCaLFee|Ls}Ec-iuz-}m$@|T0?YCdvvz>i z&em(FlN^k~DAgI}A_dXM?|>Q(47wx-#LD|Yf0UWayUS$?53iUy7{$>GL{D z&tC1nls&K7U&$m>FUe`j6)v*s4ry`svdg0WuipBht3W=egvZCTCIZDMnGJ7+$LtY< zjP(X)o|Vr)N;&>xsT^lwkC1)=~HuNA&dotH4<=v4C4z^_$k9u17&KA{wrCjI6qDD}EBcAC#)V2zE-6lMEKfzG7SLDuAYKf# zL7^Dz!#L}@EF8_DFq`lA+Iw1RE=;q7lTxhceQj(ePg?k!uZ`l8CyWfxd5>j0eI9gjTU#<8U(UL3thv4;e zr(|LI9kIqeWR7jhfjMkf1qZvaikU4)$QZ$pK}JMt4g=0q*3pTJP_q2+s^s_juS3RU zwuCbX3q6MSd9iA@jYLW_PbI2Ge}t9wt`jHl zVG0X&?$7nYSY;AHCvYY0ld=pW0=cZA6^d|JB-C4Tgp-4|1^Phh(uMcO@A!ODNRyV- zpxOMOs6e0#?~LFn8BvsRjVX)A{@`7`Ivj+y#D3x8J(=g4pT^4Jf%|4nf2$vwya!I= zImf%jWJ+4eXjL=_J);D=K@3({x(vUuRyS3&Tg(aT)5AtUNr{Aq zTFX>_mH~$LCraS^8&?@QEYw67Qi$DPp5@PQsae>o!#EqH#=mZoux%wbB>Vfkr_KKk z0)NKiU{_zXJ1ir#-R%gRe?A==XV!#y)`14Xkhn;MIIEm<_F$I@Q(6I0VM(d9QMxs zOW*Mj!h)GV$wQ0d8fy1vak;~CghRHOp4bJ3^0MTSk!r)kS$lS~e_nZ3B&e@QrFJ#x zYtL)z8kJODZC_G$^hf=qj}2N76xb(Uhmqe2Y1wTvLa@{AZB$G;If{C@wbf=uucykj z(ihean@q_22oipVJfxc9?of=*A)aS5M>&0(O`kByDockv;iyjyY$BSLh<`B)xsiY6=P=Xe4Y1 z86QLa4C9FkbpeaZe5fz|zBzEp;I{e>qr}zHk^-x(<)sN0e`lFXFk%fO)6&w~+?y~( z^!6UL=Q!7A+>6RMz1#Yz4)730LhoJMore;zJrUy)r7+wFm@pc%oqVpxijBv#9#QIx5wg zMEmFdZ;=qo@%O^TTgyOB$f8e@``!vOEd}o$2I!eSfBmIk3|4_MP%pmHtIrSXn|(2V zjk>x92J^duDe3ZTA8vlkyD-JL2~ADGe+?B3VCxnDwW*Zf%V!@B0M|x5!e&&WYzU%^ zZo3yp%l|Ge-i|BuNB#PN0Kdi;oYhyHz(-&~Ws_46$NbimT)cg#8?4}YRfU$HW3>!e z5Uf%Bf9wj@#U`fz1iPom&b%kU;O3J(dU#{(HghzLNOtQt{jx+E+_CE;ysOw)%v9wK zfzqMQ42sGM+5ETvTCkw8XsSRGcd)8{DI;J3*@h$=rl#G34aB<#FX(fl?A#ezvK!3P1rKibIS5P>xf%mf7w%7@y#owhK#ufZ)PV2B&%AD1c_Ts z%d?sdLV@Cwt%aIwFtXS7L&(wYO&MIz7$eLXI)wFG2EpuX%7v(o&e_=IEkwfgNL*ReQ$eeIv+q8A5+Xj8Lq5QD{UGFKnm{f$@8f3&n3x5nx$ z>{7LPBwNSRlzOsrNt%=Gk!ZkvJ>P=wnPR_O(t^w+`~)s&jdXbNp+p=dGBI!wf;ony zk$B6?wKIMrukrYs*}w%Z*;q{swG51ue{aKHiLhEqE+4>a;Ia6VQ zu^cKD{q6iJ43uqlo5|1+b25?MxaEFU|DHPCRBy4)fz;1;&d`aDb{RkV41|ymiYNTV zC;a(glYW1|7A63C8^{MQt75rO?{o8?erIE3WerR>B?4PqPpLQx zVw00UH$Y)|Jw-kRvHwE+9syg#XDPCKFH9gY5y5c`lc~Z8Og4_itH|b$e^~hJC~#Ut z8LAmMWN0>z?e;8cRsLJTApNJa)*oMxog}@i!}1O!X;By2`lFqoGBq8&0?&#-ocL`eyQaO_NDf?gJntrZwj7Ec%pL9WJ8GJB89Y}kne zW(as?3)M9e?=8kg>y>i-K#~K}_QwuwH#t8#^G*Ei^z;s1haSA&f6Xkuu>1Kv9)mOb zmXHvwGy>``@8UpfmUQ9iGvH;gKU?-DcN%Z!##}2tbKUyPpU$k+B?Z|ICQ8bukdu>~ zH>tK5-|N{l3xfWS;<>oExUas4$5{1yxauyK3@VxghIp(yzW0Ns1O%Ns9WKChVAnmK zIybvMrEYKob1qgXe*&EEdle_UByE;79|{f#sJkh82XP66h}R$@rB`c10h?II?HsF#LV zK^OjxZ|mJ{a$xyC=M^qm5G4xFZsjKf9uIF!$AwXI`9N)Q1;a( zb!*`E#Df*>e@qG-5ji;{Z(rWvfeni@d6Hh3du<6lg*2DSdh~BrC`th+c-ORg?xP2~ z?(;V;^wso-Y-5*)+s^{K0`rrT|A=-4Dzc5e*&;;Z&@fQ-*v!RdqJzuM%4Dk7DW08I zn@Ir7AbX?=kKaqD&98?~<9EI76cr79^q0sBxPQ2Mf7MYzCU=UDWfc{(?$=;3-|eJm zEw(zc`L$oql^CkIAvBSoi8msmb*Ta>;XA=be7lHt_?>U7a>|Aau++DI$>Odd1Dxf` zG6Q0m@0a31ty3hPPG$~u6Il$UOo3^a8eHLOOd`7BlL6LHl}EJ`LSyjESa{hQSj z4;|Mde~7Owz{E<6S?S5Zy4%*pl_p>IsaHwHUU{q$$l>LnzkyAITg#mE9Q-NqL#%zi zk-d`&mVAw1y65kSQm-^##282yg_QF{H}}<|mWx8rLv&_=-frigUK!U7C16~k+$U?s z2NPe)Ra}K3t*)xI)aP2F)3;ysc8{Cs&8oR!e_9AR%e|VNkIy~M`yP^_FiS}`M5Y1( ze9?EF%~|RmHPFNL&{pig&((0a!c5TjBGf#h!NE#J9Ld@krDyA0n8VnN0y$b3x*g3W z&edyFqdy2zaoE?m>akj!EN*POAISjYosgeDFQ|3`Thv;1h{0!M1QloNb6C1OGg+AJ ze0H;~rT#SZl{HeKZIxPuC!Q2_&{i zwga|!thq32^yARV3o*P3+Yk$?$gptTBidgC8(MLw%tV(v{mC!~g$GHTDD=1ashV)c$LXm)3x)rnvgX&CHVoG7@b z5{8mM$G*2T!9hiWlousB80)zM3pxVSG-MESNmhVReM+rzq>_k}P{at-C(iNGyw-NQ zB_i)DtwbJfZ0bnzh=gn2OS=+uus;jH16}N3`h9GF-sdgy#z@MlKah5PXMbv4Z)p2V z->k~rQdkisG3nibLJ^>Tqgl~Ai3dNhIFN<>mF@3f`Px!cSVgy?mkgDqycw^|AjQqu z;n8lJm6Bb2Ub!J`I{0t!v)~M|BhC9auPH)FG263yc>mAzB%DbGvY44PybB5z8bYGe zXLH)(6LtAzdAR@!C#E|XO@FPc3wR5J?}o@2daTHl_{e?fc-@v{#V6I{knLP?z0vUl ztZBi-`JY!=yZAwT4DI0S7RV8no+_Vc#qF*)d-Z4dG{#~o)&?O}ozc{w?T{mwujn5X zqzq(pdPt7)Nf+AwK#YR04#fC4=?74ek~P9CkGFhxA=X%$GDEn!i+>Ra{rD%&b#)J_ zwoL`3nC&PYcU~bpNwDC~HjqMFZKC4Q`--WH@Wcvd;iAt8e}PdxCF0t1T>_~t^(X4> zfuRdl{fnN{=kY5TV&gwyA-rtCFTNePEGC6I_+sD!D+>YE_=+0HrT6~6B5keh#jcn$ z3WRe?f;9QcIjoXam4Axem4i}G@u@yfZZ0fgzf(qn$gJFOg&-^$@6&Fa*x-ixu#-2s z6%{@`_iF&{@XYz&U=t9ebUUx=-XmHI3W6hb8n{P;m2JA&&uiLnbLy!ZjO)ox1}_zg z1wsIfv#=HaKt5%*uSt%Qiu@xuZew@6UDPGgUM8i1VR9j@U4Q2lO`{K|gkTC6GTCfZ z#hQH0(Mt|j^{))zh!`jSepOha+z)>QNWhP?*<4$Bi$ukEyX_&8nf&k7I#hM+@Sa2oOvfft_Zfpqd9$) z(>wp;7G}i0LdgClIF{Xxdqs5?~%w7)h7;`MJX+phsgoW8CHsLdlUi`83wG!(umISb^RRjC3sa_q<-=%CoIjjYV zFBoY>T&$ffO9G$jp6G(TcQ-(OKph}E5O$G$X@3PzqAC3D#ID?)2=T5+TDYN2g@xC( zZu>UsBO@c6gsnR!wxz2DtOz}g(RB2cwsZ*>=AlC(K(Alwb2WzP(AaRTSFE)n92ONG z={vwH;nROF~%6rd0&@+Pn+nOu zk$;*@4slo>V?@31;o?lrS`5E4yURt_5v-!@94o)346d@YL8-tnPQVJNd~yvpW!+{h zw%EWzt~3*Ww4Amu*}+Ygxpj#PiETq2A(>UWbxX(vY@M6}uSUq6m??K4Sc*m82A*vk zZTez7J@)Bo4kuy^ML_3+7A4P$?#5qD$A5j^zwuMg6y1|AlknZt3iZh5q@q1T?BT!FrpFdkz6WxOd^(5N|LQJ zH3<|dm?X(_ENQ@BeyAq0`?kLGAbE2v>LLOf2A)V+9oCHztA7&smH!>M>IfTb!ha*? z5@@)a7$opUwfsN1TIgFrG`kAO8nep6rY zzaui9AsGF>U=>%O)?kXoc%!UXGDwUWoI38B!fayrbVscBMK7Ki^e*UanfFg5Ngb0r zQ)xx*DGXh0l`zeIIM8H|;ZUN6hkrSBUDmFU*3CMVLBQ%A7Pd7WkbD`}hA7~)XJ!E7 zPVsD4$o*K)2godQt2a;D~+!yOiw|4CVyA+jONwd z8VS4UKVL=*@Nt>2E>6B1`w{`_oGk#{FBAwqYe4;zRpSfET)v416wukyU;gAEzhtx$ zo}$6^+K(_jh~OweqIa1v%H$&?f!ta);XnU12hIdt-*6wM476g_G8(E?^Eo^Z$%d>( z^E86lMQY|eO~|ChAI}&>U4P52d1#U2?S09*w}!OAEPcy|Xv4b`M{V)s$3^x1joDDY z*I;a<_Z=3GrgMyWq87uT29wsr@*AYetNm1HUJkK_fk)fzcv37|g2>cOAL&RN*M8Xo z6;{n$xXPb#y|(V?`!~)4!teNh?2I>SwAhx0DhQZYTW6%XUb}S5Hh&NQ)FO;T+FPg~ z{Pu@Z2sotpyCc&)PC|{Pkz~T!y3qJMQLo#u5xDZYv@|y-@cA^?<^Iu>Q*jpR<=eQV zuqh_~EWU zC1U+;LfxHC5E?f3Ie@Q`^e2Off3K!VBnGZ+GSK@(PI%9!=fceBPQs5>BZ!{Nvdp(} zt2yGUO%<%)+UE)w1s8Q2y0&TxsS#rLVsw$tgzupWq}oclVSn88_4il%LqZXK#L20= zWQD@9_yh#D28*+f&yJ2B>skE-4N+$|OrfRc1$InSCowu}$tqK|k;r@v^|#5zHE7DF zwYyVi6&jz5;#4w? zQ^B4XTEI*ZhkssU&hai??sUdk7i~vdw=)^RceN1|kH_k(8w$)tTIh$q7nUUF(r^8# z)P0fqlM_{z`p#K5Ch@I(=6G}NZGO*J#D!S+8n{k7ELbHX5TSIkAL=`1*F~m}gzb+U zD|cw9sD9c_x!ivy{GJ7>6nylXhe)S1YeXUu=0p+GM}O)=PILA5ekpdBufs5s{^Bz^ zaampGSm*TQ9@0SGi1SdI!>+Hd@f@L+LM{y(PmuGo^1`gBLgchBv{&XHZzWNI;e3;W z>mdP!QfJF2@{1R)Q}l}~Va2vdAZ+3gI=mee$Pb|+<%(2R0eQUDs3{12t^2qUx04D$ z&xEss<$tY)tNC7vqf$-|H@>IhM+@M@F#u5r1IEBIL$voaZr8SU3>W8*9kV2I&v$1d z{X$Tzn*47s5r{RoC5xd_fbj0|$gYt)sdx*?pGn(EWL#Wa1S=iKFN8F-vTP^wbubu} z{??RK;v1PVPKGTcF!@`&%(YqiY@G$>6{KPyZ+{NSQJDQglWuDf+6s~bpNWzhj9|R3 zEFhwM8#BvaJ8UE|8;61-aA|wc2H9Chrst)hv$GVoF^NezGu${U^KC$X;_%_{(?lL@B!7`#uk;z=x@#w>q7t>FEX}&_V0NNvBC% zX@6fZxpQe!Go%gv;u9A1ImC^l7T#G|_hF*dNW`n-K?!zI+V<3Znmz+PMaDi;DX;8Z z*!jrpIt7?2D_`p1(9^~sfL2%ASN{R3iH4beP5wtk6_1mY&ryL+osMSHz@|@oln?t# z4a0p3slJFDU;)P9#&WUm{xHz?pE~>T@PCOJ9w@bSp`h$am z_w_Z_RPEj)0%#XOY3b=2O}W@-vu#Tx1O#_mJtFAU-~4%aVZ+U}JtnNT8{$K;JZCor zQ)bU8Ep*m=_9A15iZ~K#&~f3Ow){o4|C}8hsVFHK;UL0x;E<5u_}oPe6k@-q7i+9`8bFDk%P0_+nTsq; zNx&VZjka_MZ7@z+!PYUpQ6oWw2%G5QYL(HrpJ$W*`$s?^&#M32(%PEDcy=7;OIQqg z{~BnXIVFii@r8+wI8t3VR%OIBoqvBev8t5yoyKX`Fxqwu)#Ak{EDo+qK(eI%Id(#J zYI1UOZegLjqrU#(=kjWHLbkOl2mhtlhqQVb|D6d-VXm@HuFQr3> z#6N5ra1&j-DM58xBf|!@XMRHS@VdwVB*GIF9(wKb=jiB>rl(82Capg4Y=4ZFqeBx< zLE(=Iu|_wVzK{ClV}!SVLV&`t)D+>14VlV_T~)9LvRM{fp7b627=5$)CQ$5C$<6pa z*dJXJpZ2llT9L8X$eXqZQhHVK{yy$nBu{vL2BIR2f&X=D8^o^?WHiiM#lAXs-0RJyxWUxnjfxH;n0 zhAKw^a{uenm)+gjDoIL9qB!x0a5L^Cx&hvrf zh$`genSt19jxX$QpQRVAVy;IEg$fUo%3Px)SDsiTIiNXH%Z{irUDZ<$RFwbrgl=y~ ziJZ0xcoZoSOzR3b^?yypit%G+$DOO%y%~-cVT$G#@wN8v8bH-d$;j}Vx$nK$f=U)! zSzUcWXznLuNmlC?fHH^K##f$u)%a_KHxbFXo<|ctKWG_*M8oD-3v!OW4HJf-K2y>IRjt&hSowvW;(XL3LFG|oHA`?}tM2~5x zUH1-<3|*kjWLNt7&2GVy1^-eaj#<*lT};jRdS3gH8~47Q|pzXn(oB;yI2H`2!MDCI&tk3}^&wSYme=CU&xBKMKd%M$FL2cjU=Zx@af~eD6K8HZt7T2|1f0R3i)rOiAH3!c$Bnrt-32m@B7&$C- z3fYd|RYIh+Lop=Up(~{4m5BcI!*XiLY!j2gzgg*6;(vCUUO8|kZS}m~PkHsZU2snh z!vM_93${Kp1bj|NkTU}}!EordNMVu>9I?)?@~_Eki$DH~kg({5y=C0)e>L0_a0s%c zmkhw5-|h=a2Ze51E!x*xDJ$bgvvC-kzP|@LCkww`9RE6UKA!*as;ts_Ds^gKDL)Wj zA(zD(6n}seLmLjebh>dRtHD2B)F!@!u6!h46c3q zR@n!EKk<0;X3`fFfLB`}ZnsqGY~t>MA^A#x1=^H(;NA8hSDv%QYhDb$_N51nBH^To z!GFQ&`sq}Ql<2gJWf|TtjFBIn9KCtupmap<#$w%RQqMIjIz5WH3!T^jOYg?EHyy`F z#r?-m6@P3jHDpLWePq_{bu3WaBWBnu8-E2Jy2kEr=-`@zUITh{$F=5G{iy12K>5@p ziBV6D;)F5EZ!I+er~+o>4W8KWI~;;0PE$Hq*4B;}aR4Let#(j|D`BV4IjZ^t1f6&3 z>zCbh9;fmV}?Co*hGG<6?<6r`ps7y8i}g3gEOucweOy+cn7FAzhoA ze3O;WXN%BwltlZY9A-AF7)M7Zv;tWYbZ^>~-EOI>GNSXU_1oJS>1}A|TT2Ej zT&RioZyqzjXPksx>%-+!@mGY}#I!0~Myk+99Bxf}`v+_7}3=%xoVR_SfA0St4HkH+T= z6b<1GxZ&Avrnqi)by1Oa$eF!_EONM3!roYBL>(RXbMp%ee8ct(u=AnQf1&^Ee_?S^ zC@!E~E4b2LX5Y*w}-`59IZn4>X zcJwjiVnq#@C|!QsF15-!eYqQkIzno0%*%yl(c@%r`gzKnSP9t>L(eYBD0k<-0Gi%I z_=?lgmaS?I_Tw#15YmwEa@BRM_5?WojPi>W4W8*fi8^_UBEpAQUp`K(UmcKVLlAdL zZESRVpfmZQWPf=P1AT0E7n>`r!(wHjrx^awsL`yqV}hTE!^7NX&kZeUWW??#o@_^m z2)l8`ms3AmUa}h`sa{A0S;v%*jY%zU*R`R)o_N(}3{Knou986aHAicMovd6t$}V6N z<06Jz# zkD8w@GxW;+=4NMm_UzPExW(F`Om2@w)4PiDsRyxZG`Xq`HXJuZ38>y=CGidTNpnMX zWxcP$0Ji@pOjA1M(F{J6{uuJQg6H4*Le?pYU9!p2(WWi@W6s0X?01X?OPTgaB`U)hFZN|@c3WvUML9cOK zC_nea_d{q?V5kb66xZm0#`w5N4?ry_eZ}WaU^rnm;<%W$v+z(GGf4*%^v$kd8KUm; zNk*uY*$QvOsbaOB-4PDE<0qDHh8px89ZdG7#N%{;_^h8N5$FzBvaw6O3Hx!HFn2`Uu36GCY z3_PR1Dt^FQ?e4{mHL{{<|7ap7KLs+FUKIrRRRq_Mx+$a_huy?<8z{gUbjc)MJ=naR z?+O{gRJEpA&EF8g+=r(a>rijL5r6uu*0e1vX!7-sALWN(7N{CT5K7MXq0}w-y(nmF z`ohz*@Oye3C?8Q=qT#daM;SxEh!7``MX!KB1Hby;p%D79ejMJpH6)x(zE2z^c#ZPZ z;vPEodQ)fy-RO;{PmR*vP7mq4K1Apo}(1U5Q zIV^skk z93FbWm?SEyG)hvI426N`cuKVoJdhoQV1Ju!QrCx!!;P#a?GrZI%@^9JBbUT!7@Fk_ z?wVWiFGGQwy`4)Gflr@ZvVXA7CN47*68z#OGxOH9YL`Dpx)y90$ObS!_c<8 z+Ov9mXGZL)h>{C)PD)udGoT-jbz+t_$*pi$J3P>-n{Qi22+5 zM0sON%!kr#`jGTE%_+GuSI2y3MI23LT}6>p8iGN6Sg;oAwxpoCVKO~584akKGS4~>+!D-*#Sm)PD%zMg}+N^r>EL*?DGbRBaw4h zzu3vcdd`t@T*ii8Eg|F}e}IW!N1`u5k-Ey{Pk*m0Eij*$SEM7dtcAcVMqqe~&CDhz z^!9HU0us5l*X^7+-e?%eg70D74dHezoTJD=&!!+&_#e4geVCnDP`5ZY(^f`NjGBm zZly(xqpraqU{|XM>XXz4^v7A6R;f&U(N*qGXv%^{?pG{`Jw!&TgxH$_oH}zbaW}wI zQZrsl`cHcIyp?FT?95DQk~p%YcTAZ3eShGd&TVR{E3d44hN>|%x_1^&Lg#buOiLo! zkB**nU1j7@WL`8JO_)+{1LR@cQ1tLTtP~WiG-P^SXzXT1!zFV>Y*)b!B4l%LpssFC z++^`wFvIuYunxKHHw4y3n<9%YF!}JY!A{?3vf?J$fU3Q*b5vgh5wVMl3$@p~z<=t) zz4r2wf8!N-`C;ZD5*3P3thgAdmlw;x#8*0i+)t1C zuw8a(janQqUB3t+Rh)5;oK!>Uu7AhJsv)jhLx9fGIc?67MoD6Vu<$;DeY9Bkls7P{ z?)kV-*e0fizzcHJUo7X#w}jRblU0?r8fK>UDpbVMBIhOL_s?Mm+*G{p9)z~~Mas4| z``*59a(N$xXHQD+M)AAK(7jF7YBO_WgyTiEd5^_}Od_F+gt42{N7sgkCx24NG#q&# z*JR)=kcl=MwEaD)S=XgKlZk224m_iNXEkgld{?&y9v_gGagZqcQpnh;6`YuSMl(=W zI_tyi^DOCYhMOXQ>IKXzQE!v~8GlIcG@9`P>>>ew4trovt{!SUX3AdbWg=ei1YHWO z845MTu1qN3qx44+UA4^5&wq1y3wi$#;&H{b-*0=o{dm^iHvhv8VRLJ1$;ZdX@pl9r z-8!&83Cs1nC;B5N0n{}U=ZAf$(9oEEbI%DlAqM)$&!O8^swtf9yI?1lr~UQ$?DwBE zJ}rX0slBBLHUlHD{ZvgYE*{|Ed1rTLbIR6G!i1oXx^A4*>~7HIjDJ@3iL;3UUnLU*5?2n{6QFh$D;{aTst>BVw-_O#C|R!l4|4)ay@&6}g>OXwun zGi}bUMraTkrE#&_^OEqG<={5&`WpfQq3g$lHQ<_@lWiJ44j&|37pxk-(e|)=qyY9U zr&s($!NS}6pu@|1EPv6>+n!l8s(a;axw0LVOq-p6qJq*Esp+W)ttC#SYNe^^ZtUdL zMRvDJBH}vBT5xplfBWD!zjKk(T!su5XFKDg)A~?-qT3Je;4o9AGE|FyHCIsXREa+Cd?f|7VBAIaWq-)PnLEr<@jOiz3-+oAy$vUdJWy)qoorXZfS`fjfsx_+y?PMs6cC8X8oU&lV>Lq z>EB-8+>lkM}3k&`-_Gsm)I>O&_6{Yk_R8kpx$|4rg~{U${9pg zO9jj|)YYjaLp}Jn`_+>2iDC zzTUI_sgNV~dE+k7dLB*Yjqlyb@9Mdi=2!Kj+^U>Ak`AfrUt;2=Vq<=xj> z%9C-$rimz9r9-o~^fMe^>hbg9cCE7nQLON0D}SD!v@8L!6rT*1aG6yUEocCBvv%p^ zv8rSDZ$ZNEzY}LBo|w0=EgPN~61SYTN4667&wpA+s9#|iZ2LpUCr&(aU1My-<9F|O z5M%$y!9@_wNWRIWh*W(;jeVHJg(V60b(ZE4*~$=`wzh>DJfsEtr*0uxlO2De!IG< zL({7iY0l;~Mx}+sy9dklyO;p>MMCY+_5_Lf#?TU0dLlB)b*dks7zcpKtH}LT!5V&< zDn(Jf)@qd8&?#zfR5}np7d$nOkK)}-Tz|rqY=OtjBo89z&0CYOy67t-2;yMRKmSF7YP3bosUxAuVj;6hyc+OwL0wbd@1yhu*%941GP)+x(rSZ9 zn^2vzQ)mKmGf(d;^zzEf28h@Tlsf~mC&bdr$gFQ??YIpYGBv+-F&8xxG_z(!-)*TE7a{D^~ zB$UTc7^aE#ll_mhii!#=dzF2`w4rO6gBEdjDnG9D6OnRmxN~J_fPdZhuDB0(c*#)Z zx*Jvg;-e9r!Bum}2+=DNV*{rGWf{fTpl9dQT;eVgc-}E3osFuiRy-4E!f~ulKd5G` zp_I|WzVTZ5Z&%!)=bE%N`Nc73?MAR=jw9k(qCcGqZc4Y&ZP%(c-o44lnM-NOoev2i z5GG)EDg2-8Cf@EVD}O7dX&OeVMywT{52V~0Fg3m-T;w%_d&%OyT-S0Xlv0N_5)(-z z=T<{-Q;dhHvD;Nwd3BF(Rj`RVS8IG_acS>o;nEuP9XsKivAk}11r%HxXBitlvVV0Z zTj9oIhML#9ilGY0lfOVc%Q(pmmJHTDc>g@lwIRNmW760@Ry`joQ=iY{vj!sQyWm?NJ@ zv3w_IcNdXL>+<_vogk0?kRaR@x3qf&bVuAL@BsT~*$4iQx5(HbG;e9f_ve6bgyCoj z`@jfBUH8hL-hY=?P3_+${}C=y@cRjphul^?lNcMBG)E>2HKZ9%?n`0F~`I^9nw?Iy`ffKflF$(qOQ%ACHXm5oUIDb=qO`#I@xW`vZjz)x>BguL0lG&gN#bhfyU2u8g=E;*1-j;nVHB)4r4=X{M zZ^Ol6{x0+D1ESudg~c9EH(5fL+?$fV;R<}#9SO~>j0~;^ z-&k!?ZvC@6hiWIY_v23XJ>BW}hkt`%;aWl%n_A2%=gB_IU7h9`Q%VON*qCO$W#5P$ zw=6xi8k5}wm`yh6^%lBhS1=mS6Ap7&b$=V*=p3oO_$BK&1yrTS+Vy^}))i7yJb(g4 z!B}I@%k%LiG%@1U1|XnXxR6`CrDoG@v@4s<4|bT4RMlg)o@vKG75o(ycgr_CASnPTUP(9YaCd2F@I1+ zbfBg^vL9n^L1_+J-qtLKVXFz8>(1dSeJdYg0SMS zvJIUYm)4efj{v3-xiiJu>-(iatBH`L94)dspTo>w6M`H@5M1@o21?M;!Vhy6NeUX#G_MURUoWAxczv2N7-uob?47Z-;f~mPn9>AmW!78gSL$%EP`>e!WO|LiG`Ckd z4upjNq-3N{G}(I`F9t=-it}RfFk9-(i5OzY-#Fu&yo+$4lqT-<0KTgqX2}mED0mHL zX?_Ysp_diX+Z!S2q|YW0?te5%bIs!?gYU{754SY|bfpy!B7pX8iBXty3qSybWTFPU zG1wp5+S?M7tj{G>xOcgZd)@x?v$#_XZ{a zdLaEt0Co&!;|Gs~FLK?m>g_yCgYQ=+CJwMu#=X6H;xqlJ{)s$yanYZ>p6Pk`mr>5Z zVClzFkn1a45r0(Z1zx3-ZOX&DXrYv%JhpF)8*AQjGQn4)>7>337D7LMS&oDPd)5EM zn)5mnb9xPec?p`I|76$FGR53kOQ|8A*VciEu6OzNS!6t?`C}ZBxp!Yd6v?8 zg7Mb}^IY4jG6UQEj&Js`-d;!-{{NpWk8xY1$?cA#drv zp>M+)fw0pB_e;Te9t7DWC{ZaxgmQdId8*S1UUloFx2~Hh(wt%1@RN49&_wprB-8o-6*MCF$(vUy>WXF1u6RLtvLq~^?-@&I8 z5?Qm9$1&ewW-Uf6T$rgb#`_-W?CXA77JdTQwZ2s(lZFYRBt>sI^%P$dUk-TG1X2*p zQ7R8Bemph-o38^ZAo?HU-t8>n;-_^tM=Ype5R%s#$bMS~2eL5=L6bo8KdeNW;DVfw z3V*1&kV%fgoI|f)36kFW|Nb4OBrh$1y{4lxQ&QpA4k32Mjx&r7>>2mry$-{dWR{n~ zs*?^?h7$$2fe*#**fG@Syvb^6WW<&V8*oesFuVyXdhb8WM*5=up_8{ih4sUL*65L}N0sdVZv)%j`f`1l= zVKX5FIWQ?{^u1NfITWaX*6JpT$e7s9(gW$gB4v5X%M1U5yP4}^Te#D6_Ut8eyCsHP z%%@*1;HQ1<1XnPG-V_$P0=2ZN$_N6irFY>EmO4D3eH3fMO!bu?T9ICfa0gnjuOka8 ztAc=n;&qCMr=8cME7%ULdgD%jihsr|VdE^3K~1wZ1OjHbUtfPq*P1>5{!Qq%m4};K z^Sg@sb~*?+)hDfAPzwuF-S_{-;c9kosv420sg+1!)OwHKgQl>%w9||YsO#?Oyd*#1 zcMC8LJ{<9k%q^(CCPOP?&Ux*sAJQi^_aili+~0rFZTjG<0vU?`kr#A}mw#FcJ>f%B z6OU?&jlX{T^9%j<&d$fb*EY@lV`+Ridl@_j;XEUHw6+co9rMST!~YfFDR3)#E-ydd zb!egeAj9Tw1HNSI2dY1;kjuShS+B{p!*(udc4gk8_N3Ep(fpsIV{%+2}taDmC=P zpL;L3jWWvh`n`D4fQKCulac}t_vF_0nFT_@Jkp8BZO6YLj@)Z;TxKg$y^bm+0_i57 z)o4zr%m=lA=dg3FC}O@tYO${J9mDez7sJvNO8I}_!$r_+&{93HD}Q@tq3ytrVQMa7 ztVc}lCc56tz?lZmMV`T-?9yN&ZuXJvvA9YN3v}{luCBJ*eS*KJ9`zu`k`zu#OY=ho zQG+1swRg)}U&_h|;xaP}S%tqJDSvyFiCbI4t>=WIq^Gy!`Lrqwd)nGw{(q>ipWqQXvqG7{(PZZC%HK7x3qQO-1I+pl&B|jj^Rb``WW>Z2 zzO6X_+iEESO4}E87J&R2`OZH@sQ7?sUqphCx*!SK3$9Z?9y1j}3rYB&X(8V@(-)_l z-F3QTzYC5SKpND|r9dZGqrpvmsq~kG-+sQH8Y2w?LNdT2^nWc=921e65cTpby~Z(f zX&kAv5UfxHy)U;FtkQ%&?`j;j8;$mzo}D?}KHM@P>|cPjwxgyx)X+yxCszchZ)e3hge7k^vaP#oVei~-4Qk^tf__c+(r zq`p@wSC;Z0r!KraRNvREFPd(&fHRV!`zq+pu~^9+3@#|s(5Il+)z!WFTq5;I=zk|vwJkq(W%?unm9kXIPPYtekyG! zF1A`p-G61n&J>(@ZRscX^^r_jm4U2hL*7O60qcQdyv9}{(TJ2}d2UwuJ*KY@Wjc0f z475NY`M?e{r)b~SLYQI8p$`UyIQR6s{^Brn3USRwo2fT#@u5!tFHxT*iPooj6eD2& z&}XNM%S`_4{jc2#stdH5;OUo_pxEYSBkyySD}VR;sgAc?hbGpWy>Px&O}k}Zqd7s> z%=t4No@NkCo6yjEk+gKm__<*LjB{W`&nUe@viLG5L`qzvyn7~A9lJcdoEb?3~M#V^FLON{m*<+ z6cj>2U_&```eN#yOCXeHR%zeIV4@Lo5NXFR`l5)gzGc5$w@<{6U%+zefVKGM*4CVd zXNP4|@$`Z`64IRLE&LL6vqWN66kX`5kK zUO_roJKqEzc`Pp}>dX z{vr>c!Ny9+<_T!c7iM={PV&Cm?Hgzu0B}W*!2@po*B(V1gpZLU_$(yOZ!d`{F@JDs zHQ>I!yqZ1o8&_wLy7@1#;H`R@Df~N5Zqg}f&{|}$JghpdtYSbBSSma|<~<)}`_H;H zJ*7|6nI?j^o~W^?9_VarN_8njW_@)a0DezPLVGGc4UhpABzc}XRz7Ri?Wqbj&B$B` zaHVo%C{NV4U1Dqi z0O;K0^xVu%-7JL7Tr9u|z|G0c%f`jY#>Jz}#V5=uB+SFj!pSMj$@zk{YVw~lI5?Tx zSbG2O8Ddo|m%$kr|2>17je~`&o2i5I|DI?5y#fq44-O=cv$>`4I|~%TpNpH1my(l-A7jRJ= ztZgkET`e5{x9h-{Sz7*o+<)hvm7)Ci^#792XLVHtG-P6A@FAL#qO2wWKtY;d8xbD- zzwu=>44hD!DQdn203Uh)0EGg;U+^i=KDgz;{c~sn0KzE%K;WFwtRV{i0{-J01zF$~ z@{`k6oCrRHG1h-M`qDV*L-2d$-Nxh83SL2#&Utes^3bWiLo0~Q1w}LL1{xI? zHlFf5`=SdIS{!d}{lUXCm(`?_a8sR<{&}l$b!&pku8Sr2)gPX*hm1X{Bu!^QQuMuL zs-k?7&JAb5jPXbmcYmk^`6wXjixPcv;uo?u8dCI#E4v_A9A+vTS{5ND@0ZVMI5Zsl zu2Ng`TxNXDvSi2TBI4Eet%Ex*d^g7vS4(F7u_HV389f`$)L)Hz-cCjuRnL6~3%&Vr z;Mw!1Zbm33CZ@{WU&jqOAGP3=mEXiAC0&oFT%E z5>iw@HMZyG=2EOEIgqs_f-iIxpHSA32A8IIsxmWPtfzXSUgTK3cxO2=goUGndm~t! zz;ALwWB^hEl7C86Pfg9qZj`?t#%cDlNV8Esf1@TvM*vr~R~ajrf{nZbs&gw=99<#P z?8BOp|Jmt&<~F$bu8=Sw6;{{Ma$U>bds?Vg8NbYjWAfKhZ!aP(mC=l!6o8|nNS6Fw zB6Blg3xL=)A{cf>v3w7&O9d2(Cp@oXTm(r+AA%z`z<;`<+4JzMnA+XCYz(d}_8m=O z^|XeKkVz1`27+N8`mHdNO2VYT?-O^csIktlXLj22X@6I6XBLxr5juV!c7Lv+>Um;U zcdtbTcM;Ht)+kX$=qu;<`t|Q?d%kB9xOYYgL{S??EJ+xnu%Fl|3jj7i$-hueQ|jaB zb3VGlfvZT=F${lDx~tzFEsco<9*%2g2>Ubt`nPQQ?y5dVb-$IdW)#ZBv}u3#vlXCV zPQn$SE6E*VGPQ0x3yO+fL|qWoESDc!5Qs#rq>%>Nx%ZlmayEMG^;!Q`yHEt(5~p%{%^ z^sJN2v(P&*eV5W>NQ6lSSJ1rLh9_xJZ@PYjnh+=}q=>_`i(YLO{=T;#v6|INS4F#>163MeyftVzeN6Se)qZvyNn05N;!=e!RWuV$ z%a?zgC=7T#jyv(l{No@(#ej#k8dLZ zQUG($>!2O*NK)MR6(_x^^O<92W@aQcUJQTcg&U6fhsd$lr@^oHKErc2iB*jcuVSuu zOASz%P}?ZM{L_T>0XsvPo_>!fuRlG{s)!BW)$>f4D0a_QUZwfmO!j@=q#{He96ze7 z(YG_)a-x!QMGT0UCX_9lbU|VNfVjDO4(WTwIC++r?Vk0 z-nxk*1N95`s@r>kQLU{^n7?Ie7BU6~iOy~tEqX>uEq%#ULAO(NdS8gC< zN&YaZ9{I750WQ7zhbKB0mnx63@#SR~8e4m}=e&vDS(yD*EQIfIhF2Utey*21m{c9Rv-$&)fe%i@4|q zV8S|7F$y~XvtRRg3@~eV`RjidkUPni;DPcRSlJ8ApFd%N5~w&>Kp{5k)a9wM?Q5%v zSw4$l{%11SbrV_)W-47P_p>Trnevmx=gZ+KzMcq--{%`LP=N2$<#tZ2ig6xfqsGWFOisow6YkI#_~*o(Cco3siB1TP5ar-B#vSrn7EG% zmKtvC&?dOBCxq1xKPi8S&Y-2n@oP=Be1P}X@vuW@?-A7<_Vy=u`Y8gJWxFdO&&^ec zJ>50!fxpWTIK%qjWx7s{1KJLfcs$-LN=sYJR?Dh=8|85yN_3LJt-N$8;Vftxk(uai zk+kdSW+2RB%C`)3M%4xVZoUEPp*RPraDMwi(~d2G`(v5Y!3ckdLl~?Xo^Nd@061v` zN4(UT38kxM!J}*`@M>i4F5rH{S9?#$wJ!$T^Ww#9AMUyNomPJy&Hsp<>i!!6OPLra z2OtkIMk@|MB`Bc+lEWq6%(0=^`nA=Dt&8I<%j)9&`Q3^Z~acd`M7_~9VPq!L$+7I#1(bj4o-23 zL(w9|-Ca9Hi(7GbcbDQ6cXxMpch|uwUfkW@@cr`g@)DA{gdsWiY+Gxuy&0UvqKG*8 zF#O)d5~L6HYs5!QS!c1QEeOR=%^KK~YuYpggo9$LN5)=*W?H?*9Z?1e=t zIWoEA9X4Mc9Xy?H&eq#dakQ6E*V~S#JK83#n%jR|KwCvwyHlabf5(Ze!s{l;>aYnw0ei4PA#}P&23-*Y?1e}fwzLkOy+b=7WnTI zDK~<_mtp2>fkk8BtI8KfGX3V1dAT#X_>q6#lEZ-$HW<)W9Qko3X)An$Ax?|sW&4cU z=^2d%L!J{ZoBaG1y@B)W9BJ3NU9VQkZzFMyesA+%1>b)y&@`T+=GpOqp+ht99P=0u z=J>rhBr$wp94Qc@JjyTe@sEzXn>OKs#Dj$i`6}+fE8L3R^%Y2VUm3ApoOD}YkxqY! z41l&|0$}zX46jL_$325;mvaGtg2^s8k6oNLkxi!8v=veFUv2dTlQA%GFd%Gw zUbrpO3JF7`B^h=7cD;6UQWu&iACrGLqks262%iPJ^~=-%_oD60mI$hTXCJGddAbfQ z)N=Ut%0$u(!sF57Xs1>Pi_Ew=MMATRt@}-cQu}&Wlx3sGsOgBL(WdFo5 z307ApfuH~37LT9b2VcUM5GAFzx!iuuNKYc+rG{yjY~@BzBvP_xmp?>Ej}fGm}h1b!q)_k6<(SHLrQ z$P;X5n7qqWm9ghJUqubWJD$u^!_To$LUeia{p3FTB(Uru(bl+>K*A(r$&z6PPjwhw ziooGZIqnJuAmxZ3d0t(NG~|Hz<|yKg5gH~A4++1>nBcNuikdIS{!f3&aP?nU7t)Uw zw{sE&(la)^0hR`7@OtY4QGTO5>+hPr2D`d@Ot$s};fFRALvAP3E)Z_kEvAUa+_)4EQKpwZtB)SiJfSns`{Cv{-`XxYO ze>s(OAQ91@8@4_|1B);=b5)_m?YAuw(uqWAuw&nSb11Z%TC;z!Drs&|nh{)l$Jwue z8&7PJkpN8W%ZB!EVUs_iBRtVp5<$HBN~*-0_t#VO3ul6F4~oA$?PywEKt0w?J6B^v zf5-878!eZEz%CABRowL`Vd)=*kvL1{^Qa#mA}vWFq;++0>>u6n^-+sB-|jO6E|4-t z)BsjPVuCzQY`cH=dO?C92G%+`N5HjInII=*hb)9DMOQDVK16|sI{AEy3l`7`xX_d= zN%^ffZ|a1!OMq(_po85gUr%Nc?2A3cJAdsLEU6SyKZIQG$l*?~?n{c%-RD|ma%^U! zeQukr(_()>=(53WFxz(i*7oUy>+QvXJz@|Epm4Hc-PC_}yXt4?$6_+cRc*8B$5qeT zI10-I5Qy-^D^O`j<}6@Swc`Gb00@jG#>x_xEQw5ayNrpKz6T5?5FNI!| zUEqk7?i%c@u&Nh{5#gG?|G9X2jLQUdMieu=VdeJ+VkDkV&p8iS8T0Qs?(nMl9CA!X zmgjsaLX3Y>0>HNHhgdZPxAxq1&-N0;;tXE>Mo?^Q1>;ouin+xN}G!yKo{&oAH`2I>GiZ}Rb*p0k)C(Dt?3P|Ku2O{AM=KzT?qgeAP>zHK=PxL zB|w$0`t-MFkAkF3{H&#~646knP!mI{8Qho-9nj>P?d<+84l42YxQapev-uvigX z*zx|hxR#3ltZZb|)(X`fPo0#B%bdIELnYU_yIx}eSNU)5AdwRmrTs$g{l&y@5ag}_ zyA*$M+19~o@}>ED5>Bf~j#t}`OGOZS#0$z-`jA;gN22eH-hu0s5)4oHX5x9|nb!bd zL;=QM^ORMGu)xq_|H+w&-9YXfIt=nBr{m79_c%Eko3*C%fBzy0?#-a}NF*Zu9k3#o zr|X*UkC$(lL)n4z@lKqaz(5Ks%6Ph4@MeDxS5s@GGlP zyaXLZUyC#w@;w>ApTv}^X5RBHrBHAG^f%n1A8CbY+S-ZUJR4kI^ zJYRcsngyc|ea0?NCE(DY9E}r=2 zsg~ly&-Ki6`azISM9X|1QHu-1QV(GHZw zf?dG%eqEvc{cRLXiWy*vx>Y1#aZGu& zViDia=+p4jJFd963mI?Q$49A(8&05KahZjw`$0z4M4hqLZBIecoktP>bFEz-#m?oB z6bPz2oY7dWbw71J3uI~_$0Z7E;b`{8#XXjZK6aB$Pz=5U9lEC?P zHO9EPu$Vcz{X2mBi)^*nf-RJvDA|b%w`MjcEMV1=qewvT-C%c5#_t=E|HQtS?4)b* zga>I@a7-L45tvx#y5hRIfvAyN@ZK8ggljC9Ri`>%t`6K@pYKb3PMCiU_Sr%CDwgZ+ zrYlY$`nTQ6w#qejPuJ0VFgYJb8<=?Kwa-5q?euzPL&3Gpfd$;1EQ993+2Cld=>E^w z@1+|#H&R5J`?g%9=<|Rtu@X0c+9#d)_rV>iKRT<^(|;#ta^?o*hko z*4F9+!vA{o=kqa9Nl||o>cjjj2U%QOa!?@!kz?Zcgr=vLt+Vd#X~6-|2KvwfA@l@w z!M=8Tver`QrdasPeH)A&*^tWFRyN-|k93l>-E$l+w@=b2Wj$wRyL_skf>FDCut=P= ziTK^uI{r**EM(xA-J%K#q)&5M) zRzXD)795>GQGL$NgwGh%6TN1Bd$K^eFz63cCP+>9i__}qW(IE4^ExW8hers9POZc7 zdjB<)v5Rjf>Ue+A%V%gX@|jJuk?%IY{g(;*yyR|f&qbWSFO(N0DfchGzC%;bAW0zp z%<=8hY*0{81g}jOtF^>kgfA_vwnjwk>}SX0MXQ<}+F&^+=x`%~g?zs-H%W++uXCE3 zuamqk2K(c*%1Nv#VqmAp0a=j3L+P=he7C!#T(Wc;9!r0N?9g5j2Dt9aOMIZ)Ag z-HikJwAgOdf4Xwy7GrTa+r_%x*Ydd@PZ^W_SX-SuLeF`pBQyN008Ns0zxzxT7tIG$ z$KidQNOpM3HE;fbKjQ*E&*nf04%L*GA%lW~g1>f-Q5Vd++DlYn9F^Qz>1(cv6FhSq zERpZ|OjLjGoLtf#_m?hdof>_Q2x6;-jpGh%zp<|=Bv@!D*smwc?<;RJ-&f&A2Gt)| zBRM<0aRCCaH+~SBO+=}-k9&xM>aC9Z{_1s=sl*#yH1^e))fam|Mjx|shIv3UR%bw# zPt^2zZ1D~ueV=GcgW6Ojs&0dXVRsOxI{rEcxAlLaXywW}rLOCJpA6t;D4KgbIh-iU zlGohO#4j$a=sYSVH5rT|j z_+v32aMgz})+p*U9G(BzT4`H+Z{5w`;G!}Sz#7fGPi2d^(@ zypDe>__LQVoEN{3A%Z=l^Vl6C#L0s~HCrEM7snP`uwc%*FTesi5YaCP@%Ac^c1a|r zpQbp9<{@8rO^asu8_F%$FL1ewAs5G{P>6kfwki15L9dCTGM?JNz0-E?Evsx^X4A>N zUQvN9ZgQB(KM4mdtcl+LRxjzc(&2%0cV2(x^GU&3SgJY&CIR1yJ+H&*ILGYw@8iOW z3|_Z^r5Y$z0F&iL`}?-X8D10QLE=mzznRalvh-HQyel!-(R&6RqIMOgMD3P8Abb(K6NsSulDwmv0R3y@(+R!7_&lATLg)?`l z;y$+vj4$(bMU9Vha>TfRZVs2T;4;kMrnsa;bT&B3R#&5A1N&&NqYO~VrMXv zXM@OPvKs>G760@oZvl8nJJ3!KM|Xc_8`>dDJ8X1b(QI zjc~N8XPy@UsEyh=@G-i}wN-!nDk?-Xn3&Q!n3kG6Q1-I$=ZEDa!qkvdC$SD{QK7ol zM|2tzh-!YT`t|Xh=5^(YP6YATu1SmfgSn zteP}Q4qWmBd62TAqN0EP_O{3*P8=%46+!EIE^=}F)}&g#(F`O`z|uB{*)3aF*X`bK z5Z|K+2~jqa0k~?&l37dvX^13@U$AVGQwwmrpSs?h7AhaFRi)d`J3r}kKJF9yKoMtz zJM71+D1mR5?=G2HG1FvkGy|1E=S}7w{4IMsuJ`0`Ta(GfA_9M_-+fYW_e5%9X4z;9 zGB~QHIO=nVNJ+(}O1sC#U}K}6R#B>ghY!(S5*cVcE6_?r733Qmk@StiMcMV%pw+zV z0nri7mcwOHDh9pHpNNj(^C3)N)$h(ACoKF8$-^Kg2$1}<>D5|q6hW~R`(xVfW)cPV zx{;i*esAu(Yb}3qs^4}t_V(H`oA*=oBIBrZYMH)R9RWNsDGE7&?)P+L$UK)nxM(vx zNOK3OQ&Zp=aBkVBSvFTG#5!KER~d!3gxi|7^6C?NrrttRdJ_I_+xCXZ^=Wk$e(8fn zbvUm5eUpe5rtnroUSK0hYvtnP#O|MsDBvBi%Fllo;`W-;wezRxJ|W&S?Pid` z#fQ)PWe@7LT(Qe=Bq+(c%j-nG3W6&rN?=9cU*qba_xxm|Bc}I`KAm+2sxCSrHT3V) z8Km05>@P$yjXDy0J<8ce^I~Jgz%f(1=q+@ggIMhiV&mc4s{>tgvmkpU?sVW^z2=>r zon!iE3b}vQBejI@4}DvctY)(4v1%%i37$Vpp+ZGcU@?(38^{r*Vpmr;;M?Vp<8RPU(XJA= zK6`-cz43t#jQ;hNv?W|gyW)=)C29DQ1%U_ZqicVSe5^5`pa)mSEuMn39|(^QPwF-f zgl(UxMFG-|kc)@;H&C8$H=KB+<>#Cfk{!?P;Wms6O%zV7ZrPHBem^kG+7%$6;ZR>ERuFQuBR8&SX;63`A|q*7ZHcl4_WH6@M8u5lHxP!Xr|E- zB&L58x&P=S2$U#0*;Wu*bkSKi9{sZ$Mo421GTZjN7|pW1Z;#eh*PU0F7|{nHIODNl zuoR8+p8HuOWy&!o)=-vpILPU}h#;wQVi&NlBA2A8v9JPMc2VS)#~L63Bjnrw%W|mC zwW<^C)=@jhV>>FiMnVWK!-C70r8r-GJr{q`OKw^SN4T$WPOc&%)I+O<7Fo2u)k*#n-~dzO((L}jF|6j<+yq|7__W^ z{TffTiAnoaPu7K~>5b)3ui9Y;uD-|?4S4|m94=iNAWBI}xx?WHe$7XOa9U$za6`Y; z+IE@nxH#k7HM4{^z*aj2$~%TdBej3v#p*@hwtCAWjZ(J3iypFP1^5eWTWB@ zQk0w+7aC19JAapg)+I?1Q^6St!_j1t;Dr{$oAT`ux5j_10P$No zT+J@s_2L0GW+VV5%Szv7#dw}?a77}ADIViZC6GEckFA|KIX!*6hRXo)ZY>jqNxmew zzihspt4&$2tN&)S-gUAEX+RRE2K^ z3+aajP9w|K#8x`MXO1m4hLi5(c#O>Cci(v4x3Ga#CLJOGdjuYtr7m-$0|a}s~IB7oVm=La|k=cZ> z`9wm{BJ@l)n$Wv70HB(YflfZR*pb@LiQ*gtq^ncWMv;ECH1=FEBFZRgii+3sBP(%& zQ{J}Dapf$Z62a*unc06K$YOW8<~WJ3c3WQF5mqT|7|3Jw1Se@mM!HK`@90Q~CVs#n z@5o?onO^|761$9^F?QG=7>q3<{G-V4)SB#A=T?Otd;bdSYqCG39dPC2fnLB@B&NW+gx10CMkZc;0 zxJXMh30^p1l-I)eFllpDy!?|%K=*zH%_k~>`qfh&=P$DPmzs23k1M;J=H}*SX)*jz z-1iSnUCzOTA3px!C<2{Qo;@EBT2I0Qo&2}Nw4F-fQS}JSZ}nhfm8ncS5Hiu)9=4D2 zhLqH6Bdad@?~i|O)27{!CMu(L`s-p|ls2#COd%aW++xv^r@`?ssf*iISC}zU+u?Pu(q@B2n|uz9>+#Tek-eiPC($%^D%@1 zuG`?BRx!fTvNF;$w3_oFuZQ;S>%yE@##gN0UPlU;WpLhKsJ3N*({ z2WGb}IPsx(yQfWj(ox=k(IWw~Q?X052nnw`QrxvDw=cSYi(SlA7@kQ>X5W_p1u6*ICUoP7Epk8F#M=ee)j4Mvaz?~pt5ei zJi;$nT;1fsBCM_@#Uvd#;a$^Y%$qOdFw~?j+j)QJ4}m9N+_v>6{Wtl&H7*rx?OUxU zP@+}s?5E+N2U0%QB?qv95(=+UzdjLmrTd8JsqBg&ELfN0TwH!8Lzp<^!-ieAi~GXo z9WLjz7kc$76;IYjH-~wsnf%A807177@tG2ksXjJjGQ!{7z`$h6io|{h-?<-YN5^S9 z&S!so9wBJ8?z_W*&#Ra3zo8T6Yypetn1>f=cG^@(&ZZG15EoZddu=5sZC@_qc!WxF1F{WExdN}+UZVf5Vu@c|M=ghDIe!dSbJ0TWzQa(g~%_0sMY=-JNw)5>-i=uf~DocoFVtyhj6n7IObs!}+|n>VT^?#K_l~=UF@2 zo0-*@@vjF*cawBQS^UpB@6QKBelAbIy7qyCh6to^mVlid*eC8`E-^nl@9rw5(3fZ?l*m@ZE7XC?`gm1%(LQ`P zJp|9%*zp>IPXAK3!7Yrmz*KA3BwLo*mu`UPzj~t|n6?-UBmGppEwg_Mpyt+=faeX% zsb|>YRJ&%Qcrt}n6#O2}1@A8hggY?8n%%uz9ZMXyfW`9-EuW(g{^oT(k#}Er>q-(P z%TKD78IO59XBxgu#PvmX>C~yXcq#1Y2!p5#BKGoZffl=_#;~Gl2kXaZ{!N7hHcJd_ zCz`%4V;)FqbJCCYfINRifnHW|x$1dv;1F6Y-$5bt8xIHyZLfz-VE?l0*7%qyi~D-6 ziLt|wf!98tV=Lx6CH8o#)4~f$V;6iZ6~DwXo`zC=$!_QKo7c0)`TFx{1fBu$WrL^Q z)KcMRreL8<&^?{@$muzK5{>PI1@MkrjHGFQq!obh$gpeJwMy^O+?s3 zK+cC6UPnTZRO8k?UwVKlMIpgcIK!-)fZe=!fi14(1g*4sEbp3l}*A2m(a zThFsrn(eYEsSb{Y)Cel*@?(*+Ri&lJ1r79pzeFG}TmFBPmhMa+)(dkgl|}lEiF*X+ zk+1~!aSMh94Iu-AG<6>ZJbOj(;z-DpAs-K?w5=XtfEVAel)ru^*5yr4z!P zpu9af(g_=OH!o^YD?RPmImzwX+PwX|^RVge=y$+R;%E%Ev7{=(hqO)dx##mN?CQ0p z*@E{8Il1#qE{jFw-#M0f)fa^Mg(xEG>M-CJ1#|~~GFaDciUr3;GLxR=0}A+_AC=MA z`zC*e;i54}ypMBcCWo6}Nb5~I5SOk<$EL!D@Ud><`+I?Xi4Ccgo@^J z+uO->4KgN;e@bc*COC%mZ8>_cC@I$R!}~9eBD^p_(q6RvuN30T*pkIk#pz2f@3;@G zsBF^XzaZ81I9RIFcybop-HTd|3=@A{I_|MMsNHx{){>C$oec(NE-Su>8H!Q02qbzH z6b_Nx=1BT3;I-mpq@eJHg=LUtg||145PM7*OcgncukVg>wZKmW7mtD_WN7M5wXX3G9K`Z3YnRz?OVY*1kzO!zVq>-=v-x+^(B9m%evEE zC8IbkNx82$Hp1?t0cc=%LC^Hso~{h z->=4sau)95cP{sZ^uk9+pOW#|6 zoVV=$U~8c0<#i#)ENjAwLi|=RbC|zMK#*pG;x3F5d|gJd610CRn)f=+st5xlvX_q@ zBj`b@pV$u^A@4PI$j#P824pw8VgN{(nco&|Iv|b3~Q9*RKyMd;VolV4MQs7GB*gbCkRbJ1{wV_y>lZ~MgzrnDx z%hm zm&zZQ863j!>6l?g%;Ww_ymN$5HsE^Io zy)xShhk7a;wNZ1A;rP> zk%F2Jva)|3br%0vKKTB2!WO<-hBgl_8j`~uP!5Aj%<>lp&I3`}n>(%Op2IwD% z(LzSPVFm2Re8p%Ep4hS#q09y!PHgi%_l=S0@o;z@=cXmwPJd8L=4e{!-)E=d%F4k` z;`_)q=dq@`&kS9^X_5JKzcD@TDArk>4?jD+{_W8!G;8SWyH>G4{?91!4ID1g$hZF;L(K-Y z1y4f(R4UgV|JIvTb+tO5=3P_DbsCahr-qX#kJq?cbz#m%eg(Bf(zzwdqtzSm98H0Z z+J)kYf3|B8w`t3halhgAoMFdWYXaTC($;_XCwBI8e0<_xIncrT!Or%RU|{Zyu{mnm zug(Z$>kJ z|EI??Sw`rdsLhc)YhFvl#~Z@MAKbT9kEqA=}h+c-A;kf^B5xOaasZ;`HZ~T!QA67iUwozoh?xOL;?`(w#qKKwVEY<^!1N&i**;!XeV@$Wb^9$b z=LF%odt^MwM@;Y50U^}oetTAC-DY)?^wWoo>8B)nV6+xQtUA5_ZlpsVIHYp@gFoWE zOe%>XX>V<(YdB)>JEFoEsAW{kqq#&u-vDn==FfUh*SQi+zXhj*((L`H6H6VW^<%8H#d*Y4rD ztVPA9bI1SFx>Z@QO_tzgq@>*Eku)PCFojBZyjt?cbUfKy5`Z7Uf**fbsGGWK6Mh@t za+Cm4q5~;s>ZV{DrsQofNFhnf3F*&aA03)6nATnykKUg{@i5poNYN-3zL&+!s)l)J zqNrs{%?l?L2XsuL0Dy;S+jp&s!G$8Y6n{~60;EWO(^y9LVg@t^{}(c5oA2mJcJG)&(At&8XB!Hx$d_D96k z*bp5geI7^bk6Myt?87zsJo*FczfOURlp^52nWTMdU>2RIq!54i@T4O?;y%}Fmdg-G zNa309`j+>x$MT1sz}5%MmK1g zf~jzQa=i5zOMGn_BYsb|`;}QaZR&wku9w-qQvJeLobPrT_(0Di46FncMi$i%Kf@Zr z8)?m$vmUGkq2+(7;xh?KOdq%jv4lRi!$gaZ6(jqq>t%m+I(xFlAPut!ZM@Y$D9s=W zTD5Mvzr$ID1(0gl4%Pquk5EXW5;pvhQJ=o9_r0t0b?y3Ng{fq&s1pA6-#4T}7dt(d z5U~F(8C$JnJ**EREt*$G+{&j$UJYO?X?SyE`ir9FrI;PDw1ug!2$L7P8m$Z%)iN>R8R=+6*u^=_XgyF9_>g*d$(38}&F#8$*N&zm|b zu)3c~^GEjVttbLLhxAX)a|j4DL?|K2&)fd;Q8VtJDH-K_?jR%oTpiH-l{65yy%X;0 zFPC0F=<@DON9ls2-^$HgISfL290~_hLO|odIQ6w;E!+P3aRYbl^Y@@_&yX%kQz{T{4hHYXNG zDx^wjX5#m)WU2(UwQ4B*j<2E~s_yB_|MNYVP^8sx4xH}5Nw4&eN(0L%wU?rXS=#_1 zS*$zN>--|@Xscy-LC)X3B*Ce+_qR`-BIJLP(m#`6j|3ldj~75XK8LE7rqAxcjPN`X zyuI7E2u^Z?^(?g7I#w6pQ$Y0E7ZLk9_z~s76PXQ!&m#87@z_qI5B%X2N6Wu+`wLw$ z_r!2@qFyai|4RMnDLf$QzknGb=O&LcjFBeGYNKSog&!e@)sXn=eKsnGeNrOYDW8AR zexHV>pUu{Oddajl@?)f}pD@-$%c*?y>kGJ?0Dw6d%eu53wmNEenoN&ZCxj44`Xu(7 zu6dqkDXbqsrQ9SG`ajZ01Mw>Qlij!oog+~3P)>i;V zn|zB^&^<!1;E(g}UvKKLWg#!2|f z7?p|M1sGM2J(fv;!>En(d^0z94@*N?o~_($xKMgtxZ!q?U~=LNvIJ&C6H$NpH7#mB z5`?E5{FEdPG-UIqf16H010;VMVFy&B$Cjx|0;XWp#;5Veci89V8T5Hp=p6SL+u1iz*i&&EdOkW;U_&F&c0K^KQnPy#a&stks zRcSbsh1N*%_9E=?z`@9uanFBND4T*H6lPEaj?2MwmY-L8p_>UwVC*t~s94!nSJk49 zR^LH_Cr&`#2Na)CAC(M!e0D&d670HrOHVKJ!Drsy-s*90t)-&hBwwt0-#Rxk;>ECO z*(+7Nz<;W&T_azVD6pFhP~uub8nHcI{`utai8UXw8~!jR8L_8nY2j2U^Mu2jn+Xsq)SW z*WH~$-V*l9(*6NPI`6HmZBUrO0_=830vR}vTffUa<+irszk9!L${-^nyNt{w&#*!b ztL}bd7Ga4cd!EQ=EB${jK>!o$RS)voA4fQ5oe^MPzWTD^p?Q7QVv;UZbeur+O@JO= zLmo3D4iL|>W2J9J*yr68n#)R{pf=!GSuDk4G0r1~)Fa@oAXyQJ_E;vvybcSvvMH}C zpFl=%<^_l~R-kmL!>$_dGn)vva)^d4C`if(@WhDPR7sAf8a98v3L;9SY5WnNuX&y?QtmM`ug6L}hNl5r5J z^A%{rCUX;L?=4-AD<%Tou9A*7e0NE)0cbD%*&WmX-TZpJq0xj@_B@H(ce(0}uyNcd zzxk0`|LA0wghzki()oe;aDQYNboE>>-(oc5XJgNsao(`x8KinbW4gL&*hd0HGfbFg z{)nML*uz#wT1t_A{Ll-uK@EI4IDKK_IZQ%w6|2??NldT71&w}qlt1G$HVwDx z?9AxMu*i}i%{wE}^FODU(5r(+8U_EgKSZ+_h3QdL@wb`@$0#@a5mt>gJ3D{8f^k1r z*LtCpsg8g0Q1C9ejs1~QTFJ+!%H=JA*OmT2sCC5Y>xRoVpUjQGK)Q}NdEIrl`*J;t z-{)_zmDKUd{5Rq0u=j@{IK|YolH(DoA|oM|CCGefe>Z?WJgL-cqd0zjod7d)WqJNC z?P)S|K&Md126i0Ao1o;O0iaQc#$x}J{~O%CEt!ARaL_7_WJX2UK8K~N)lk2{!FN`o zl}V29@^xcLGtKShHwg**m5?cb;Q9~owsnCUa9rB#38aUEZ#Ei>N)nvw!7-hEu_rR_ zVXXZMrb9R?iQXRSpa*M{8HFrn@5{R8nzQm_q6DfQUfz@~RAem8Esr=}wzu#Bw(M>Av@R;}+f= z9o|wMo_a>2EcW6#NBQ7tnYs@ z9xFaS=q!If?>I_Uu;Z|P!)?*6pjkAX{0xms`K_-fhK}@qiw`!Pk3$wv;~Idwf8z)@cVvO{@{)V8CgGhgSHn#NoM&_6nfzzo-KU{& zQt0`RpNOLHyz4jYhx>;cDU?()y^McBl}{PcPyf8V?$?*AXlSPdFoa<7WL^rLPRYO3 z_1JL( zS#wi`e#uqqAC1tWfL|d3*l&7!^cCE%u~u?ZQa)k;y3j;2;Rn@Nc4)hU@5_G+Z&d=N z-)byWUkR9*cP%Os`PR!+$A>px3K10)cVbfUqadJKvpw#YXd5)H*nFWr+C7Ueh!rWOE;o=(h7^_N$O221p11ys$1xV$&m1+6ydnYbh(|5kH+TX z0ch=``TdrSO?Fvx<_RQBX#0Pg@V%|&QA}TbS@yKqE+8vF>GLU+GU=Ez4@`xxh1*X=pGOqNb+V<*!gAC53pvhax#s|4#pn6>;jbZAg z34rcvU^^7nMDtn?byN^0w3SdxqQY^9*RwjS?=8Q3{|!Eq5JnN(VkZztf(w2G<$aHU zP*`jKN_0M%P+tif_>X@bhPhnx=QzO#>G|lXBf_oxH>N-cxNyOZUYFUgCTzIxaDuQ*Zo?79J< zCxmYWE~5CYnPWj)a{AZ1fMS>nF5jtuNI^vZMQj2B7e?RBf_&5H`23B3O^YVbV|rpd zW^jH!7n=Xxj7nj;fteQ64+u>o#P~ZYC$%Q~her}I>)|$gUYDIU+$*A^=@+4c&=4k6 z&uufUJb2cP+I(45*Fd22x})NX1%yftTs3mHdxyZ?t$bd+798rC{TM|O`Aemb&&26T zQ+bPnpeHf_0nx^5+TvM%1=u>vH^3H?phO@FPYqFxD1-_mO@N}y|J;q?+))7I}(&0~Crt4%B#lJ3;P7s8>cv$Y2i0-nrIBv|BNMdO8LxNY)L?xyINJ z2+yFMo`K%K?FN9B=Yvk02u;TQ=wJZ=b#7IBQ&7ClUpk$ycF{L;3ywU{0SLMsDg@8O z#WxpWxpP4342T~R#q$xzqyq;Q3v4?H2_{*gp*=?K9X3V<1NLat)Fab}t-?ifxuudc zP9<|r*G$_uztG@+0C+J7DGxu6&AEoaQQo{+5$x1t^5DkX=#b z&D;9QAlc-_1vK!(jQ?S))${%P!_I$v<6j;~Ac&+rItbQ1e7PNmBMpNx4jz1be}8s> z@AkHgB@t-p6NUmiW4rC~6yVB|!=z+{e4h!*=8FK|8jzgA$LhQPBlZ0qCXlT%kjrqFIwK*XVVqvzm zc4&Kz%%+OByOr$6Qe&S4wZ^hR*{}$t{RYO@{4`h8de$Ma=+%}Vm)!HWn$!g?J>IAW zX>dIj!ye6lZ)qf^UT33bTCD0z)Nz<4ZtD<>Sd|=ZMNY^R{e~!Zt#EcdIAM&hwc&e(074&)T_I0T%ci2NOawp^ zpjo^MBvJ+HHRp#2ApN18kzS@L`fVy)R5~o{H(`@ z0tdj2?8^aJfFFY=_LWxK<>dpp5W_slbI!hth>Wv;`cz@$liLiput2p|ik?xK_Utu( z2`H<2oQ2d#H(A+yzWKlJds8>m{hriEiBOq=H%gSO)N(zXY zACVXuMhpagGSI8}A_o2~%B`$SPPaF(X66l%PgmQujURe(q+rp_NZFk^Wp;#Kh^J+M zMY1kntv`opqVJQS-=F6xz@H$8fN`jECAm;<5otZD6|nK|yQ-c*1xecfl)z_y9>8<+ z{qwR^L!`j|Q&?& ziJ*QGRw8ih?P1|^NTc}83!^bW017D&7tOtG z(IPm4G0@e5>({91(L{Fb1C8;2{n)y;$V!7eK5Wftvyt_1HOBe|&xjRDVj3*Lw9~u&`)VX(7=H9yn$o z9e--x7j<^J8AwZypI+UUmSPMKNAsBm=b@kk$9XyHZ*~Tt1xhI)H*?B=nmByzO>z}T z--60IMws21RKT_Hc^(oa`a*(mz`#0b3?K=lQ@~6LaZG~^;#v5#19|<2?9Y?I1o2Z)er99n60q~nLmrL zq)!^(-^|Yo7nhocT7_?a6OmIoi*|XBoIKxK_nn0gN`wtclzuW8GUV!m698dF3?X;N zQ{%Zpyb)8o4F^Dp9j=`k9)B`_oncvf>U#HD=o*Y9I7;R`fAW|&)-4#h0vIYvMn>yz z#QDp+|L>IQ$IeUdF%|cVLisP$)kK!AMos*Gh#b;7XP#)?LuGuGd#-?n{ZMipHVhOd0+7FFLK=}O{_TulFcaG6GrDxnq zxeXbz(`Q0M>w%L=0>Y6FIgx`W#(s=MAbOnG*l{oW*8 z661TJhf3l_;l0g&xCK@Y?)GFnjhpGrX(|ABmv3eCQEVn<;PcOg+QxsRhwL7o{GC5v z@9I>P*tK{L6o8|lnU`8hQsr?yX;1r-F3qzDeV|fScu2kC|YRK_wspD(U{nO1au{sjH@dxIYZ+@?hh#@ zCrbT_)M$g&IFAfRIQBi||FrkjUs1o?zeATGE!{OV2t$h0P}0&NsdRUWG)N3cNk~e! zba!_%v~&m}Ep=!3e9yV(-n;I9aGo_k&RS3H_kMQ0UK_zfV%-!VG;*8NYOT9zr)8Cv zc79$y4zs6!M@9WxoLm(m^JhZPbCXra<$jh+QzeW8k}r#J3nL4n6Fjn?h-m;#R8 z0HIy3xXXqMm)+EdT$nvNXIY|=ijf}}vymHH_q=H%ba=>*zoUdH4*+EU*#?lFv|+7jz70$)f7gx33CGbHeMOh-moSo5R_q|21wF5CxJa_68$D_u2jj)geh5aKaRc z+&4FWgg4xpr6aod=n@lpW@cs&T81m(sj2x|?YHqri1FtY`S}>NwMy8qRb#rJe?Uh* zI?AdUP_RVCF=eg$;-23cA4Jx%bPB9Jq;#tVm`LOJB3}EKm{9PfK%(plI7D z#2*{p75Cc4Ih+Ncf zK>DZAh%n=^aw+Wx8QGo^vkE9WhY$lGj()KD)3nXb*4F=dfY7T~+1-qC>{sr87AM!< z?bb2*57u3DTn{T!bBD8=dOJF#5m%KWikHkp}N$zz`Z-P zy=BAoJE+k7wgsEG zMqPIVStw}xKFdV(c~^aTcscWe%UZz(lT}aQ^!{r5Uf1WQ$M<7Kd_W*jOq(tWpC$?) z!Xt*n#qMvi2hU62pR}oeOlM}ap8H=zZc(t#QdXb``T2kShLa!bhf6BPsu3#xeop+6 z(uAk3&RSHa?p$Olhib^dX3Uf4{RPzs^=l$AoAtQA8Z?#qPH5|Lm zgvjYnphl8(x@Abeh6uARh0p!WVZo(;(K=(hjD7^Sg7~Qo{O+QE1$1JL?sk&%L&YqE zoCv@LaUtwD9dGnuUvP3iD$dag-}|9~N!xcHDG|Vs7Q2(rZGeDxN~Qx*wG@Y1HfaG# zI`>}Lh!_$$3!m)r=mIH72lrj2k0SVxg^V@AlT77j9k2FQb6sUFpKUE~ic~bLm}qC@ zznL*PrgzC!IVS#pD7CuYW9&L;>**Dvu8&=W#Ak$CBb{AV=l8pFuB4*mu}>*HFNB3pNc zr{>Q$gM))eccgYz>cQO`0V^h9I>4?!xxJeLfRx(-13o@~K640P4r>K$+cc45qH3=8 zNUB)CckIsxNJk>iyrKDd}}bXolX#7-tsA-NLX$ob}5Y7Cp@T#rU2b#)B6 z5lal=t#(qzwCnX;5}t3nqN4_LfqB9LhbIA>a5CuX8}M~v9^V8tsA+9#HjmP8&BuE z$BjIHc+TCUPB?)H*{Wd?r6$~vPxr`&s$_@~1T#@w8b&3N*#^;TqM$nW^5k1C+EeZx zGmUJO3MY{^z&m(Os7ZRotvmGDS+n@;BCMM~e~~sSzc0yUB!w>T?CiV`hImMSLiV zo_z+}r>3xJWs>+3pSS|7R#PgT|zjb?Wd65~ZUd{O)<=73;6 z$!m>d%=GnGRj+5nl?qxUvk~8YhM=^~3K#_coJdDBx1B5Dn zb?LqnUVt2B!u{V%m+?$tKBA;<5#}1%!xlK2*kQXxRs4v-4`PTILcVba$C zV0|Y*%X6AA_1VFOxnhYpj+QUUIZ?FC0Tv7I1iz^WFpp3^wb;7B#=EbRzM=I zI^nVGo?LBXv^lMFuL}}oy=6wTSL7ytaji{}=5Aj7xk@RanqLi63QXs-EZGF0H57*n zDo$J%8%|hvH^B=Tj6xw@7d0QUM%Vq1XKh3(ax3 z@v~Ij+pUyFOU6Gw+v-a=jEkVjWSwl-|Co2^6uXq=Rm^F*Nl8KDoZ!8>r1L+nepFq= z>8+Jss3rW$3~0}Qgj5jGS$x1z5Mz}%Sl2L*8U_FqNdyp*s5#^);}1YT|DnPR>=#aG zugB|#$$NPU$Zm%0`t{@)V1clIYTphbqp6JPEYkRd>;nvP6CwffX;?aP8$qOEyVM}m8e|`2;Mlb%9Zl?}s~ItW#8=s$mAPAf zFCeD-r?4O))m&;c%DbohyB1abk#6#TW^ldVpo5bOU3?Z4)c@w!@XjRN6ZiD@&)3&} z!Xhr^L=Ncc(x*P>fBrUPtK-L+BYvI)NWzO&5z!KLK;M$o*vynN#o7N0Xqr^G#5`BY zDz3hh+lx&OeUQI!WFBaL9Fn(0kCnh^?d(rZ&k#Gk^v3#m@e*J;8()Z2&!WpBt=0c* z=|zXX%@VhV(V@AFzDXX!;PhO{h|TX*E_D)wlm-8x4UsQbRkXn*EZ7b{qN4U0${y;= zw3lPBx4GC%^SGfFri!~aesd2oQYRVrmAyo?hm+O*Ww=_Lm_xOHU#Pqab=nMl06cn> zf+6R##{?mX3~9(#*G)=xG%*6fkdJQ_wDBcZsb4jfOzA_y2I^%stH0s$Mz|rn5h4co zf_ep|Wamxt7VqaKzQmi?hP-qJWLf%s*CqB)gdb5i(E zJGsgKAZu$G!7LbBZw+yJS$?s$&sKijwW9>a0dKFj$^ zF)_c%6E+R}jpJ82Hjp%-#5DO*-uN6U{mSgHdN-QI;7zdlIKF->(>eY$=Nt0-bJ5u@ zUKXRboYAQU`Z1)_7KMFH??1d2Y~JTITdi9&a_WD70r}&D+)4narEvd0XelZ{&xd8> z>CvH`pXT~ow!y0M-i(oB>MS+TNH4TuyAOT$fU&BmfYNI0>0iXPg9MBW^Y?NT(dHUo zW69%j2(m*jTsb(hAF;urD9Fl2dIa|8vJ{+Jg$;{s-C@DfcA2g<3>{y&vz zvmvm5$8hs&pau6;XQLRl$LW-5>00GO0so^QA9g%mLRdZ8`wojSUyGAC`r4q^k9Vh7 zSf^V9{d`V#qyKMPH2;%yotpt+6%DAlXdR>Gv4MtbyTI*UU zvyskS;F}l^pD$AaC}1w48$Rey96ZMFj4*wF`%($FCWpq7S)zE3Q~ZGaS>D7yj1o2# zlx>)8)ck>?vOhyZ#1tXsAHBgSX$FaD5qeZ#KraYGXM`3vhVXI1T#l!QHk0JxxX;wo z)LX>w9-5{e3u@|mfhuD6dD%Ae@~K>zjNXIGv{Qnm_?br-2Sl-NClA+%S<8=Ed)n8H((F;OCE^(_dS! zi!#?Bl)5Gl5W%C-ETK?Va}TJlsi7r$yUt^e6Gp&^NM;}Vbvm2}2()T?D#{&y-F8mQ zsLBN9iY(bD5QaQ=e)sX7-q7!!%i`fI^Yz6g&k2IupLPh!1A9+(G)HE|-a7l`i@(2D z^9@+w!GOR;P5`c?HG=Co08Xt;g&V&S!`JLV?XzAz{IF(YtL@4j z1T8+D6MAZSRTi`b&oHpB#>A(2QD@)}@KG~12a>ZO1rBquD^s`K^DtYgUPg zAyoHgDZ#iMvcV88=-$QgX$lg1B0k5j9z5)Iv#3QtoqH`)kojsv*LHajIkv^3%W6gf$K_~8zVC%?ux%7 zpJ)}yRpSFNfZ=&U{pk09Pk1zM$qIYW*g`@%UWN~er=JBWn=~Z zHJ<&4iKyv*NuOzh+1uZ`8YdOsX+7I7631C2%*p^03-hQ|JdCyUIRf1z%NjU#uf|MI zK6O1WD~^&a?DAB-Rf7}M-X%_Kg!=rO*`kzvLn$%yeuv%!de{jeq=xzV2JL>gi{!MV z%UNGkZaR!mC>J+>1;m@Y_WZmac3CNP?pJj3)rCU5G6K8E=X`wnBG<4xwU#gc7_`zRe-1UP%yDQ^_N~`1uBaH=+A0C(~@4F4NgaRUtB4wAeHrWu&#zfVrt_^ zHBTZ}y#c*{BP3z!PD{n%^t&_-TUQUoQMV#u#R8tM>8+*8C^&OAS-Tu_qz!JhC*9{N z9v>>aTZtPRH>p*XE&5ag5->qDLkOT;S&!Brr7wrlRK}T7tXnjjC4o5P61vZ_&wv1$x2y?&Y?Z&l-zRuK_FJnc3|$Z-4VW#x zloVe5PRAKpMC6)s@^>X=4!px>{~K8gVeA{MI}rfsa_&M zgn?dY+czq*-phZVi!FHbe*t)tYY5#MRfg8h0@6S`2->~#07{{D4E+iWyYbnTNgY~J zMsfy!dfF3)*$Y4#> zr@QQjSK?1MkU47#O_(xeb>ltI<-c}3M%xw59;Nc|`LROuAI@&V^tPvSrF3hU>eghC`y2)#kxlO|jVu1$m{cR$vH~ zM^cwgNrp(LcQKQkz@Y(ga#FibLgm}fYHSV+wA7irsc(4YhfmdtoC>u}(t#SwiIY+s zUPOKO>*>rj{hRjM;AG8`wlrbCzk5Nd2BjnVqn~24_g8~?JD3Qu8#3K$$g;W<*y!?q zJ~xx>R`NkYWNa;i(m64tX&DDgKz|3V+XlUWL<5xIhjbfBuv2ZUu|#x3a#&3zv+a~4xT%9Z@Ol~Hnm zmx35=YRks%Zu>cO5;c#+Jy?L2#uq6?Fe3p(eq0wSZ;r9PH;L3D%ed3Y_E<-MR_ek{ zfn+Pnw+^A=dLEwhg9U=1J%F5u9N4*QxbDNpqxr)|-_{As5;}lW;k3 zD}jC!um1Nljit+CbYD2J$`opU2jL#-g-uAvo9dyI(pUn~qXp7ox|3NK;S|gM2J9fT zaS_KQJF|9!21h7z#|o_%z@K%3WyeOdlL?>OtQmdEtiTAn!2^X-&DEYhI6QcrRbd}j zQqH#CUOIOY7a(p%AF{Fn7%M_YIMxK^w{kiSVS7uCGy)Th~HcozjfR1YE!3|fl zyVbb!Kn0%KD=G7n$59%{2u7SXDESGSr%*y@BcONcjE3|#q7gnAwB`+H4DPW6{FOAN zlZOMn^~${YhW2e`?8>mPvCs2{!4A8O6QY9*3}HSoo!tq-JJl?`2SSu@G_bFFBDfxS zJstmy!%KSB0LQA`2y(Q4_?fiH#-C}WF?Er~+KR0^z>`AEOrcx2f!Th4dwTa>JKd;f z=9*OX53Qr3868|s(z-X0rDlYg{~a!Ib}si< z4Fa4W5PklH#GksLvs^OS!~X1lz*W{6U*HgEr)VaohZPTt3xJ+~wz9T0E#fAI}SZfoTCW>;wE49$T_u+z(r4XK(4>AoJI9o*xJ1SC4i$ib zf!yldO9x77`53c*BLXvS)#zA}9FsWj9sK`Q34@HdOOMeHNRy+t>EI`;e#OJX)M@0| zV=5G$kYK*&x{@_*rKagd0mynPdXSfQ3Xf%8@fry3-ngbG3tg(TU-`x+6HQi$pzbHr zek0c<)WJl?$3MAtc+uwKrm3ANuAo4j*g{Hyw9xU@92!!Wp1FH|ts+OIzEv z@Yt3uN5Qu6nvFS!fBU8#LoHUC4ryLLze8k*z7YhWo0`ezYrVP4?;{GvtJJ|kgb34f zBn9F2p1_WOsmelbW_yT&e8>kp`)VmE9VLRxfR*Va|Zp`>L>iTx{dAm|2nJ@UkgULpu`=-W6$T zgc)-b^roRtIq@W}$NXFB2S+f%S>YMY_StX9V-*5O#&sQ#RyVaEcf}9#K3K=XT9dFo zph-y76eny)qw#_k6$R;WaIq=1I0;)3c=Gz<+~aRG!Cn_n}1Ih=Z^oani*vpI4ru_&9U3zVnux&@fWj#K%W3E>@%8GPt;yz}~Qq z<@gg#Q~+!Pf!v6I<&(C`MFXW^$Sh^+8U;?l89qj={uBI{S%vD9SJUr_H=eW@_3 z3A&o=!RF5an*`J9VX%))Ef0i1ZFYWIrr?Ke?qts$a#ngW+oq#nHhxXN(o{G?K-DAo z-XSU?f)oM3nOss;Rdw2I)*suNdF3-r(fSTcLN3<9)|S&)Mg%P$FM6SWP>##4mOTa} zf%z##xQnIQZnYC?;V7IIty~~-6;2f31zWKgZ+!PgO+)|{JjaFg*WmYH)n-3aUcq+( z9BPmOeUvp&0Ci0;_EfJ$ zE_f@S&3VVGjG3$Bw*@MHtxnt9+DxJkdz%xC2fkxAiw>Qyi63*}+*<_|R9>6u?GHuQ zM(DX(`q^b;h{ocd>!35MF<^KHm*n484Y$e%w{a^b?Wg0h_d=XzzA_VR?Eg@yE`#>l zVpl(IMC&g>D=)1}o_Yd-XC1Fd<~P5V8#aqw>>N5+-+3*zRUXxU(_C#B08*qqrn017 z_}9!biLr#hzI_HgQu}zBXNCk#$M?6>A5U_k-usw&Ea-7D?80eEM^b_-Vkg`=Njd5XhrVL*HZ&J{^X)GNmIh>{XY^+p=>RG z32ME)bbs96%_K&DbbQVpkzbbz@4O@{Dia3bizgx=cXt2|=Gu8;(y zaAvezo_bYT6b74GA53Wc2Xm#_b>&7X%E@*UPKB`V7$L$A>{}j#`IC7&=A+;T(>S|CL;Dz9R<&t08X1B~ic*%j$KRFmr1+gR`u;scFuwz>T5Y3uBLkSNTE| zFfGL9gfMY`N?PmQJK7!Q_Zn*UnzGexCMOy2gugIWAk zp(0{pZL8HryaRBqQrq9$u*0epIl`UNmt;}6$&62E1D1)p5Bp=KrluBm!m*&0>n07i z3|EV)RYiyZRkzY1yL3t%GnqQj>Zd!zYDksA1`ob|8==;w?^>TBv++td`V#86$=1&14Q42q6F_zm|Nd^ylP5XHjtCjuh2lO*n__ zn$KZ>ZT(Mmb!a&bo=5m~d;7y0m&M`uJFtgy_b6;d8!*WWKF-U(zP^s^O2%4hQHRa{ zg1CKs%93x~Dfj6wVE!^Td3x_?85i@d@AS>1e&(E#r!Cuom<1}C4-eO&n^u&qK~EjG zvWwxMl`@zTX}`#t^*sr?0u3sB5hvv0kU8#uZWMI;JEQ-kbd}A~^LNajxR~E?HS$T2 zFp9%|U&p1FX(g9wO*dbV8@i1(y>)NC7p9_uANK&>gd>vr9VTd jxi0K5Z5-8fSmij;(te9W=PRyL)hV5AGh^-5o+mu;3P);O?#o1WSORK?e)& z?(;S0{qC=B-Kulmb8pr6#~o_AXL`DKuf6uO)_T_7Yli01G2!_;Lc5V(+Pr$5KWh4& zG3Z;3dhRlxl$!w+Y5E}t5f$j?&@Uina84NebvHnrL~X#s(pt-mWDTbJdHx~$VrYfDPcSRB*eJn}rPtrq=q8~O zq8Iv=1o``a<3hM{4H;dviD{kOFR_E!gc>2`j$2T^rQ`LXJ+$K((_(0N{gvUM$l zq1k~Y$gy9Gt<$8WwA*%#`Zr{MY@x|ZIc=sKhR(p*nctp?( zG-?F5O(W2GtM@K@u%cGe%)fA$do`i14?DED zdUjzXLciaf4SHKF@#vaV$}Rn1qV=7Ag$K*^*!w4SUkqQrP63RT)?Hy=UZSwqf55kh z>V$lMY1qMa-ZX-w=%F7YC`RnZ)_S!qkmgnKzEvzMz#PsG*GLx^S;e5a#;kHntkSTA z1%KGH|2Zk^bysRV1#@p2B9sjC(PHJ*rJ-?I?M$f8d;bKExcS>)2m za5G-$O&`LEpm#%G2Avm8die+(SNo8EKoy{Wgqx>^zVMT018rX;?SK%SrZqAAt=92t zWczoWiu6aA+@GJHnIq+4df5C))5&N={LOzo(E7(en;Jxxt5wNGm3V_0*^=jgX{$b`5r`F@s9kmw>$j&M=HL}Vah1f<$vqjUY8T$J}#@qfp=_q@c6rBO7u0l6eBe~3R zVz*)QQAb*}%JDtBixU)MPBGG-$BF6Lo=td}prLk=Jbr!me*ZnkogO9;F*;v3?3!lx zu&Z}Ngp4_%$jV+;^rh_e2|GLmd5tW87gNi2v;;NT|Dt3Uk9-GE-#NJK4yR%+AFA&Slb-$SPkb6Mh1Eu)3%mTKoqzrG~CB zjy3j`%E>9Ntj0=b`BbKUsK8wq(s@Q`Q>I~~uuY^D^H@+c+Q!DuUT>gkUe*zRv{Ef! z-`0q6lFU?T&@a(99a*74cjkm0H zUAddFx$YmX?QF-B{uuO>YbU+SWw!j{M60- zmbJLisW!dOfR&Xy;VqsXU8%Bvi>c|`VtJn*%i0|i%@2VO&Cz%fx1;)dW9B~MA$v;; z!OSs-OH09dUUr1;tJA@GsYOO_w_+VB69itgG49Z>_)q1+p(}9dGCo`C9t$?tEiByY zPAV@8#$N+`KZ!dd-KI(i1^d{2w0e^FybzkV)~?E)J1UscjS@92`LWY~u*cp0nzPdF z&mo%w>uiNl*WKeGvbCOD{MM*JaYOsLUFR|`)0b_Z)sETX#FgUjI+Af|{9$@0?*?*; zFJG_BN_0HeO-&?M5t~NSmTa`r0 ztCWu9q$obXAq0-DmBonAJ`!ejyUDgvzi_}mWEb&@1|tN26=Ukz5*{v}bSOJ% zvm`HIzA3I;RIFwdr`KGX{Z{l!4z||$M=G8#jvQUf8lfVe#-_oTw3R;cm!-I)Vj&jS zq)71|1dQ7h_FC@wzn^^ym0P1d$lTledG*QUMDZq2$2WDa$4$QtE0;1sIE+%q{!ltk z!@_|)fhk4l&@aA!Av7(JoZ68K9JP#J_ng)cVIMTwnf=!#2);!9xLeS(FWYol4{Z=QDLp`VH<_(deY3K zBUr~@o!!mO*Tz?->e1hStlr~TCT&2tH2-{K>hZy`0pi$yG_;61wUjJ5VI0}&{ibO2 zI@9N*p0z@ML2az2&BJ)CtF2nA{1HMoJV}1?NaN*C=j(=g_$1V-dg|LCm+d2f+;#3C zc9ZmZ_JX(oJ!~0=;*^okRJZ0I*2MW?MO%ZS+g_6mu`INVIV!m~PanPK^9^Yrj_7oM z$FXE0J7=hW4Am>1@g+;)#8mG`RP^tlQL4`MRX&_G>>lRKpszehkS>}qb~cJ)3W6~> z%h_s7q#y3ZyUQ1l;6)GoVJw#zZD%J}-Mu1_jhv{bXlB>t#wiYDNhBAxY27$Qjd)&- zCxbM&aT8Vigst!tqmZG}Bcb*5Q)(955k=PVJ?dG17VDA0Vf_Y&vGyD;4xC}!rgI>} z^d}?Y{`~kHPp2d&vKNy3=IQBSPliGl4hbgDzD&DP0%B~P`DV`jD!iTT?@2ud14f30 zDmfS)&rIR?W+Hj1Pv@ww_DVtlrWbQuCR&7 z`(F<1nzUdJJB5AiD_sFHmWdF2r8Q0G-fbg)(!WpbA=s|1e8vW$e?C%#ilZSvT3OAu zba840Gd*$*8Civ)Exo@r8L`g7boQWmS6g*prKm(c2(@ ze>;Dzo*Y(EjD*>d)Obhxg(X{rt>hQ3Y=?J?bN}vLUumHKE7!kYc=cmjQ(S3J^5KTX z7}gF$ulAh%r7B~RGxqiJW<}$!A@Qkr!QZcH<`3-&3T(UY8;zCMzZ=-wS$Lxy*$Ax9 zNS_Jcm$0UJ_l!Xt4d+vWTk{zVwh>T&GAVLyyL?3c^A)uh*A*b9L@$ir*=2m`Ra#k; zZJO}ea48tI*rhhsgmwvGt4EyS)XpQ7HJ|`a^ymT%hkcv!P%EVS@o(Ks9|U&p_r!-o z%+x|SazJvzuoafRy`A@o>C?VScAu@F6)i712}AnOrpk_^gpoZ`waoq)lwUM|PSI;B z2}&^lOx?)7>3!;*poTl`!K1-MYw<(xDRJU9*gmu0{x1Gk7FpP0cac?P_l}8ssa4F_ zJ4e;y*KUW?Q~Otm@K%nEE^^4HhL}w8!~*vEmn;WKOZ2Hn|0yxbb6eY+S}4dshmh{d zP{_dni7zS3X?Z)GX!HFE6Jp?h=r8=NG0DGQ$S_?KI(VK8`3U63!~Rgcp+(zMyd|mWUP2!6l#CEOs8gOHv?S`<~Dwf$A zd)o0s_12E=7QDkP=-mBT1U4t#x&~C@Vkdv47#2O6jurnDIAv1Il`(IB!T+7=8~0ml zOBSPaQ@78)-q+pxI8+?M%E8gq?&zzU$K=&4URa*iUsbLO->7BNXz27ux^n5J#v$1g=HCkk&9~2{)5%K=Y0n#Zai^`G4e3gQiUV(3lo<(#VMB{Uol^+ zsU?r>r+aeVk;+A|5>(P{W?iOV zn5ZG5yCPl>Nu%>(5kHUxT*r&iHyHFhY9R+!IE1NM0b_N9pH=yP55YXGPlGscicFmm z-JJ9*0W^YTSKlSA1~#)U?h0TD@ag^Z>PH!+Md-n2<1HhK62H!s*8W=4@174F37fK@ z_j}k=$m=V5DaBA^dlcxp64vUt`F)zZ93XlYGEWv{)y%TA(21{D?GsDVJ_#+9*8-Z@ zO{+L*+mTh;>Bb9xy8e0;a*|wO(5EA~B$r$a#K$4MzTbWiia!rZJ$SN(LSWGVI=PK^ z+Z5GT`snwoUE}Y4b2#kbr&Rb?s8jJEXnls$7AyC?L#&SX&DB*#3tyMM2`b)Qk~3$u z6k8GLH;TF!?x-Y>*Jpv79x+{^xUB+iO;1|W{2Y5DpDiwbOXSur@r!grJ_@&vd0!r| z>a0%o&?`>&WKnmYeQEOJVGhz`<_6MpqSNh zMF0CpeTaLznD*><&6bx+9V#Bt_g2UKu+a|F7~(|3r8E+NhcU^?M3Em3$HdYhwfG~P z#P*JQE!_C}XeXpQDsbX1FKl%K;YYo6ywHM;=hOFp;V&J{Y4F(?3%bkIsJuIKc*4yC z(CN|Byp>edjTXAqsl3v~1%d#h9c~}YWc3A)tu`H6Pu+WN*v=Y5CJ z9H@fa%Ui|NMM55#XY2c>^1U12@VZxdY`O2AkXd*4LZ9(iY)5;OOi<(AYRvMSvx_VLSWjsjkLN!SlSrT5_*ahkxyhZDIO(i= z##=B=s!qF6+T2dvl80Cr&6VQb7@Vs6LC$=C1BQO4Ue}-W#;J#C=NVCvws;)sk6xaB z>O|46@w9b9UR3lne7kVDXn49$fXskmAF|BYu57Sp|3S<-gi{4o^kJ>WG6-qzZP6Aq zThJQgHA2uSLLgH+jU3hA&p>~JZaiFEZTaw|8B7H9vAPe0EZt7UgfR#w+}>9z&C_dt z7V5C1aO0^wLzqEt2&>|&LL~UBbJ0dMb$n%FNmbP!R>we=79Xd58QS5iVS3tBjPch1 z#pl5G)QW(!fW3CkDRe}HP(`iprLRqsOo$A9jSUSZCmE1!?KMj4*moCRim&hjaaRnS znS(;SJYDu$jAd(%So4{501~q|Nt3>R5aSh$pU`Y79@!ecX>Rq4>Gy{`gaP)osSl)< z@x0dzH;@mQx0etc;eRNy2$P5Mk@GAP>1iH{9Hwjm+!ZtI+=ASEs)B*F1OnXAW=!$7 zEIh~5;~l!O&l=Z?CnWKD~N*8Hdb#BBoFuE<+Dm6`C|gcDRV7y3t0NoC-rmSv#yVR?;$J&cO*z{r z_w8r#N=w=&XS();4^u6q&Z*W*{zl9e*l4{2Tq64I9llG6s`BKjXU8Hxdu)h@Fopk! zOf3QN2l>c%c!_*<+M9ka#(7@Z#6rK`79qvN>{ZqVLV)jqI@Yb0 zzp?c{6lV{?0jN{IzWqg(Mi(9l_&C}+>uzqxByO*I&+n@mUs$nQAM>kdYA&bec(8rQ ztVaXG;I1nRpIKMEcpC#|#7s|)O6h~3gY_5DoN0pm{PyQ9>yr@?ZKHjE{sicKFF{Oe zAK^)79kF`Dq(W?m$Z05cC}K|kV3HG@+}g`8!j{l9WSGQENLWeB^q^hkG4OhO6fXCs z;&M*f1tUvnHB~u#MJPA(_p2~HXtuQxE5AifS9<6?Xgh+x@6yW%$K+1BTxYzEzq@v2 z#Mb%!4(Qk&6Ni6GQ%J~v$*jxy-o>$uDjQ-T*DP0*$Z>aJqoD%K#Od{PGFe!Ah|<#4 zS?fv&hdXlafSMuO{W-Q?Ntuo^>sI^vLYAoA3kQqvTg zwCf#0zqc<{7_gv{E)K!G7DT! z=(*uB#eUsbHfP6wXIBQNlgg?&lUq*Odu+V7I?=V09oRk71RG|xnz=g?1a(tVV;O%= z+o1)Wr#SFk;vZ_-{@QR@hOab*tR8Umlr|0V0)2Pa{l9|zO~1^PdZ#ZC^;e(C!q8aDrmmPV3`URg-W=uj=2P3)9-WI|E zCEj`ZA&Nim4G_ANkcegxLgS#{zP5vg6{U?!i~0w3{#9Y<6!ym3&nm#Twq8SEFE*`! zOU`ii^wV^50JOxP-r@Tsfm*N09j^>4%zaC%McXsvs@An)B;CPY$hLNIH<5gvN(>|; zT*OpVQ!wa%B)s~^y&sTF)+wy=bO-f#R#}x5QYIilLFZ`{{1EtiA^szYdKfXd6;!JF z$nFe|Ok3De1Q?fx5(UF)6qbGFgk$~-nLELr4A@sH^8x7>dOfr6SmkFq{hP(4 z<(h9kyESSxy4%?(j-TZzr=o={3;B4AdGf3qjCrSjzNNkEVlJlFg+&M)P1Q|0_-9$f zDc=0=?;m+CO^>cTnOU9KwzFg4+)B4s{_r~rF=06JZiVWaWbJyoq)1=^ zl#@h%JJ$7IK(v+EH>^Lkb~eTCO&z$WSRVYQ{3cPO^5`6aQk8u+Rjty}s&`KRuJ&{; zzlFP16(BOxjCK7cW#@y#ecM>q%DoEqO;&=sId?nHKGj+)TqLxGs3N-iU7c1+iUXDk z*_+QVKV=c1I$7Amy&7&ja=q1DK=1bgp^3DA%>M!*z|Cai>ou!X>?bc&K8;VED!D(O zH||5{5waSe8U!lUs4IL9snq)79{8~HV8R~E?nO++YHA!9)%hv0#>Mn;bINVt<;Rqc zg4X9n+tdUrGQkKz2sEmpeWhCqT(AGhwEh#UFQCVC6q;*qOtv$1n2z4F0VX!Z{8G|? z3fRwk=B|S>6ZOT$u$d{Y=5`GS%Ybtt+}XwvE{C6wn)mi4m%rzy6p>LJyT3I3y2ocp z@5x|0(ol8pSNXu5W@OjzXfI*Kx^!$gKgXI?6VbdK4GL3jKUZ`o{clo*iW9-kRi!hT zZi<{eFDZd9Uz}=i7^+U1>%RHke6faqKxJi%cHCFF_J%!Ea^_)WDEuF@)}fHao>upC zHnUHW;uUZRfr`!iJAa>E{LfOcavwXga+d7W$1#sX$#xRhb1k!OxUr+4JZ`rz{Co{R zyQ`-cpDX_!kiJ}Po8-R>kA{2;Co1w=zoWqk&6GPmQk)D?*xu@)1NE$TjWg|k$d0^0 z&fn1Xpx*VJ0l+kN08}YyO9Aj7^EY+b{q#9+eiS~W^f4;nf-0BBJxdleob}N7X z0WMLQR@O>|*Um}y+EHY0iNj`p5HW631Rciz{^uJ0$Lgy3CXkD-;}ww8KCqn~2C-oO z*b@k7HzI)cqvY^QY4fylt1vAjKw?Fw@%R6KRpP%kmSKH6 z(?#BCgFkEA9i7=KiBZEmk%1PDbwe+x|NG^CmB7)|*(GaTD?;FojCAkY)3s`=Y$1T!J$K^#4k&6bWyk ziTVDjLd1VycYJ!M(cSX1@&6LBc6bO#fgNYAdrjR%V$Z4DEU9=b>1a+Y&5LY5k+&}q zG61-Aq5dmjKw-jv_&{Ul7UfzdpFMMC+CG69f)I*S0PN4`d(i#M9#O+ZW8ZGEtr;@M zwSun+{TNx8;$PUu7yK+GnSUb%6zn7H25Ml1o=QhU^-Sv*t?w>W_Zad8Kt~t{wzd8x z?&*-aL`?*}5+I{Sp~+*%o{J_e@gnFSyBm)rYTqVZ%v*k+cNas%*%KTn1r(7)3VuxWS+@`nLG^*H~X1i;3R9gumo z7Y_6?HxgWcJGnUl@c+2;SqJvcZxc)7VLUiN&vJb&#uxc>8=gM*WsmxGIsf|HvM*l}}l z^6*js$N0E7KolJRN2K|GJiI(S-?{^$IN91;IeS<+|J`}OY1Y>N(9U0-;h^}R{r^uM zQEDo3s7ORez)MsGd1(z02#NxDy^a76Y;DDDX@MPog@wGv8xY8!4g?B~0DRUID+UZukQiu<^OuhjyQ+{mxEq@ zUBY+q(AZB{dk+FxPWz?joRf&f<7rq#lxXYW=Ecl9s|+6U{HRK|#eY zR}lcGK{U)c0y)2e?@=OM9&Os>vJSWidRX9pzxhzhfVGbVR)0N&6hlmbOP&IvVjy7& zA)FwyoHjh#Uq-F)pazBx{Ar`eV_&%wNa{1WdMo=A$h4f5lc2{4~6y_2|@@$9cq3Ej zupi@#j!9Yr3e}b_km*6cBmMZvsvj(WdBXe@vsVeaKT(5wOJ!cPIar7&fDf{lCXz%2 z@nb;PrwlB+*~DY|DD3f)K1~$2rxzW>DA349BQBzT9A~COq_23u084PbThBsdl`Q|Z zy{>z2vkSDBo@{$QqR2DJMc4>3!v<~YA3&tu2-U!0-Oo>w9GrF&pqS#{8V-YhBtnlq z;DzZ@9|U2X=b?a-ntEz4*$Zl+cfm@yR31`?2bgaZA6;TyN$@p#p+UMuq(9)qnWG@| z52j_cor@{1j>)@^Iyc3FXp>$rcP5?_Z#*DGpgupW2RB!8G`5Vza|D(tp*5`M=L)E& zu_%f3*ipR!GpJDHXvyB-hAdWpb2A2{W*^*VMTlxt|LLG~xMR(5*S#Xdf<^(R5P8Tu zvMr}~K=(dGnpnnQP4~@qND9sS^}%($?^Lo&uJ$+;lq9tv5MHz-;w|Sduo0+CLcL7y zvK83^^9#HfMCmnvmX(MEND})Kis{}3RyBM?VFEu%Od*C9f2wO2G^l}pAco+q&|<*# zz(^g;@(8Z|x&xrO7IspU-V;v3)og3C5X_MX{~tgrYV3JP_ZJ-tKQR^k^&W%;i*Y>m zmc_j8V+ahHit#{SGevCu$ASsH0X;YnBGsERJx?^*eZzjefehp$aQyjS&5+Z`+He*S z#t>vOAia5>ihWETFe9*kn8f-;ZSRhV@aK6#&Vc~Mg01P3H6DyOo@6^@ItJ<8+aFkG zaV`l}VQDVUJo8o9GmKCJdIPRuX)7g4uJjiWuB&5_C1g;SNl(?U$tj{4e&_69K2T^! z2VX*Na7%rCeYv{2x+vt&*k@BndIaXXw^d?hp$4aoHtbg&UI&YRPWxKZ&WeiBAdu^G z;QCOf&&dcQvxPPY<8njBeKI|GRdtxJq!1X$&7pV?2Zw_cRMabR!vF*jC?$0Ts-O-w zL%BHJxYK*2tlaPDxtA?M?3I@R%soULC6e=?^TeMv@Zu+*7{9>Jlkq=9Fv;b=OHB0_ zNQXy9-whA@s;Q}eA^2a=c@3rqUymmhZZsXafP1bsE}X)~tYYXX5-Tdnr6eQ}510M& z_p)3%(tG@`=9eKI)_rCK9s$>ubhdV=`&LdOq-{pLR$_oGXk@V;rlzvUMS~x0A973I9#6SC`#75ZuEAt~EKDjZ-{LC~vt&g@ru~?Gj#pB_rdP2Px!@a1a4R zemWZ>_glFOT!;*^v=ol`KyG)q^3eHBTN_v>{vWmJ_hn9`v<$pdv%!@dfdzz7dN;)mCu7;u0jJnRKJgA7QsMv>@ z$R+i8m=YdQ><*Y1C*hV)?zEYRM7tm*Bt(C>kqf!KJ^Qj4B6G_{@ZAUw?rtU{W2Nb< z91(bSHokv+y!-5E$;aFDC+8l`g(yX0-7kEBEJGMsOmE(puZJu+@3EwN6QRC5V;5D$ z4mK`-L}I;D6FQ8`pR5t4Z0`)#^&pxem`ivLu{_4#oK-p|37Y*dT zx+^d9*VPttrWkWW^F>h7lkz0v{+W6>Pk8nT(S zK~b+)2`5RP$FJ^*ITW|1DGy=npSSbKV0oPCQlP?3UFF55i``o#>#3|H4=Y$FAmCMwY3Oaa#sEBp(^;`!vNTG6Wc~BO zd~-Jn>Rzuv;>rHLTz+*moKX#I_L}L+s9c*aAg@ z4#+}3;+bc%@D9Tc<55nY+l&TH76g*|#)&~lh#DDbk>ApSHaa?502nL^aS5Dq3D(Tq zTvT&-+Oxmwm3`(y`nD>T*U9RCtmW^zey)q0)KnPBx10=&jA1R;YHRKSDS`8IbBwPE zzE_2uty0u9G0-QTgojg2&dp(ES%M}bT|(1j;<`T4;K7(L$3b@Rd$wD{ zZ^73m;JF*AZPJ~ru|FT*U;sA$@21MRAK8IV`f^n)G2?)%p4=|Se!ti?av5L z`(tKR=W^x;CBM`AKfVg(1gWi$nGHNpTOP!#)uZfaXp}artK%4i7pi>?p>0}KM?(0G z^exh^|7?@7`Y@0!9mdM)kJG)bu9*F6PDrKn;Dc_D1c~o${P_`obci9V`7%3U`B(A+ zqD0wuQB8Sh)AL^=oXWRXyC0Mv#fIYZLf?Liga%9t@DXOV{IbjSRv0bDYh~Hf=;Wf2 zfLH8RR5cZQRC+BhM?}=M)ZFZHuejMQ5(ooYZd0!b+>~u!@12FU;E2_AYj=F7`qRgF zh1mCM|H{<#vlvo;MOe!q32hSlyMmK=jMQT4t1LzS8wYI^F3J?c<(ZjfEgkPyAOqgC z^_CzfCsM%tq9Al9Pl?j4&MWUDBTGDhHWV(OpN3xZgFU-i(_Mt$UVxbbrBl4l%-uFi zw}?R$Vy3recNh3WJ=`8RO^Bd2Dotf_186xAD_t6atQ^jNV8y9bQ}|>*fSDiwE{|ya zt<+=fPP>)xGm?7l?&Zb?i*YjPY}&gX?a8LrR)4N0=g6QJ7o`e1?i2M+ad2?Lny;#K z0zzosZ@S&%qswxNggjFTi98*~laZ;cK#5bGG@Ki&sX?EggP$wm*d^NsMf{bWS(rPg z-z@T_r=B!_GzgvEdy(D)5&o813o7JQbOXZBU^v3_^-h9YyMy$iGUwjshU}%tXE9_7 z7OIVKAG5EE{l+@IlTGH4p zx$1Ji6&jp0Pd@IMcjMsLX|3E{tyiv)%4br!;n2sAB9=E}qnd^ZP>&>V0@VJgtF+F^ZE!ob4hgTvOd^cDN z#PTs8a^5xx7Fw z#OVPB8&L2|??y&P&sX=RwdtRpG_ec3Fh7KlMgn zy8Rq}1VU+7*@OEz>pOf8J_-FE=%6}tnSpj{Ih8nY!=KjUhPCyCs0_>=mpn-B&#dX~ z*O_46=ku|dZsA;s2RAlR?Dr8{gU-ttysf&mE{X5H-M%w1Ip7^08Ch|4)1Q>m5hkHZ z1;`>_t7%q;=iWwoYV*$M{Z(n<`Nau;!{SnWFA?ZGW+XSrZMBr25L=>8m==N zz`9j7ZorjUj?25u*w@&JuEIrQ>9xkecjvJ4W6aNo1ycB4v9Xa|cSo&+F~c#-y-`5; zQVzMCkUI86OY*{bx76w5l4T+ia0LzWecU=rZgl~Z$rkUDgEKOcJ(e!FMk*_RkC*+d zQ}ROOFu!f}|8UeR&RnR&A)BB;u*4DMM}$?_;dPYfxjrQ@Bq0e@%LC#zIy!ykUu5g~ zS!n%d){U3FeRLz&>((6HrvZa5{(WUVy`GY+tdZn)kD*ik>;fCBj$Ipn8$?8z`hc5U zO)qcW3KUdhZp08(LBUUGs(EvNN3%0pC+h%XupIo3h=%q7pv4&+Z!gy`cHZHw4@pU$ zoAbG-WYrcFEZ6yR27vML`Z9V$?0wD5BL560?w@pZ(?hrduFF4kRa7);hvNr-veQCRZWIPIeQ5%NlhaObQxgdhK(ORwWLy{xy{|R5)-l6y za7uV>ZBMDVdEnx5bKQ!!4u%F0eBAlNu2|AQM@#9%OPMxDX5StkRp`w#@Hbqc50aB= z+L9mmg~&Tf%NcF8baGS6^EsOPzfnV5*@zYY3EKS5T~z#MGQZG&{K{-^s_dep_=n@f5D2}2#uTm(8hstvx2}Xd@SgN}Cpx;=$%nTl!>hqd zoB{&J0S*VK3<7s-tl*ImAIv=R*q$fvbGd*Ary-zwQQu8FS8vjtp+l8__1;5=Y8Uoy z!hs0Z`O-k4%%BW^W{@i;DKe$ers}Ud536pMPVW3%S&{kRDoER?e~TBoW(y+MndkH@=_J21N#*ZR#hZZ?#&0*0J39D|M~f*x za)cVMd`?=Bx6e@0cD9yzy}UpD$Q06re4Q?xH9o(HxgryP3!d@oNsOsb6~SHVlQIg zkp-_#Ouw$t0zeMcM!{U!O&%18H4(It3hUt!6Rd2~pACR&6F?@W%& z*mG!qZXz*%dGc`kVuBpgy$)2eE$d+!7V6x) zU=J{?Wt*8ujz@hK7Vmilf3ipYh_8xDC>N3i1$TR_L{g)epq7|Ocy{QWoRRFFxoM9b z7U097cvFRb4)!J&Q)iYMg^#|4{$y$0LDGbOK>K{HyF;E67mNnVv96^x$M^8z-xcnH zUs{jq;jwlhe_mV_eQeG$ytE)=KO_WYRTi2zv;|38ufJ=}*dXBkZSN}O(zqJ1bY@nSL9nh8@bHlfPbm~+ zl<b8=HC=aV^~3JVNLUF*V27nQ5<`^9M354bBH!dnBTr z7b$DsSS4YcKm@cePfss`+eYN9yw_NNg*unD07+ru@vl)RBi6tA05;@nls~odZU{Nx z`{sKQY6M44RUA46<}msZntf&JtUx!n!4A)$7k}{$Kl<{L7#f7HrKi{aeh`bo6kPNz z>G&P8y-|jxO?p;?9`y2$B2wnt#`bm@u{5q4(bm&n_?fCKiGI!Mo-Aa1rSc_zcyu5= zTCO1@`T0pm5%foa#CDyxU6K}Z5$Ng&Bk{>Dpd36z#T$vhpk%a8pb_hO*s?)1^bl5} zlq2(fO$71?45b(OML)hM;`(~Mw%9lx5Pf?a?};EUQ1Wdtt!Vwqav{0WXS0_#HSdF{ z$Q1(M+x66}0SPyf?ZG|EdfD`U3OCVFn8yPg>{h)18#0A? z+xXA%LaS#Zs*{$=R$DpmRjJ7H20UQn6e<)Xar)W>nA}%e1Gtdhr>4E=$*LRv*OoK_ zRLxUSDHvmz%J)oGIQ~(8h1M?gmOtg?U+zIONEhJ-rQI={4fQPd*!Tr^sUVYg?Oj|Y zURQCdGwae7ZNQ-FYRkYp=EX+Wo#H|ko9$m!NJKp#BVa40A@+Zry$*VCw}^Ts*H^~c z@DeO6v;j2U0SuzEY7rtIk6ZAULL|J6$A`Ndm6TMUaJ)YAde{wr1=cSLQ~FRWbA`VT zSzWAt6%NzUOCr3y-uZ#(7b?B&8Gr{0csQM>Av3f~of?miX43mOfvh{ft3-;~2Wh4| zlM{exzoFcb6wL$~V7J1@<-L9wEJaH9X|e)tZ=iqlB>$R&gEwHz5IJVW>y{G;awgmq zmK{AWM*d5B;G+S5#&Z@50QhwZi{T!rV4CF_DpBf}(fGM~j+du(eT`|=V1zt>jv;wZ z!zyo`9B~_Gc?I^IjGvL^$G}GkUrY5AC@RSpZ-90X>)tThc{?g(BhUkD?yo+lsu@LYizaWbdw<)8 z>^jW_ocu6fFFiJAa`M07+oC>G1m<{uXu_t{$S?@XTjMvcgA~33qG7~ClPvRFTC;L`K z3sL6D+>$@$cQ=jEP$CftTgD?35lS62Nygc_Q_UEEJoFDq1i;d+>eML97f)ro%J~H3 zU4aQ2AabyiW|jT*p3mgMRI+%&^v-$W4jc0?f;K^a+!>O}harYIeSMXeKx4xgLtx^h zO06Qz^oq8IcEmmj<2|v2j7v+a*kr*jVk)m>u*yr+T57b%uBG%Ja#B{C2~I9b1#P>w z@R((P;2YGffzHpPE0ZigtMf1=CqHJBVs=SC4}T_KEjmI>_WT*5H2kdDie~kVnX$!e zeXhIwUBSB#UpMLF2ZvH|QrCWAFQ{)mBYyR1+utqgWoFbD`NRaVANusV;Dr@GUFlvl zQ!KV090hsuMUK2H-6`^xvSZORBS=lssCeOj>Yk%1>rmI%oAzJ6AnUC6UiiX~85_TG zhk<#`@|w#TY0=HDs^R>7%_o%*0^TZ955-ZQvZ}xW1H?Q5*wK37Mv#6RoL-cetzvSF#y~mp;8iX1hb8?fGd>~ja za^gH&VR0raE43OC$wm}MetDyc+Ux#D!bhmH{u2n~b1eLti;E}_SFokB$9z@z-SW)L z0roJMhGBSaW;t-n13$eC)|Q+}QpR?Fw=isdJrH8pn#&+0B)9#D-*9L}B{(vWJ#6H# z@2prp_~SAsYHpww(?MLC^#!4}2X^->Q&kI9==m3Z*Eo~a!}s^k@}tp9_7mu%-c@f;-WgIixB9$v!ycp?|AK*iI<<}jQ) z4yE#G&P(<*X$Xf*p@=x0G^iKh=_KNi_WAcJ$79qI{eX&HMb7l<+tp#;<8a-!OOtjk z$}5kB#s(zv^IY;S%piVJr>!UgT4+}d! zIj@sCuE`6^qOrBB*tKK2q`Zl~iH+5g{q5J*v-u{UvG+7klnTlAJGmSk*Lmm*N{{vU zow~82rjX=AC_%ReSRzNBwzpf^iNt$JN9mVJb2ghAWjjjK|?@F*UZ z1ogp@ikV=Ap|5?4oGaIV@^tTjx+BGS52NpKAW!XpwZ6XdRK~4{lw9;?Kxr;OL+1Og zvdSZ2)Z#=bM%RFBEQ1CQ;j~s!3&efcLT;XA%2E;dib5Zg?>Zxu9;fAh{ zY>vI=zSASsH9Ke?qmamxW)^V78{E|hE|vu{dSYWitR3b{gb!_Pb0!C%e~hyo-m&+R z1jx~zZf-<-AqJ#@3V3@eRrm<66GpO2sFWVqBM*yxCZTE``R^h#HZstV^Quc*Hi$83 z=~6YIV*UKTr;c`iSlr*9B_ZfLIT&H6*$Lp`bsh-TA->s=rm1=KG*Al+{5AI;8J+uJ zzeetb{^psfnP>5p?yOTjXqMJIDL)%)YEH^;rQ*n*;GDc%{Lz{=j+*SnX9oo(ueJon zYw{eSR%`MkBF@LYHDTc3XpDomb0Jr!GaxwK%;@O}DShOB(ux~}Qr+&R4){vb1koBc zDN%tbLH|eq3}^wh9igGBlT0)=PS$PB(s6;eWiv9sEw-{bUzln~c0FD;8Ghy~jF-@< zh_GhwYg0(ECHtvF-Bw~lqS4!F6j?;1SJp1`q5MzQrKMthQFVysV1sC%!1sAWnY9wGX$eKUL6H+P zE9WD83BgK^$X-`LTL|zQ)J{C1R5;xLdBVW|=nTCOG3)C>R^GmB207ZrFv+%ZChi-0 zO);*2Y_H~^E;K@I82v|LB$jA~WH zT*43U_rZ^PcBCSXnuSgW(#C&CBuY?xB%M~@+>hGOvKsSzySe2-lNC9blz-kZ91K;s zc6TfkV-DgadTp&Q0wnb>lM(^fThw6`Q2K^{F5-Yru0W1O`DQ&i!k8ll+Capy0eA); zo3QvE|Dj__1k(y=2Xnq9JDW<~ZcmErnV2&(#rIQq_?_a1z8L_+ekgr#*$breAZz2(^^wpy=lEn!GS`A@sp=nkrX#FN9r z(vM@f6fbiy+S>NZWPr^QTtM!=ztN3!bPxZsJmZcwGL&V_%XqiwXV2*{4TiPc% z`PWh*)nZz&4f^LVC3Cj3rK*g7o$q`la?FuDK`^AQP^4?oi$;@Bv29W4AG8PQ^ow6y;f5w&T<=Yvso`HQxVIEFyuerf}kO^TSb>AEo)Ny44?j6OiM7z|Jcr+>Wh% zg6Qb&xY5x@*FZD8NWX%gS+7m7qz*hk`F42?&=d^2x_m(dwdIF~&NvMuJbjO|b8+ow z=4v&uwG-f$$9}oq;ycT?X3D1)3+6&)9krHg053q$zw7bVQU#<1L%_IlsMt-(*Vif4 zPcfFJ4;^uS?0gvk2S2!_H8jkNe=&v9@Vt;l8D>%K&+kNgG`#xQiecmFZzw<9TwTub z@$-{7F1B$V@yt?SAThI!SSKMwbV*FX{umFI3~=21&JEiqYjlh>O)50L{GEgSa)_K> z49hRAB)4^*ra*zu_H|!WrJI&k|JQ}an;DA?{QIXHY&hToO%QLAQTWH)f0yx>f8(U` z1+c2?mw5rhrKg`NDv8xUBovk~PhEbKB}<2oEZB!pI>DF;2*|AB(zXV_ND^@M`K+}0 zDiZC?7y#qhUBA4{;6Bt!K)Ou6H@8QRNE}H+-neBAkFJ(uO3Bq^b5k5KCNZdap6_yj z>5G2=`I|S2Oq`GlRft4=fBkzCZ|}Ec06`0pJg2s{@yUO@J}PB8J}w#9OhRaE6jIAg z^ERgCaILA@>AO0JCn+psJ8BDhV0_KEhqQa(5SsXFt@4ZPm8zR(HUv!i5)kJY46zPP=r*WB=4~vy(S5+t{dk4JX-995c4Q6ZF7^i%!nAJ8s$I zUsK8_nOcKEo$2z|f08Q*WZpA2dfZniLuO`X_PjVCU3m}&a~EA};zSM`!Nu)4AxV*z zn`_?uBgj~t8GXc)CwdSn?KKdnIAxcQd2d7zje5$H?bD6_s5tUIq!k@=Rba{~j9Wjvm?;GNY~ z#2!pc{Kq1re+H?quRv>050Sl+QWqH(-)-p--$OHFCl}b^;8E3Q(c^Sn9I0H3Xqo}7Y7qm<-2)^T;M#1@_(E4{=31ZN;N;7R|R(n*I1-YHPhO{0`ZIE%&qE^L>6x5nUY*I5Lv@*Q6r~ zOS)`af0}0j4tRN4JYb?FG?lq_;p>tqJ5t|{$<5J@aKu8ofLF*|GCg{`w>N1_ep@m_ z`t@Rk#^9Dzw6a6;reev#L6C-LiSVVT?C{(J*k|U{YoP|6nwo+F#jtC+lHNeZGRq;q zPFkPTkcPqL53p-TbB1dkH{2udV0X6#@X8TU*P9+bxKfl^y>WZ}WxlBO<} zn~|1A+B`ZYC5tVVmc15u6%_Q{9hnpqaC5Bm86O|jYx|ggZ0MH=IW3FhcoQ~t3SHN? ze`eok$kYxr4RjP3evceh{dSJM`k0tM`F<~t|B^TmA?gJn>rT&Ko+=~ZN z+h+y=Q87F`EUKqhi30|KNJtbfHpTluaD8d~ZbuQ+K!}xK)LPi`KYp*Ql(AP;jpUt_ zXM={0DMBTzial9$k_7gGIpz6jx|o$4cv@D?stsLGOjuI9lbYkfB;{TZtQmK_f1hb_ zv4K{7-6XH`%i~M4Pd6hTfht{o_5B`1K~47lL)079RSYs%$oCsORxK+)n?cA?Dh`S{ zosZbkr5S&smVSaj?(XnRy;ec%ES4;Ln)*F8r?h#JA-qn{JUiwVqVdW!H@e^DsXVI< z*G^AYS_}tJ!sI8OcwsR{vnxrze*sTo34|_XIQzdUU^`Ux^*6PrWtkNa>Q7G4l2Vfe z2NDw3hyh>2suAm98>R5hXLl{kN$j?R+(>(_QVwsbqc z%F60co%#8^;%Zczqa!v0hlrnCDAUt6At~R>RSz4H@@Un<;2n8%e^CKe=?@FKxXIo# zH|N+HgG^}ink8z1Wl}lb&n->D4E-;hZI68|EG%N{?N+{0#}bvh*9qQD47J&{d$}z8 zct2fgSxt~MXVxgm4r}b83zwZhHt zWh~6kH&4fvC~}_Ge-uowM6+P(Ro^#Qi-MFfadw;1tP>aviQIUF-+pYBC~*BGgAzIZ zcWApJ`9s}biYYky+QEkJUmCEz<=X>?a_>7Ox*Hpu$=KN78XCsGYx%Q*^_d8B1iXYI z2idiQO|CZ1b7c)h==R3g|HeeobHE{%z%5b4|Ji}D)(NOFf2vJ++f6Me*#X;w{Y))^ z7M0M;VtI2o^HfODE4R~;mxlnP;vXt}9PrW6i-<k&TiAgfXImYK4bjtVG0J3yZv>A{by7F3Q?>V3fTua-OU3UsuHrGq355hG=Ed(oifwCT(ZN$oD2WrqoRbdrVgopP2s?E zQdPCl!AzH@+_=UK2p`0{q>~|u1B(B~io-6|r6-(de?cUkbi4l8$x`8`SmKT0xRdNZfU;yzJCBoL$fyLOzHI$GELWJz;Og{%>5Ym znm-=DX2S;M)!44H2-!WNfIv!w0w!H|hx_StGO(&ZIc{@%y2WO{x-p*$3NTJ!(8n|M zx!Cwze@;aWGInwb<>KNhq^Dhv^@9Xb2x4#nRDi9yISn;Ecz(VSqLr$uvYfg?y|^n7 z-xF6le_}7-{g~)5xZshI4D<(C;U2Ya^Xrbre_IdNuiwJLl6r6uLxC;}&#R*Y0%2eg zG0bp=Tic4?IdngWrgY~rB1w|HoOkP~G+B;~f3g4@8U>{*@Jvfv%x7ZagO+Tp)4{bn z-0ns#)z+TDv9kOleHL@`KO8mAG)hi2z?%ldm*NcS%*?en7eo|uFJj3O-9|2HO&l4$ znlFDM9UXuXic<3fyS)8TrP7aaKUKuWW<2Wd9LM|aun@+eon>PvahVJK-)YO5sNpN2 zf9JzhwZtnNs%qf<{U_}vYogW8hxJNb7OfKNZmy5w68&DlEA8>Df_5T31hf1F{bqZ{r$UO0yWs8d=7!wG>d=!_mqlE)v@h!-EHxva-1~q)(!72|zz;N7NR$~Clwu!G)*6bh96$dN zi4$8Za=V97|7tIE^{_i~X9s1>DiX!#d$xLN#mWkcNi;sGiJqS5vckdJ<{AP7fBJQ| z?|!>++A|-TRhbO`d1PqVUK<}jp@uAm5C*1GZwJuzTZR(DYvWz(0|Kh+zAj-!L`Or} zo#t<8)KWgq&EJXcXT2WR_6RX)8#@=-E-(o{UyV#*cDOV0sA=83^47;TH2l73a}jsN z!b;KieYiSRs5o{#UoTeD(1*!k}Fu zec!*|X@7gzcd0z@R`VR(MB$gYV_ffg;_*FHRolxEv@*Hgq-mmb)7s z#!Zox6%_h+M42o0`f|Cwzy9KL&@@aNB(ITE}tXon3AfBeci4;S5k;e$_C zJkB(noGiqRH;72uYP()q7l6FXh>!nKNW&BwnRPX`@^kFPYj-$=(P}&i|76K*A@-=d z(ekuhO+Cr`Y=cEH^*Ix0x;15G6|5Yt@l2p!=RFS0zV{dPd^J!im+OYzwzgmqi(Tlw zb~k=OjNxipx(WtHe|m3$QLw)|VK=z@1@Cy<#izS)yPt{Qd74<1J^DycyO`iZi>G6-2BBaWk8O%v{Xy}LTO_}uCa3JS!-r-(2+scbJV z&!7`Xx_ya~!Ve@5L}UR6(v}=h=$#NI9h;gz)569-EJFnSfr3h*!o+-5gOI=v2npju z&b1J!5IS*tf7$kqc@o7R*BO9()R@`}u_bt3EH{%62tViDq$+E_12G08Xtmke9s_jr z&yf5&G;xgcM975oJg$+J{R8)1B-IAMa89RB1 zs`DUNh?rNkO!mgl{e%ZQBSE!-$M+GPzWmJPQ1#Eff77=1pZhm(w{LFDwZfpc_rv(` zFsy||6;l2x0SJU9W$SNhI>fzDE9^{mm%J09q{!rUcgZUR9<*Td_Lo>dP-`S8Kn$NkAeW<{aSQ2xK6g zNGZFWe@m?-D)R4?({CFerNvT5S^B#}HB~WeQ)Xs*Xf-C>+1WYKYJdOqTCdV76d}L+ zI2t*yzXZTn#+ts6Gsnp#FD}KnRaN_fPfK%utY-)d&-x!sc25lP8xj*yjiR7r9rrkZ zdACssRULwMUNE$Me5MS2CSr@=u@Dk|MvB#~e4N~- zFDzlhYoH30ssrVq+tX>*V!9>fU>dglY|Yk6>P22@AQHv_6$BD%!;vqUmYd>!q=0Dr zMJIAbdoYF>OODxHO2+4tl2re=r&wT}ez{59Zz7GBG2mq+a5R`gXmQCN5Bm(z&R4|maop0`gd-O3pN(fLe-}{L zh^qUCu>mZE9Qhfu*rGL&egBY7Wg`a7PaSfKluP<7bN}Oedk|cS*9M! zvVyLI9Ldgb*`Fw*;kChDKUhoFe;WD^e|z(ng?@)-b)bWcoV`3c%7*)_Ezu9#6GNyb zRktLr?DddXlQ8b#npMZg;#lJkWEu{C2lTGUbcnOFeP&qrz8*w(DT|$8`Dft6+0He3 z3M#?Q(=hnWb)ljn)j;9@BdfIpAU$FJDg#dP`qT4s5W!(ixD7fA0H`Q2fAI0q-QBNq z<8r-Lc{DGJ<#4vk$Msq#FX|v9eP>5i2xP5_GfO_5Bk118$&p0((_umu3Z#IzX$1nE zOcx`)nDnVL$AhO<+ve}c$+w{vK>WkppnzN><-+RfF^zndin%ZWe*V4)h}Nx8MyAXO zcqu^!1|xRfw%M-cp{4n(e@~edo;0NBr-`(TrvsnRq$&eDnHgA=^On1wxF>>w`}oKH zgExoMr>7a5By?n<@(RsCGt)Jde;ZiiC5V|W7QZ`~6#qzB+Y1-<^HXrJv2b3QoX&1w zi-?U5S?+mlNPZ{+misq*3hO8`L^*-gyDcY2$`pt57OIF{5a?kHe+z37iq$)Ea@uF~ z^hniZ(82Zj(+LfT(2T9LC$0olI(S!EEpvv)>*~n-gIoYl)~|1-8c6MPX<4FOU(_D? zbsBzOj(Tyo9Pr63qvL4vVdj`V{^Kky{sKKsP;+zXX)=Lu5VGY!UVTsq2@d%1h*&nu zP%{C;MRviio0yrBf72v{GNpOG$cy(vLWsQ6%Ag`n{p0I@(L#)77U&RB#5oO|7d6|y z-X^q;&z(Viz{qLc-Zck5A+Plp&NOi(J&t3KJi=CdERn^SV_+s5k zL`e-kjk~~1H++nMLy*&bxnZ{&Mr$$V&i10+UNk*CeBfMJK68*n_#%td`N@X{WzZU; z^>o7f01JD4e+oGSC&m=x!6UZ+Hl(|P;xAkltf9TE=aO^O=BUPyHX)_d*tkOJb4Ep7 zxEmIB%GSk486Y8I?&6(!v%cc5`?f;B&UyA_bhfpWPv@otfj{Fz&h(n#UUTG-=FYkQ zqrjZ3h(=Fj1!8D&3PkgCa`M}-M%S&=Ai}K z#%FXA0qz;#|EP+Z*aTM@e^pV@ic3im53V_zF0Iu3)6`DF6yPMR`Q=bN2j;>bvcX18 zG@VQ9e?py_^OpfV^n7E(Gwe0*2PIRK=kZu4~!Z7~duJe&oy@~7Xd`>6Z$$2%sDd2Cse@7a3#>27vZ0bsj;kd||U>jU#tkM=Q zc4Bpe{l?~>WAkz+zf;}jzS~nBt$+WXGZf^1e$rdPFo%pR%eH3ub`)P zpp4BDsa{fOnT0<`!zvVCBETR5!Nr@9xZMqJJTW(C02T?B|NFZ#2>>;(-)q!S4V7a9 ze~f8Pp&PXspswzOeRIa0qL7=H8t*!jm9-2|(9nP7MD^QbfDa>O7xA&{4TLDalq8#Kacpxk=?|ZT+f3kO;Hw_%6QGO23*S&YeiZ>Fz8V z-19VEpsg`ivff7?#72gR;|A;HDM7~Bf6N;rDM?SNqkU3QfDM|qS;7uDl;2zaZ-x@w%qfe3uxt$(FR28H1f&oWe5%P>_2z2I+0Xq93Ph`8F{{!x z=@i?}XysI`G;M`xBYSYVLQO+VLF5a-q}Ngw&BcPpilG=xNT_M;XqPmyNylT7e^zqv zO)TBVdwN*dYp?c6nO?tbv{CPuip!p21&rXvysHI-bvZ`b<5_z4~KwWx$^ zzognuxjghw4L8C^d4f)=8YHD$lIW}~v^gv|w6-vQx%puvnA0D0vO$bSemmJob7}hZ z*o}3IHv{wTdif1kRk9#10)fR>f9P?35Fu;&jFEiCPWI!=T@P0T46W1W60yH#y!@o? zEylf2{icYMyCdj-bhlC%57YatY4Zp8PWvee=~dGyWMs4&6ck!M$ynI@#pHyD_GU`q z1=Tc~dFg==uFYda zkY7w~A}T!lCCGm@CGWdW$PGpfsE2CxJRZGn)`RE+Q|J?@wlmz)tAgmx8PFUjBBT5E9FP9k2?Km`a?O_f2*agM1LqYi0Ln6Cfv&MycUh$+awlk1Djk_oj|Cg9e?bi!i==p zESEX}8SG{7x@cYRjUCpSW3X&V!emhUs|gDBP(j9dR8)Edf`Z!=pLJLvqsGX)18XmP zh+J`+gR3)s>=T{-)ejD>g@aO{E7dP$I>kh%z$0C}DcYLmf2xB}(}c*xa%;im?H3Sj zTt%V=A+sLZF zL2DGU+!XO4o&vp{*?{2M`Ec4=0;gdR9vKJS)uQAQ>g>Q1qZu?_`uw@&FhRrJY2P&$5k+pUzQbdT}7V*D%J(gE`I)8k75}`t~T;NiY%hR z9@=?-JrzIMT9g3hP|zRAf#{C+r8+6EXMTh|JY*{p@hC&UgC%>T9o9>j`fe=Wanxhc zI|Nr_d#mH$Gjrj>-amb?1CW=SXbtqg0kt6Y-^c$1f4>+=5Dl%o5x6J$I-z3S+v@?F z*_0>Ohw;jO=a!>@SB$$i((Zcip8$YPG^?aNoernF(m9INSQHdQx1S&KDHB2{bpA|? zR2r|WNX;G|)kp!ck3QYS-Vi`x>)gOmK|?Oo`{j2(!&FePJrElKHSRcbVj>^ORzl`4 zk6&J&eup%_>|?;tn-s}Q=o<9zpFn@ZK4A<9$DdvRlZWQ>cN?S;rcmh$)8RKQiUtqvpb8dWFa=e}wruK40 z&g^Skh|93wSIdwDRMWDy+UBQ-mvY5HV-{nUzHvJXOW(fHLlkdFL#XjO2t%>Sf3<^S zEEK)0@>Bs;*lIRk zB$Dp%tk!YaqD|(aH?!`V6N`q|F8d&|)pA(2^e_ZF8E^>w$p=kx2!9kxS+iWEd|CW9?3@Z6azeJu`@9YmN& zAjvj=hwcgZnm*Y3B)q(Oe-9g}C!*qJ5nnz``T7GXURb3fQQ_ctJg-KgvKSJ&TRM!T zOD!A@1Y1}rGWskkk@V^Ip@in^KBh=;HQeAdBAI^cMvzqJgJWrwHoNnZ%qjoY!zISS zFUc$ORMYdLBLJzt-EZd)+_FW4$6r1-3VM%si>QtwLF#tyk=X_zfA!E^e|Csm3X%P% zSE>;tea+i`i@Azz1-!k``5J{Uole`A@4h_8soN@`QP+9xjT(&gD+IL0&CI^%aG%GN z+4bnl?C?H;H(0x&;k?;%F9?Y5b&9fEP176!u*KD`)7an-gYo2lC?S7K?_ajI$v8OP z_}bnem5i8^|1hJ=D=1kUM_&+l3GdrpoKdlaRJUW;Y7*r`4dlVRt6l!@WrHNld= z(cB`3n>M}h4W6X;dK)yy>$AO+z2cjfi64|**NI4IXz6-SJ6cQ*Hu979=ex>nZ{8rY zG;N*3IZ|Q6hUhcD^>r!kXj0)grwtK5Eu9M4Ep=^e3opprfAtPbUwFF0u3=yh1L-S; z4CMpUKZvcwuCLE03lM&$W@S;?cRKHi-VyVUf2AUcjqb#-u@`Ef73waF>hZ6@gEsvg zB05D&tCcQNksGQ8F~^cHyLw0t#4u4&=^i}Z9Mz1Z{E_Gf=zY-hq;kMK7rKYmx>@FvJ(;IS}b9y>BMce0_;BviS6ict8dnBel`+1<B-IW;-!cUB@|UWK>QkRfT? z!{2>=_pBwfQ+g(C6KkFeS(%|18Gd*3+~>=|VD@nI^95_ml9zt)+j~nZj}29RGJ(2m z{s~V${>Ds?bD@on}IJb1qMmWm9OmUkRNP* z-89x+-v;<*KR0%L;AwlPV8=lWrTXcSiu0S-e;*B?wB7(I;3p2i2{7eFATg9PHq z+}|ffwmh{7#$sibobCX8DHCM@5;Cd@o`qRY@1I*Qh4 zuKb(4Ja%d)cGNdbT;}y_%J36R8|b^hcrfb_RIH3x>GS()V(aVxg!~&)|LCt<*W~M} zU3&NpC*`Vvy1W~Q`Y=ZNppPMW+@*KWe>zx77Ip%{By7!@{)FJu=u+*CO8%ugQ48Ub z$Bzp&3SNs_$^vTAH{KHc4~Z0_7dlO(ol7g>J%)7~BB;FYW6*g_*~o-2qrO+X`D>K4D*)3)0fS$*bnUt)E2i~x_1(Y1tiL$1(z-oiH!|igz)@y(ZvrRhL9th0wRhSyxVE0 zyJ36BnNEsx)*sz%E#J~Wri$h@rvMjzqrH9i=l$S+6z$VBGBUdGxfQ@`!>H3AAF}O1 zj-r&5=t#7L z2~fYkxL4{zv8^xo~c{m60xi zw;Pu(Yv$TkI<=YVze~K|f8E0wYnUJjF`i8rR8@WV^LyM>dISe8d0M5dD|VPy{!*g{ z_u}G7;=Pdj*{B~CB41GL~LEk^oSk(@*a)5OxT&y7BQr+eWW?79RTGkt_pB$;Lx8l zYTWB*m-#Uc@sb(nqj7(iZ!R@Y)0IIpCF=iSU2=R>=>(=?hn84rOD8iGwM_KzzWULOY&hhDTai_<58+?2Fe{p;f@pP+lAN#&=y4ILi+b#>+lm3U!@Z9(QaF#|@X}wT=nNcdcXq#Yn;wsGuTGs?#n+ zM;%k5wp>c=q5i0oG0uH|jwyoB9FRb(p#CktL%PK!QdAOLRmZ zm0&uVBw=SKH$H>v{#+!fKS=fuDZ2AM>@$ibgu2t4KYsLDa8mnmB#6aXv$1gkzhjGyEd{Jb*SE2IHQ+OH zCeS6#1HKXA>gcT?;!2{q;)y`)pbyUxe=P|KmhsXq4Rwoi&m+u(73OP&pQeLsE}IA| zB4tM#j2d<>petrcR;64nHrmo5dUB}6zsujCu@JCyH4u7Pe^pm=lX*SR-V}5va71Y8 z96vmGXl>(-HjE34iAu@h5%BdxNgyrN8f2=<<&aBQ2@(eIXAk*o<`vIjjP8@0e@^`_ zU0R~C+m>wVT_#v6tC-ijIaC{lJE%LaGNV2W=>F{)dRmdF;qaZsuU}zJ|K**_CzsO_ zK@)-Ry}__BS=}@>V{dbA)`U zb-SMK@vAl6Fw50*ih}*8JHtw7pr3oDP9y1>_+CLs@<@3U-Pe$7U8*f5M;jYd}Lo3g4n^ zVIU`tp7W00 zHlF~2u+W$3y5L$Pf0iep3MC#NXO6bJ9;&w4%+*)_{ApJ6%}c@gXu4Q#_w}Vs(f5&s z@a7+!Id&~X`Qy4fd@m0Qfauv=c*G|N2+@al{JVd5Rh`5ye*<(0_k4Y9#>vU#Jd4Fd z`7{`wQ*a`Ok+j2d4DonpXL_!<7~P74lXIlFDCv#8kwf5J8Dyjv{}9c?_v+WIdaw_~0& z2q<1=gG)7}s-x zQAh;Nx)CdmNVk{Hb zo4!0zIj1!<}yBBe>^cL_;4EoCBvRq zWA_v7ONuM=G>$Key^oJnji`jGF&gSgds`bSWwoxx%ppqtv?FTWj69u8h_vhv7#gc^=cddzMdAbEPVn>aIHmCoz8A+kHnYR1tJe@jGr zbv7N!ns&AX5k^0swwP1vt+4u+6eaoHZT(q6e0jVXF2vT3jMVz1Srw7X=GDLzjhFv= zu-dj_cK>|Y5umGW2H>}oTpp*3V4X(E>bV)hxUv0Mu-oyx+RW#u-xEi~+U&-8FlF%V5Ee6X7%=q^d-<@7F)jP`IsIMM>6)c zyS&K!h9~F(3KTFqdklyCA=>V7JN7*n?Fc?dx1kaG9fQITQjg@7)~Uzxob`Y`evce? z59jayh&le=;pF<)$tS6df19#I>)hO+_PoA6eLs^g`NZg8VX+Ib|HzkMv82l?8@wQZ zK>u??B~vOYAD^z^c($Csg>hbdNG`CaqS+Wc-X(O zR-ZN`CW8v)n>!_iq9bMu3QC9_U}|2|Vs_3~I-PrhjGL)u5)Y)UfBu4v^&NL*CAN~% zQiKrCnj%{$`#EEb>!f$Lq4@v02y@Jw?)TB0p@CACw5pri%?)LiAv;7Gn~=lEem!9C z{p8gAuD7Y2lk;3e%L8QgbX)a4?=KZI@O00YgVN!Jbwov7e&dzGTQPWD@y>05>bMB2BJDWQ0e>lg2IEIkPKxJ_(_;oJ( z_IFF&Yqx7Zh^=NnPa=UpzqawKdRe^wj&`=4-JX^j0(s6MrPpCJ76^U&0?JEsC7y?P zb==(GCFGvs!qJg&?U$40J$9skx}Je3VoVei4oV7R z{FfcnllnLs8lUxqa<4kE2V)_EviFFgY^+G@hMegnKf486$FW-5|kH+eXd_3&xiai@(@w&ep zj65##{R^OCJ9ajLtAQDpY165EQzP&f&G`5qf<|4-LJ@=P(M)u7CTN6Q+Mkr>=3;@4 zk`U<+yxu@G{u~ITzja9|w=^} ze;c(hImIE0c!-j5WH{$v8lPJ~wl#hnXV0rmPF#+kcxp5uHi_S>mcbenYEDj~-$%hC zeV~pcGzqamF4BdV#ZU-KJ}>2tSU_&r(k$vTts*2tjD+~VB*sR6i|zj{S6fJv`q3(3 zzm&jAK|#T+Ig6v$&ao!|`qk=GwKY7Pf4Tv=UJw*8lqjWb07>DNyD7~QC1>aIr+o3$ zG&HpB;D|wd6tk1v5du`E1#PPTE{^^A^Mhks2?jpC!V-8=89hhXe=4{0U=lOds%~{@ zsf>+-V*;mW0}di%Gq3;VA*0ZR)@T(go4ot@KOTZWimFFj>(Oaa?i#{5Ubp6Ef0mJD zf~KZsQ3c6U>G&ZSL2+?VKpU{HQZN`Gb#2|F;?ot6aXh0_bfD7-xE>VzfEdIe&ZNTa zkh#D>mzS2WLz6!n`IXT?i_!SPs}tkhbHJY+)NG)}w|bn1rXzS(ynBnwPnU$RO1Azht*%5xTc+oIaog8knGcFA7-#_o{sCqy;=>M3P3_7**`Te`0 zm^^#u-T&ceX(g@}6=At-c_)K9|IAGj%+knQU_CI=&;&I!IM>Ta{^4FLAdfsb8Q_Bm z!_R;WN~uu-%1UvEceE6^e|wY=k4ouqnM_S`KtWVdk$33?1{AQblp{Y)$e(g0KuLu? zJTfdIBZD|QJGZIg)-3OH5JQaYtFBQxG~JTPkSzoY3pZX*Puup!!G0}SPVp<3UU)nVp5YU zuHWdmeU+K}Z8Z^d?R}%D=+DB7?VY*guz#GK@Uv{`->W;F#7r>DDk}pM5)wQq)m3dC zQis&v3HggwMky=w+b-0+fr_B;F^usEaJ8&EU94{NfhC(a?6|d_j>2`|@bxLlhI>a> zkPAL(TGZYQ`t@rRfBb{k#X#6H`mH_JU&y_~|IKiX+b&}}J4yYB+hq?O=qBda=a<{D zk`E*#^+cq(UwRs~yO{g?5um9RiC}=hEEt+7D43SZmIcLo4L>JJG8cSUG*?+#>+{%T{~PjV zU7b|BaStTIenm@@`F$&+uzcOs)mA`XCf3#&!-|Dv1A`c&6yIKl*+t>75kUcc_((v- z8Lu_SWSbh6fAt8C9GYY=Vyf8Oe1I1e7+tT`v#M{Giaqm}kjUajL#We704W{Rc;DDG z8;_*$I-sI(tjBr1-Cvra#}7=0IX2Q$(<58z=!r`^vOH5@M8w9ja=_4)-eB0-9bBHa z%x{T8cGcmmwM}{e=KfcU^;!A0Aj@Y@gHf%;>kxu zh;6UuTeB~QvX8XTo3W0+`} zh4>+-f6?mrw~>EWaHSTh7|@lL?*{i{!(&MaH-W6%xE>TY-}&t5&J|Qw=dB2VJ# z`WRe6hrPA+ox(4dR&CZuM_2Lj$H7yo)Yw6wI_<8H)TR1cPr9v+e+>Gq7t|n-x_W2cDYx6vq>Y9KuNyl0 zrS84VJsKKXI4(1RQ5K)ZJfS%SN@DoswqDZDC|r7qt3lJ7FaTGu_&%az5F3ykGwNaz z-iqsD-n5ogR)@wJbo^4>A(iFzBYK}MpJrhJyW3(f>ZOCvkat0xj1XGqF_JRehl2xM zf4ufaIuiiWl@DrdE#Sq+=N?#sNR=slu&`?-JiV#$US6L@EBeBbIUfI+C<49o0st5M zERJY8)n>ca)XZX#|zJ}35(t?-`kTvE%)I6pNpmc zAJdQ@@VTgBPb(*LWo4eMCZl+7Zla>se>GB{4G7w70AE0$zj2xM1dZ|a-i9B|5bk}zP&hpF5eH*!KwpFV-9FL0bY&HG!{r-?zu@#JhE-dUbkpsK_ z^2l17@N{_GK>`BR2NZ30b@>Q;-)^l?$_>0yyhl2V^FN*{)N}Uroq5Xj&zW}l6>x6o z>3`|?K}BE8X_5@w53nZ#xB$|mq@DIEBmauy;pD{Hn}ezBxoW-CR71nWGNSGZ04z!_ zEgpbR$cI z2ojNxW?}i5uWbdq(1Fu797+EZlaBNKuYa|Ui8&0UbYzgj-vuPJ@yLpbZ%ekv^SmF_ z@7OF)CW^e#{x2&L9{F?%q-4wKbqIb{nHHuC0OeU8&2Ex%+sPig-<~LC4hbW{$n4*q z@>`u8*(4nAat@hq_4lR}&`6od483suZ0hm1Ap->fkkz5W`{i~e`mp>j9QoZ;(?eg~8xoYw2xZ1Mv#FNRy**8||R*^2%ryXbw9qMtE?45Ye8Pnz299 z8RSwc<@`xXeyJ`}hS+3I`H-9wP2X%K#;;(rDjd<174W7*I)Q>c99XHYK?o6=l=x8uE`h0nr zTiNi{iW?N?rGye66!cXAVu|=c+T$}jah`Pk`~auq<69bE7-nh0%3NVTHEr!~DnnCvTp=N$nf~AK)M!=*OF&;}Ww$!YD=U)| z-rC%Uz&z(Rd^!Sdd*!)1us=K}r-QfhOHP*l(b?Vym!Z+GdN1h0RZ-FkT|y?m0-z>B ze}8}q;nzh)9e?~a5Oe%%`CH08kH~QsnX9_Ni}sQ6YBRl6P}fWAUPwqmzN;_980daF zeBgeyqamp{etHfC^qTjaPd8?4s%o_!iR$(T8@*(=6az~9h$Mf=P>4D0KiGjXWH7`4X59&Pw8&o?WG2O z27*5m_{vWKRIXiSgZM^vb{Yc!jom#MUL;G>D=e7@N;v&_;$?PnGV<`SHjY>uG2^9L zqigw>{u%?`VRv`4%-fTCLdAutUK5{52vaFa(5QkO5db{Z5QLcdK3CAoc}L&U5_`@e zwxSY1rhl9}<`%sP7K|E=Jo-!k2>FGDj=tfz0(VcMr6vAFMa3gd<@jk3CN5RW$JFlh z$v%-fw$2&ArpxPo`lpxMsnCv%wj&=h85%BnWs=U_UY{R2qU~1L*j<1G-P=j(fSC1F zefYFJJzp7^w4Q1WI-FUi*2x&iw@ZGF(099zrhkFVzqDB`%#=2N+a9QAqM~Fp&{S<` zf9egEYW4E^jeZ|c^hxJ7Xk+KR2b+WAk+g^~+LMtD8hUKiY>>%TRUs)Zu5*!?fBYaV z{R`>L!UD#^#P>){KA~^_+NV~$Wv3xFPxYr<$Ym~OW}*L!uD1+|E9#m?2X}XZyA#|c zxPQA_fDqi>Ay{w^?!ny&n&1%J9fG?PT<)Aa@AKa8yH)o`s-^-p%J6WfH9Bzq^n72=&n98QK>Ubpm&R} z(zxA>W<_kxJD%9TfIW?5i@gv5++;!3Kz|=L9gDs>p{n|IY@MsssABoMEIxJNw@in& zwUWiy$w(3*?=~92XBo~ zk}}0Z8zK~edfb7Avk$v?*Obl8-)lgii-&Cw;Ojo@4BKo&!QI0JJX-xWlU=xf*r)de z1$GB(wt4ytTC8=wXh=7vhNb}XRR&=ysBcy`&v0Aaq7T(wF^*M zf!o=9TN@N%Pw&K=+=+^g&gVlwu?o0Ofbr@0KNaSBC!RS7R93b?3p`c`0*rqEx!+tx zlHgDo9dM6m-?5Pp{HH$;fXTr|({Ux*Tm!&f6>@Cz^>uY$$l2L*%A-bowSTom!Y(c- zV>rxpgfFkJ5#S;bsL-h-T@N7*1K#|?0MpD?;LpTAn&jtuU4oboVZj(gBrQ$nN-^pJ zgCFop*>P^dWBq8~{Etjl>2v7uX#hr^YHD&gC5rC|g3kF%FwZV8Q{A$&@f^6hxH|XG zl{+zXwYnfckdW@8aU{+b7=OP{;~!u^f6j9=@NrXW_&$Xxog6Qmw|RT@)5;x*SS8M5 zKnCal6J&|2SocrB-OZC8qbEgQGpbd3#^7-k-~|)&i^}y?QB~#sGZrK~67uM(Ent3; zNNr2ve{wK2GA@`y&dKvVPD`sTmffJFXUKA>%ID6=A79v@YV&e?p?}tsO@CT?ikYFj zKAUGE&*XRV=@TL%gLkL@ODvnEN-lqh*gzIhcc@m~#-IE1($2L~=*W9US@v&`@p< z=!lV~M9FIe0e5=9?0>Wv1R{!^f9>l_bYL+k<#pZt9c=IJ9)yC9#o_0_n!h|I z5nS9!m}9tW_TC(dRE|#k{Qd*n5er0O{>4s|se9aiO7s0*eKxsq&4{JH~=zS9} zP*GImFY^9b2nE+%m)ld1jm&T#L7hGwU&X4A|FGpbx~4olLG-5yCa0@wB+f43Kp<^1rzAqE6-0S{tx}nLqHM) zl_t#XP1X%kmw)H!qLcYGU|PhgKX93U{1Lnx_-y8Lp1yn`NO^&~@t@Y0{QNOK_v3cv z@-nK>b#^^FFDA+Q{pGfFltG9YkAOhb10{p;5Ka^KkR^7Wys25=j~_i6kP0QUKAS3u z?|6zk8yb*oW*6>gb1VL7;(iK8QHuhD%zl1JsFWc$D}VYW?*e1!88SrP0q%hqD9T#4 z_`p+eU{W9=O`3`bmm$x~RM9aSQ4=E{_s?H&7hKQ<{Fn#WD0-YR$t?8eatECn_x`MhILoWEFO#5k}+lkbhJ4upgM27KQu48}tstAq{y%4y2Qa zfWETq?a+I&XW$f^v`ZSXLs9i37Xe?r$5M9 zMT*vYDaDGvSX%xdDl6CPeA|SBgS+}0PNr#ZKjmh7yT={wSzSGq>gu+@bN`epto)Va z@P9bd)kr(Ft&|A0rWKg2XCN~1b+%tlevPdzUK4ZJ#TL}n|%Ex zgbZaZ63|VpY_4lY5QmyAl!evQGiMen7Js5eAQm6heh0`y%Ta1E7d@K0{v8XCN^SaU zBv&gyM4DmX60_K$pomrrq4GXAF2~$L9$OGF9CQ{J@B5bIGXa~nqGYtf8Yl{G9%bv{G9Iu`)r~S_W!1wrVl>6xzU;}_{Bya3$A4&J zXU$skgAi9f(EVhHrM^Kx#lo%=&m;}+aXT()!s(Zb>}LbB=lHETTs#oCNG5VPtiu8mJPFP=` zfH%UF{u3m+H!=2 zRg5-FoKSH2#|Mt|LmY7|frzV%$z1?Xc+~$q7k~MF6zS#px{gM9;>UDHM|;5HQm7EEC8Amd+@HzP#9Gg? z-;JPJHqfmBpiu;O4YY6^%*UHLhR^IJHT{fKp56oBEaP+3BE%Ix35Vcs|!*js~zos1***;JjYJ}L+$&!IZ6)(~1S zZV?pDNPFkM9teTA2&&+RlG-s-Y9OgQSUYh{(H2+2$#oTZzch^YXka{~Nr@p7U3UoNEyAKxQ^y->~0 zwH67C(r!Laqa0=Owt5tR>z0q7q_F;pzNugNJpz3wYj!K(%Do0Ah2Mub zt5H$LM3ylNEz6Rma9Mz0gy!T;D}8$GkNxWE4!lcCpC$$z#qFs7}8>m{@NCKSTL^NFH0eiv!m;8obNuDPFeWc@zZec~Om6W7p`y3yx_Z=93NGV$7 z`SrNo^u)UnL0=uppT^yb=Cmb4S_hFDoIrxo_lS}RKRD>e%b63 z(zFQ+H6{u-Mie3;*3X3ygy6h@fRpUb7l?`O;HTNa0YuFqA8kY=#(>7!$+Efmvv<}G zBGugddW#HS0G9$GkvEnG?HY%6f_DzO_bUz@>rxU1f$N<(BJrLDVijhmD<1Z-4GpT) zU4N|QP#WSe$kL%M(4b#t{yyJ}_w2O682$0rFXX1ItgAA78Z@)K=J0T&buO-RGBBh( z$`{vc0Dn+qsOhsqFe%N{tQg&BYnLW1t#J@ThW-9g`fhA0#k%E zVPOeX)SLr)$~6b2mR8~%+C0uBW0-~)fI%O?6s)?MM*4%)LA5zV+DBa>v$&X{*I>j#GqTv(_62&m&)dhrd{ z`91~v$2W2^oGlmdc%e%(a@&0`8mOpn3E`}UTYuq;dBFQm&#z*2RY28K1rn_)N$%5( za4vMreS?))ADOgIjpWsWC5r+6x_<;`-KEd4ii<*lz?KV7gb0J8hW8Z4&p4`p_*Tj{ z2o~4X)dJEeSajjVL!u#k$xaVHP zeGZyDeG!GPjwgrDbXPIu&4zSzNM#esKKsJroJ8gP2!^Or0~O8nA8{9c|_s&*?0Q=*P?(%bdA} z--ZLSiONPbu38+F>8}G6Kz|Y3`2Osh!O{bwaD9Kf4~^h;kqVObfDQt5l7H?eJ@5_~ z82ie2vX!_kh%^!Ti*|p!7zT5&vRCmkC0znWUVA-4-f=>ygTWK5?#jB#$d?w+YhzPfyvO`4#tPXJ(0mn|^i zo)EYP0s7T#(xZS_>%(pWZbplzzrVpngLgVZUXnj4VYzODxw)g!dNvCd4YI#n2OQs8 z;L$2DX`-ClHUGIg$f*U)Y`g`%Z}bT6kZ^FdblKB4Ze0P?aADqV}il^V#PXnf+wzueom6d_T3Ir?z(7%U% z(8D8-*1EcJ4ewqer>JKiA6ajC?f!<6Qkm@N0fRSsFk>YV6T7-l&`k*n$`TG67T@TS zvz%kNF1b7Z>3?lu#(CgTZnJ5{P@zQ}1=quy#xr4T@p)H%AdBx&nY#OnuUJpBgGhss zvV-V$lMPzt-b{vMjI9E7NM1&%2}b+3s!H zxJ}zv-BKTQX|L*T{NOV(g3?T6u$vU!_)tgv z8ujr;HGjFK_8tOV3G$#|1Pwc$KSq!tHccdi3L^2>KI zpVqSYT;$(@*ou{iRe=YSuUETH4p~_g4qroeQGYUpU%uWc=gxPoG}2d{2f=)zHks^T zEA9q?vqAeM`1B2M~PW!x({*z=ty9Tm^cxuefBAs~kRj}W%Z zPg+of1d@1>Zp^IPrLfdFJ$!6MApe8%L=Ax~TXv~fe&N_h{LD_tE!d`5?o|n7q>;Ej zKa|s@r!Sd^L`udDoqiI!07>lV*vgRbF@Mic%5J;8o-Sfj-Wx-*b9amBqB{QTxPb&D zD*(#p-i_R8Y?FqTj%*Iv9f&TTgXtn4j-s@zD#ET^)5B>?4ZX+s7BCTX!Ah!v2jj85 zu|qEegnn{nfiARgDD3*^U}pVk4yWsjp3umRB0HlO?3nT}0ef8a-6xEFC=y#BG=ITK zU5l==5!&$st+J7P%AfX62?0@2jPum6Qe9@>U-AK6gexSx1BahQ^SwLEV0c{UPFQ%H zI+wbKIy8T++;Z?{aBnYV^A~%yBA`Em@ld@KX1LhFK+GT#7++!B`O2*HuS@}vW4$w2 z7a=kpcZ*U8hp=|b{ZV(m5C-u--+vD8?o%G4@XjCpWk{u~5CBu|Yn@NN5z!N&0Pf@9 zSefT^%mc+0t56TR|JdA6p_&BplKc;+)4Y+dt&?OJ%ErJ@cWI8R4MBNFQLYX1%AxH^>Y=lQH|MNJ8o0NqAd*c&NPOc`=KApO z{K;&|b^lMCoj^eu_sJ9jxu0JEr-PAYL-yJi?#!PTit)Ke+kY5&yCI@p?zi)$3=Mzo z>9^!unRSjg+oi|Z+KEpF!GFMI3`05t!DZkJDua#ipgwPtk|MlQrvtfjM98dkf{geZ zvHv|QHjZX(ZBj=IhIv8)UMPqKu$|tmH!A6Pj%X2$55Ks<*z&+YWMsCOSYI;+v*V=| zX_=Lm_43wMNuGnFN3PTq;l7^`@;5AI(_||%RJ?ByF`+hL6q^c}O@Fa>w9F}unRHHy z^k9Yr`UqNSpM%yzL~*Cv)KL5W7b0IR`Qm12<}8l1r9b4-{SH>0Qr&U0{nuo-xXHmE z+>ff4-%P=04L8u+9?NcRT#TG#*(}lU?=pBOP7+H8S_{`=xYUc0-{KQKAh^;>VPt_@ z!M7d$t3<$y?#SzUV1G6_RRjhWitvlsowOAHs4@FbJ1YCDBjf~|1O#x)OJ&t|tE%!F zgoo#SX>YuCY7YQ&M*rqx?qU7mb7aK z4QBzDmKkRo8O3-KwvcsqlhwF5WI2=x46Y@)cTrJn5`6$Bcz;s0v%W+jJ#B+83{(7> zL-+HGI@Vy!$`62hOwK%Ym32H1tCR5_p&)aPlS2>nn9hC`|H|U-<{aruBX6}0MzsFY zvDofUX95p#cV*?Nh(b%;F?C@k0l^Dz!il#^3GrXZ2)|Z#H)3ev-CLZqLzNGbIk_qxTf)`8! z@hO!Hyra@HVAW*Zu8tyyt$VkRJj8l-#Q{4gz|`xi(|>70MVdQYcG}+&YjWb;wE%fh z!nAz@%#8pS4LX28_n3MRKluI8BlGu@9t=j&(7n}|zy)~kI=XazlbhC)$>CA8Lv0IM zKz6!U6!X|yV6T*M8s62dlI6u;#kSZ_Yv^2L2Kw!QLLR(dc6M>zjKWrFqffp*J?wvP zIc#hm=YO^$INM=6-^I}mbto3}y;s8~MswZHX8DTtR(yL4*R_~KDmNy#6~|2WiEhV0 zradG++4sG#u`!S;baG1h`@&59zX6tp?OLaSXg?^W5j_&F6J@$!L`z}BrS)EMy;|z1 z;EXL2*)GE}!|ywFm_9J(CIP+*wjy>M3k*N+j$?d2l91~2mYtW$5+60 z@ADQ^u9Tlc!_znTy-*Q-Y2vH3DwJLr_j>_iY@$;`={oDT_r8fEC0qef70 zmVYo*ld2GDjvV<_ZsYIz!41l_0fA=%ks?Z?c{&RqK}2z$KMW3E=3MCr)q(+A4Zzf3 zYCIb-Y&b!TDGE=1Fn0Ko*IA6ebkh@EXR1qjv@>iojD$xbIzRtJ2r`Io@qn}@uz%&5 z0WXfFct4i^!o5}_!L#au&hT7z#bfHnf?Vmo)3Yqlk^s( z=;$5gZYYEp^ZO==h7il&c8l@e(|_Y)Vue6gvX2Xsp$W=*Cl|VMi4}F6H zBG%C!Q=ah(oeBZ3t2QJ_xtX6IL0?c07y)tS#DX+iKN~wz&i^UTOlG^H#-ze9)eu@G z(r={e@Oph#0i2}@Bsb4x`OK;sQLmhQxstD@ZZ1E_81a?DVuCV}ey!74SAT@(Y4kxv zEBhn7QW=6*A$=k)Pom;qSq=C}QMl}s-R!BbP;Q_<9fkm%KlB7e?5Qe4D9)b3;6yXOaH!-OnnG-&jsvNS5u&R%xP*Gu_c z*1mm#dawu=Zi%($37o)&N$UKLY78V!#go!*{K4Fz(7X@lCMu9L`ne~&WHx91m#pTf z58`2vlr{m3nTXl)H966B!+Vq`$B{jvfmk)@_CeZBfK`{5bEEQ7=(DxeL2GtarTBXlUBS{ypLQF7-e0M85v->zmgD!T)^( zv1s`#&=Zxf9L5t+V>CbWIEX_&mwq#PM z7I-flc6gCn-ku1>B+>{u)u5788`b_2`2bxT2LwjnKO6V`Z;BG&CP*(9f=&BD9#TFD zQ!kY+1Y|JT6AEX>9d$XD8ygE3k;%C-ia`J~DFq6DGyUJZsDGijZ#6t_Jd!LQSS*jr zs(^n()I%_b*MBh%03#x~NeDj)5MnCLA%-;!>xY!dN*E+mGBEDA|1|^HP4_6Lo0Q^ScK)>nBicV=`rfpd!A`LRlDCkH?!jF9*}B8KbsN zQVn+z9AsGH3x7f}_*mlhtSA#BI$qv={52|SB?(L>_^9y{@-8SKs*Q$5Cqn$55g39HsVev`iq5ml zUH%EYfnA*W!w(dWfDqw`_e~n5$(Tex1JNPT;(wqd|3xf~7^J|}z*F$IhzDLI5(o0d*kRqR;3g_yF6NX^50`7$?|{3KUfK_qO* zK+c}p(u)N42i@QI&j1AgQvt)SXR<(wj(?BmM}&R|xjT)lu#0I~M0cHydd5pj%2XsX z)6SM@!hT~`x~KlWZ~r_#6T~}FjF6q;P@Pzj>o!*~$Fu15S><)o@#}L19~KJPKBdfg zggKza!S|Ud1$3J`c+uLaQ?dAh`3v9#W&y@Da^3Q)P}Eke(V_ik{D7g6)i+tk&3~_O z^eIeybkN3>vOUs-K-79dP|s{wip?-GJZn|c<%$esJMBJaqAFiRlnyO-=Ib3adZNl9 z))enxDRe9qGaZf8k0dafBrBzy6ItIBSmR?`zp>? z#L3w&&~~eynAEfz7-&O6KI0MEl7Ap>y#|`i_le!ueGh=j{=7_!?#jOe$kuY*3kNQqnuJIJtHHWn;Vxg`!_%Wd2}QV z0{t>$=9AIOE1|htG3kij$072lWjSL3m&_Ra;aHu` z?bGAp9=riLO1t}RW4AaVo7m|DAhFVEvA<;loR4+8lvO6X36zws+keX{?!KRfT>pOs zXegkI6$hiWq(t`UvC*vdVRl|3V%oNE$*br5p9auScJ{}He$dOv(rae%sar6)ht% z;Cveh*`TL{owpc5#z zCndk*F@Yf|_`4`5u@ph?6Y{B@8XNz~vH}Lx#pulb6LO^H0)M{Bo)2pFBUyYL)Ji?z zi<4JfE%oH{_R0)}1Nzn5i-0bMA8f*;Zk0eT{-}XUS`7<~$WoC22`_SMRAOKCGmKN&rvYy?&mi9TE%f6ng zw(6lB<;V+|iGNj(r+-JAnu1vyPJ&cEqSiY4^<_Ey8#4$5W{pB)W70jDJf!PiqK)-Q zMf|glcQ}8xGx@r-5`jSIh*A9kSP!t44}0h9ezieLm^riHpF2^`olB56I+}R)*L~_w zx!xndRspTfJp=Q?4-X#LPR6`ILdztxk(n6$OC);zw}1JX&YL(SvOru#Gho=2m*FF@ z*1&L|%l^?<%%`buP7szzYi+JpqhlEv9>^UX!^!l$P?eQ8M{%UAx-`9%I8TpC4Uy5V zCl5z+w`>bFc>Wyjl{tM_<^M??;+WVTJofm+{*B4(@ExAE8JAbRyP=$1nW>D75tF|l z$MqgR3V+J3)YTIZx1;Eb2LipmmTh8S?C)BP5&k_}ZCznU0O5RF=a!NRb3WZ{Z+pZ5 zBHm1T8|j{aT?9&_A9?xo38}Nb{wX= zh40AgOC4T`^2c6L*kp0o-ZV{p{SsSCnYXbBV1Jwo($0?gkFz2q_UGpXU^XeN zykkm4@}mR@0>q@-v&Sd!PP46j&~s;ZZHbPZu`jI~kU5eFxa+#yo~R~>c1A|n=$$Sm z7=JqM$mAbg?V4wXmp}db(%G=`1lNGI%`xx-5izpA z+{@0+kP&5gQU%TXCO+x2q#2-FCE#y7|9}*w+QRLil*t-Y{JFm$hfq0_`wFuEsWc4` zvvM~(QhFQW2MMGCW)rGKUQ`BOvd{(`>Gi=})v^Q8g{)tYERKDS!y4Hh+T z^|kmg;$Lt56y0JHD4%#b{QatJz|}r{ym0<>;Tfiwkoc7EEJh@~jC;PkB{y36& zCS(~8+Ys>9Y)+f*k>$XWO3D4<;NF}a=YzV7D08mEUup1LXt(){47;ftzke6~Hb(Ni zpG4av#m)WLl*v;vcD_(k@|$PFbuNo%A}{i6jgMEsa%imcT3%iKdNnMJX71q{#UUhQ zr%JEZ_~fae;0Z}w>S-TeR7CNV%1RLJU|r!8pNffBxPl@kW=gReBX30{`r42fzvZWe$?4nNiI)5+&-2r&R5@9o^F}1 z3J?Bm;H#d{_L)?sgpMBBg8xs(jqKpAv)+Sc;k2D_&xaewb0vLTU^NcD6dFc}y>#)1Mdwg*r`4h|rV-y6Sf57tJU%C z;9c-8WdiSMkL^q3AR1+oF7u5p7d~=wFQ4T)l>p4gX-N23Qh(MQL2zNVbayZPmXi}2 z2XBOocvWaC_?0dp#?bCS8Ze+xOi$;*w6l{&qw>fQ&=zq$nk9Y(Z%N(XI_^b#>zz+q zyBGJDSnq8YRlpDid_1CpAw{@Av&7G37DtWoLtuvAz~Hf?jsJsWPv_sCPxg|k#(xX~ z3b?h^ug}JIM}IUC+dWuyczu6ZbFK}y*bBK$&A$p!(OIrDDl2o;!Q%Z-_~!Op#c8mp z*r6c`%KB*p7K*q0cBhap6lk^GBiCzN>cKO_IqL9iY+t%VOC{c?#Ud_S2WF*V-=MJF zC;H{~v{}>0D2bEWudza#lv~J0Yk`_$CTfV1kw)siB!4BvdOBY`wsDU&*ra1>+553v zdCbFb?Vab}qrV>~A(opMErP4h0%4^G5xyioQq!ikl@unUuGDrv!jJO+LRU_5POmHY zijxz4Y9ph|2tfU43x*_8(s-jsqhf0qk5s~Bu)%Dp-Nuaq5~)CF6ZI~Gy1F{w(A*rA z2l%oK)PL25@_j`$G-c6w@19iXa!Ghrp*>NX%`@=_-;g(d$g_n<%^M~o&@i0X@H zEf}_dTQ+1CuWC8!EY}(+?=$A@`9{Ua;miUV?f&guNG?Nkbl7T}-%~_-@}Po@vdGFk z{R1rMhhu)}=EF7KpqcQdgJ}_X%`!POEN(n(Q;*E7=dpqW<9}UdN}Rl~ot4+uDOFMkDQ`_0W(@sTiL*%IDYT6_W0 zc>Yg)%!Nm36ZotIukMJmY&z_Aw!lO*@PEx9d-6uW(08$sUU6zI+vR#PKZ@VghZ`8J z1pEZ+OT^9Fxkt@FJHd)*PI)%x3_02bIoOZ|K5cIG;3{1$7623ZXo?UHS2m%mw%jG zp&-jU0l$mO!_CdL)pMtQVQBb*FYJ52m_5=oesdMcio-YjnJ|ENor7d8`C`4N#=vXR zuOG1*Y3`h8e6nFMuHD+k=B`4wm0kGy@vk&$-v%JX0e`y=1mw|GKt4o|VSmGL4|kXZ zpND#e@k>p;Mwv{??dC5RyZ7x$_kRzUeJixQ2OVI31yFcC-78RP%~mE9!hoAbHMXP# z_sEs8a*kjWpZkf1y4~r!x>E;BURZ8>1GxG51`w!kie;6JKBB(e-k!~FY7eB3asYk5DeLQRGJ1Zh+=o!R9gqZozykw<$wLhF4K5eqC4K__p!mE_Q8N*#ub?j(rJL;H(q@4s7nn=j=Y&DlujS;f6-p|Ke!TCxu zYC?qc4(0+aGgU%5fZeqxq$T5vit(RDtND2->+LP)PZv8dRo~Jq^B8pW5lLk^C>*VK z1+}S#T7SiUF%SInr`JwO-o%>U#O0?y>3kiId4GMG7{*ucJNFte$%U#!fhS0NPD?w^ zP*HgAcAocr-Hei#^0;M#B>wK~)Tm6Q&+4;>*$y8SZu>%|5%IU*T)1iQeFq1;LvyFt zM~UYB9U9f|atww-f_qaJLL5a@>2(O<4-G;a1ydxXEByO9Hla7a7Avjk4NsOMqKCFg zAb%B6w-0{$G#Fd?)+Ci_t(Z`u#6nT4ma=@F{RP@UosX0>;5EQ^C9`%4S*4vRUZl5A zi^nlfaCO$zml=;4ju*pd%Hs?b8>jhC`_Z-aJ2!;I3fR^-G4lN^2(}(x$)eecc||s(1;gMu=*L@I&}3I zj>6FH+UP>UQIaPA9^aW5-kp&e&|U{4O@$(Tms_Y7G8mpMM_efmZvH(sbT+(ziYlBC zJeY$W7!RqC3TaDLHeSBv1|u4DZ(3$&&cux4#$jNA&3LOx1->qF7f((W4)R^}GfbA+6{ z`ByM2F8R$T-LW#_&NUFG`YRxmPN=|S^r$uEjC94)Z9No8sT^@iX_jh?i1mQzh$FT* zwj^R#s;ixO0oUB_KTo@N5zLY>X7|KqW;&leOd-8A^+l zRcKT3ml*W)so3{kpiyyd?&G7iRWVhKtDj7y z-W&oi6AGJ9U^2hB<vn3UP5FEeGG@{K2H~w>LL! zA*J>A=~?F&ZjhS(Y6u8x+F>z18vF?|&}VqM22&2(FH}i$g?QdQ6;8bh(I57tM&S+4 zwzr0Mk-6w{#X41Whd(~qF5$l=l_V`>Bbb}JIU&M$$ESobIJ;yXHh+ac0}CVz5nmB1 zS&A)MaXxtjlF%iQU%buy(K%dgs7nj&+MM^Ar%0k;V6YAWUZwf@<3@9!Qka+w#C>kk zvKGD%?g-W`zECxBp%JG|yR{d7C;hzDPa|Nm&Ap4e(28%rqTnA>5BV8}G;$3Sbm$EH zZgVGwtB%D7!5K9lcYoC{sS%!0&&b(XLDuutLN{p@ogDU45=L&*w%v9@LcMaW%|kz^ z<8)5^18*`(Ep2CJw!Xm|r8g*s+!GWR;qaGhv=lQN5_-5^?NogRUB^oli00&M!njtI zhXhX6KZxWc5WLf7Fk_eclYi7YB=fH_o+ze1(FYo%+uMN><9|sxSB}0euZLJTu*OVh zQKQT4CIo{^4S1ss-z3-XP2H)mA@B?@bVe9;f1_!eG|f-+r8-Hx+-V*B?t_%&zx8}l z_@nh(qkWNoYKtb8VXn%`^P7zM{dXUw9n=CdZrSGM)v785i=^2)g#|k(aLQUcLA$0mUOa+$?{gUFtB)XSp$+POUX$E-0^><7`%j_d*P^S%+3K8ym%>(c%o7zz50g*;p(m=RH}5 z9fj`Q`9#AW|M8YjZf)(1T-$}e=j8b3^YpY?19_P|m~Z%@ki_e?5T8RrE+F%FP~OFl zF8)Y7_ppCy`l@(#H$FQTbk2t+{j@@p|5yAg%RPF?$A%$9b?|ylA6U-Cf&Wz3BLf?p zG0EdmMITXfuuOG2dJCqxHTPGGG?Jk!6>=F)$Binr=GrrR%Bx~BMgSE8u0u7AEK zg+$XI12?6Og*mAS8(1(i@Ls6q!q-N~R#A;;=1AHY%|kIHTJV}Jbe;gUz3C{SiqWu^ z?8tu`Q4}lm;&Le%{UtJF1mOW$CjxvS?d|h3=01scVLx&Gh3SANU+L*F70KGid6(I) zs!xC0_h*FD-PiBC+(7J$Qp1+1)!SP6?`FjPdRcA$yQ&tJL3->)QD<)UdIb1On7qIC z_Cs?$6pdpb^ar>S{TIuXUgEnf5ue*Ls%EYB@XfrqG7MA3(n7WtE7s=zK?^j~->Dey zvDhK!et(1GC|2HxwY5!Vy=dSGp-lYJa_N7w)?BCBl_tPb5ZM-kxfP-?LC18`INeW; zzey9bvtQ$|Z>}CY5NM!pon3|vnzH-JE+(cTB?BL61zqx%H0%kbautSh(Ch#5`p4x> zi(YDMxRu_;oy6Vd+-R;{qrj^f(F2tp0rk~VNo!uSAulgkJe3viPC@ozHDE9{4(@-$ zwOH4?LK3wM3IW=Ze36n8Ja>CL%wo+7%r0_xEER?lf0#*?G5OKX~xu1>J-`x6J@YG_{nF_o;Z6G|M67!2hYsAY}8v4+AtTZD(={BqjJ08z} zqBKxEKNoS*l^u2okKo&0kSmr~Q)7RLpiVRzym&j`2-u79x-Kl4$a!g?4CRdqj~;zJ z4}wAMr&`95a&me|t76cgb>rL4M6t!=oBa-prsMCy zWd4TN)b^41`!Ph~^KZEyPli`oa$R$T{md#Rzu%s2{9LjT|B4cvR95!h`Nn_phqeav z3*knC-v}=FWuLbYM*@6wOicS#TWc3aN|q@Lcu&g6wtYHI{$2iK;jzQZ2ZN0A-+l4vllek)JkokNSds^DV=Ws~0Q&0kkNSo}ToN@TA^Hu;nj>(FP4zvK zS|?P$cK$)(<0>ai+dF}=FRy>faUrYUt+f%xhBKP@Hjtu~e~g1ZBL`dK?Qc{s;R4?o zAzdoWNCiDc$Qhohe7GjsF=iqcK10#BSZSSo?&gR(_WI&Af8Ug1D=ymZ-qgIaE8>oQ z^?}NNtoM^eOw1Vn2PGMjzLC^7HAx}M;&~#~46 zm2H@`0Io%yFLX-ze>!1iTE1ii8(Dv$V`11?X@5sPU+&h$SZ3Zxx{TU4&`nngp9<*- z?T&wy4nLoUuSxXbK3{*g!Clje4L?FMs-Oa)2aTBj#;4(2y0t`tLN$Kk`^{&cBp196 zJ#uAOc=Dm~EPWOuW@TU988;yS^0v&(eSqg2>=Cy4FrY3v#|So{MasoxSGjbC^Hx6FvCt zkFTYF%Q3RNnFzRLzW%%Ba{+E7{=Vn=Dt#8CImFyO3kzMY*L$-@8YBk~*F{JOJ<#t} z-zjG-<1Q&)6Zb;ATEmA^gX!M3#|HJp&5iH=-BQEpt+juTU%r~IYJ&Ue)vkHX1|vg| zkECRLP6pyN9uFtp_}DN68Y}CMj~dXe3^?0dn!37kail_9(gQWhuKgwh!$~bE3FZch zH<1&gEQQwD7MxW}4B^r*-;_9+t0ogVUQ8&~3vxck)3g6D*Y3dVaLgZi=g@M1DLgz8 zNJO}>uk(LvijT&$5je%vwutR6S(4Ps+X%kPBqt*md{bw`x6(>;-oB|)1L@O)`0)sq z3YaV_H|0|rVjj145>A*_FKkO>rYp$eA?Dk%6W7OHqBRdM0L zPpSIX20A1$^YDGp+W0Ne5EMs=oQx}4d7<^Z2L^3+^4}BW-#P}#r_Y}B3Uq#8PCN#;>I0> zHZg7QkJ5T?>*@rK)O&lB7y+ybkhV5GomDF#&{zTN>4e&)l!$eH0Jy>T zBnWT$krf|1n5IIvb?&HO+4wxnD<@Mdv6%_Y*k@nAwJ$GZyjT8tCW-qKWWRph1+{!5 zWG?tvtdbvu&=L*A89k;vb_uMdy@L0laQO8E)zghw{8ie_mXuTGOI8;5+~fU8>+pZ~ z$WC3$&LXsQX6o!kjU+`Qm zCcg+O(^6en(~@r*U~gQL(_B!`CI&9Ktu@k`gWK~vb^T*#l z<0}n= zQp&Whs`|#7!unQ(e90>~&>;L*H=gmrN~TAz((F$W<$7fFXgYHr!pqCHGFE>G>DBG} zSbKa5s_*a@IDU62Lk6f9B))y=^IyGp3Yv(b3WrPA_nzuA%e_dWU2oOSWOK=MWx50lkxRYRm4Fm=anm~{cAh-ueaMvL?gy8P>VZnWHhkfi; zZPixo{sUWI|1vf8rsth|`}FP8{T?|%tALg!l$ldk92O_qy2YCLS@@k&FH^_vOv5+h z=AR3l!O6Sd3m>|Taqn26(39oR>f!I(h?kBVQ`O{uBDaej>jwk4pf95rDVFBmA4=LSBK zPPIgE!AikEqc5?orSE%~8HG)r+uog9~!W-XU?-p^EP0Nz4k#A@&?e>4Y2t}Ipw)s;%DXo*d=h@I>yhc}w4b=2ko^2wd`2@^zqdv&Np|Lm_g zIx)c?XPC^6u*T#+3^*UfYHZEJ4=Y~>o6 zhOupXR<^qL>pB6Z9pfr0<_w)Jm)#Ln8ZiHvBzH(NLm3JeA)1bt&l zL%QWt8j*j-L5qZY_*#>+WWIqTk&t3!sY#{ATu-a=-pGGvf6UT@o?7^$nxk~Wiytkz ziz#tfF3pb~SzN-n%Z!77%Vl%Idh3*J(Ju0%(?04X!;_r+hE{nk_% zl}+;PBMe^+&j2mV?FSY1+E{r^-n5)kpU7=-jITJZS1|(am#_V#9-yJi$P)_uF`Q?! z5juT|gBzY0O?YzodigW1>_YaU9P;%tuCz9?<8xd6kAfVH@5599wU8Q|l3H1-8yT(* z(=312=(5eKkr!7E{Ii~x3@PU;qKsx-YXU_TYus1me#6bFS!(+sQ(3dGVeeTP+}ei7xtu z;XgSw<>k{+j-Rs7%$|R4e~Ff-W^&t{<1>GvJt}CjezVD1y`UR+NEm2>*)^Pgzhsl7 zvDC(@*GsJ5h9N^4No29s$n5gbmbF@FxIH=pi8LcoWYkYn>-M2??rY5b?04 z8XqNQdK`wjHys@>UFxyoB5hI|+F4{QT>>yE7wzccwXH!x9X1J+LVo3hu6Lx=3 z|3E(qkXhV_7mP=ZdLDkT6Tz;y$m`#%y}!bDHu*z#M7bN?Dyh|!lZI4|i241^1&tYQ zle^krNdX@;l69eKqZT{KlZ|%CX1XI>mN}eU#CyaCB17HcR@6o2bl}0V^}DmR)s|p+ z*;G-x(mN)uKZ(ah>Y-QKo8yy_i+F$b^x#2@ft%s{U35k%J=x2x{@ZoOpR_}^mtF** zeA^=!mTYue@87ePlIqGGNakbq8qfcDI6n0D=mMK$;lmTE+a8UJa2qvyXP0L#E@QbA zocxyO&koNsUVH3L3-(QwD1Sk>p|w+1YqsHW{HI*vY)^y%LlX_H+2!F+F$sUtYMu)^ zZi)i|1F2g62n4kT<&%@F5CkG9uB%ZNqT_aT!Xw<`se&KiH^DbY1JS>x7`;meo0AKbk%+w{V6ocbV_Gc$`r$(yV>9LL7`;TI+I_2Ch}BsLS}$&vSP{UvqEA|=F2gt zs{Dp5Lpkb`3c)>P8A>%lg`Pe~mAC#ko^9mP4AK|0V`F@ccuuwYX&QeTkFz)>LtiFG zKTkeB!DGAK{F!l}b(MICxW3CcxV`%K0@8ok0 zvT{(Di@4x6CM-Y@5$cHko1Th6p&tGnR8sjHHkUNp%IP zIzeGX8{w!8>&B~-r_z7d-QFL9<5}b7_unp=JO9>2qY!(n4IGE3@Qfrl?Cq`23!7S05@L zU0Nbh=bq*j$}0WgW0=p3S^SXeH-fwue--kUCZ(R2Ojv(%sD(AlEggkOALX5R z=S>A?W!9RK5dD8{n5b}wNj?Am3jz)$Mx51^sts$4lbnf>3{4^R*AHQTcy?fRbd;mK z5;>SuLA4BjgrT<1ym}aXv7Fq!-%}^$DZ8>%9v#In`xulzM>u6iU?{Czq|rX9lMwwd za*F}R>X#dLZC;vv5*9Tusdx2H7`BcnGyKmj!g|naj6r|Fft+Zwo^j4hQRon(oTF|j zQOWme3M&*rQmh+?t+S1n5I!|>;dD_i`{7~rEX6^hp^F7E9pEaFtSqO27j z?sdqTvf+PptrR@zh9imf%Aw;ASlMA;G^cjW0#z0S3P>Y}OGq!)81L>5y~)`_>$##| z5P>ajXH|TlLe!o`xkU`?26n;8HICV0ewT5`2s!5yFJ(p?#^@IslC9BW+AL2D7ssFV zO1hWz;NxJucA4#zGmx+$5;3xnN!{&c1%vs_CqRGQ{e{5M1p;0AsSnBHK_`$DWSn2u zjW1(xoNA9@MK_f?B|FLyMAD7FPpuD9V5Rx_$k?H8fj+^;v4npZt=6o{w?b5rsoS-M z`}6f&FT_n78y?M|#v~>2`StW98dp}95;Qok@Thzj{c$rO=nWl66S8X%ccC{oAH3OI zu7H2{AYfJJZd;>=SO3%z-}NP`=lfk%7r*U_Ouzo~cY&nA-4{j+Nc#Hv!mO-L!?wm) z@PSQX9-&L~5DGpyD~HAJ%im)DZV20rw_co_uth2-c2Z*k)77@Y;jv@iV(ae)XL05| zMBwXZ)rTvAgr&wpJodL&KP|*WRE-T3KkSF94z*Y~crA0yvNV>VXS=ripDap#p#u$I^ zlg`({F;`+@dI^TUiyge*9=~rQ=&w_W0n<#h0uD`Bdl*=a0Qg2b(t=GN zO#uK;o)~{lOY_kvIC^(;5vG29LY%nJc$uyKljPIG$YXUioD4?fQ%-~<>vMyEQJgin zd+oWq)qD?Z^Ell;Y}sG%wOKvgnG*R*b1-c?UEYSg+VMpJmiFhb-(McRQcx5!n5r%N z<+spC(NbpS@1yPc>)bq-=a+x4w{?Foe$QBg$HAAzrp1}t%Z(NBSJj8XWKux(AHjRsP0=fZp~VaZh1UnsX+B#k zyStQ#jluaXOPE;Z{$4GUS83^F*g@0zkcN$cF30k6MbYs1Nylw>_cVVUA5_hEpwB+I z^KvsgbBUGRthD%GUt;=K^j;$@d=TjvjKE>Iq zbtAc|e8#9~wQcPo@jMd~I!(GIM>|0%iE889E?9UsD7<_+I!-OHrt8(715z%O3mbHG zf#zm{yvedfepqkZ`Cosvv~HIqP(-nCah<946BAbuks-<)Pi#`dZEf??iG?RD`*ZTl zoWF+mMPTC(5M5t8>RJIL6Jw8**$h!MVsXE6)1PZ|No_>8J>UJ?;k__2QYe(j zMc(3zeCtPUH1`o%=6~9-yZL&xHBIOM<0IMI@v4ouMoyFZo9lm~vgS<(D71Zt&(2K< z0{J5ifw2FoAM`n!wePH{XXUYZRB?)&cGE85EK*Rq>KN^_$6z<$O*{|L(&CdUz$^7| zja!iLy?SHxk2}v7Y2pmxXDM_w)$IP0IpNsL2lo^w2aAbQ(C{z&GLo%P@9%DCbS5Sy zEQp2UF#(8V>qvijW~RkhaS7(Q-`(XeF4sC~f70v@U|jbPi;Jw$ z-=Pi;wmS-m5Q;D+M&V#k^JyF`IA)1`a6QJxzeSMD6^1BkYVj@4Ir*a-&1I=|ML7xE zZbE&^R`3c42=cS-6wb)gMD$|gpC(~(b)kuB znqWUHpB#U^vVvM>RTqkG#L;qP1IR$1o5ROIKy*aWM*%KtYZ5_^9czT;!9odDGV^oJ z1{A=0z)%|sgI#YNZ%}P$4O=siwq~t zsR1PD9Jw=v_95uA*CPeSCPz_kU7p|bw}FukH0-SPn77yG7B|;BW#dSDMdXsUd<)MI@Bgku|F&%Gm|KokY={u5dg@^Ln6(c4-SI`zAE?jWkkjIWH*2A zCiJ8MfMibZ*iEWkZR#B47kpzuNbpBj{aw@MyFDi47%KTU`XIVGib&mQHg8 zT3u?!t>Jr|-~!r12p=W)6VucGuvvdxZ1G=Av>N5fuSO%1{rYtmCgQ$39L;uR0Tq0V zgM)*TxlB|EF2u#H<)=yd#Yq4Na=TSUaf_yq+luc{={ z=W}wl_vPfyg0l83qPJGML@FIcdfYcwSEZ$-vaFTmpn{U^*=WEX@P|I#&@X>XPc2jm zOhPnC^`r1rGzC$TlvKeRJw{$kt}+8vPHC!dV6B>-jVyofjO|#JcgG-iv%_O@5@IKU zFk*&}I!Gn$cGAbziMcsr3si?y|J@SW366fl;nXnyPi}yr2}WtD7Qe;EK}R@zxq#AN z&=!;Snb!rm6BYb~D1_I&7qY;O!xbh}z#U2x~%gUao2r&baialdrW~!(( z@nJNv5LqUE5%jDoll*xA7|V@#JzSubXJ`}X@)wd@5Ehn?eb1LTKMs2lJHPjva}O9d zKB5-;V6Lg{N&{vaZp%#!R(mi$N_0Nm$rqUtVEE~+o!^;MEoyg9Pl$j0)Lnn7gc++~ z^+%OT_dP-uxXaP?$}t!7+~;W%jrXJgK*r&6H9UOA6INDs?^^NeJcvno-F-41L^3nu z!$93j}8_Bt=TX2*GRsSnc!(C&NqI6C6RgQm^l@-Crl#T>5qb&OT zY*lKCC?Q9pJcnSgwUq`;vUKPByZhObw~)>8i0@)~Y4~_I%Mz3V7ij5w(z|M^1AjV_ zz42g_iH)8go39VMOhl}BuOm5Xj7m5$b9jF>%Y+a#;KTHc5?d@@VsLFu?giE`GnlY=6~y7R>8FH9A;=jpq1-6;juX^ zWnh?6Jvr=t7~SkEA$mD!Bts?OfOHyh(=EmdsDg5(`5rNjOb!dwxY;g#;!&`*9i2@{ zNx_x4eUf|n3e*VtwT^HTH@BLNK0qA6FQR>TjgB?G?bLshEjI!k;|{R4&Nbk|y#VzY zxrFOLzqU?JuC<+S#Q3;MH~2-&_ew=Wex{`ej6}Go@0G&`&C#$zQZ`h{e7-{M%ZEDm z0S>Q%1gX`WR%I}2f>;sY;_E7YI77gn)ADKQ=osG&PYfflh=bJPwe|G8 zOZABXP>z3mlAs|b#@(5OmGbfN8H5cNoNba5f@^UOjgD^0mXF$Ze)Xz`;X(Cu*Kp9q zMklAFV6@s<(s7XZ{8gC1YxVChu~A<@g_H|(;9*di7)(!E5fv9nO;|CVD5Y< zlaYT&>Wa^_2QW&4Cn6JlbRD$TasA*8Te>X$F$>5WzAxc4``$AMV^LTJ3)wlfnSsHY zv0>qUq?w?AK-eIT+amvkGSzA-Sxpw?l_Vy2px)Mgr{bUu3bAwDs zq&$t-hsI-_BnLb?@YLYk8|B1^J(JmVw@rU^=SiwY(`cwxZg{G4N3*dXkq}U7t^oi5 zldG)0tK~aaD`5*4EARv0<>uw*;O61r<%*(^Z%`ME${n{z#@P9bKoGfjv zz5Z_pMP4dZ(BZ{@JGk1ytl+NiVDJAIAKw4+;TmEl1bs*zeBN7H3+r0Bz-^rz0SzmN t90w2QD=h)THt;f_D60l3m45pP_}Bimf9+rU*ZzOoe*i1-zWe}k5CAocWugE8 diff --git a/doc/pub/week42/ipynb/week42.ipynb b/doc/pub/week42/ipynb/week42.ipynb index 209f12073..581ead8a7 100644 --- a/doc/pub/week42/ipynb/week42.ipynb +++ b/doc/pub/week42/ipynb/week42.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "71143de3", + "id": "092c6813", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "35db3d31", + "id": "f44f930c", "metadata": { "editable": true }, @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "3a41a07e", + "id": "52ac4c14", "metadata": { "editable": true }, @@ -56,7 +56,7 @@ }, { "cell_type": "markdown", - "id": "e4274067", + "id": "2c1630dd", "metadata": { "editable": true }, @@ -73,7 +73,7 @@ }, { "cell_type": "markdown", - "id": "38ccf7d0", + "id": "49d3cbcf", "metadata": { "editable": true }, @@ -92,7 +92,7 @@ }, { "cell_type": "markdown", - "id": "bf1932a3", + "id": "9b42371e", "metadata": { "editable": true }, @@ -108,7 +108,7 @@ }, { "cell_type": "markdown", - "id": "1cb1eb45", + "id": "b8f7a026", "metadata": { "editable": true }, @@ -119,7 +119,7 @@ }, { "cell_type": "markdown", - "id": "0f07a5df", + "id": "012dd6a1", "metadata": { "editable": true }, @@ -133,7 +133,7 @@ }, { "cell_type": "markdown", - "id": "8671fb13", + "id": "2847e093", "metadata": { "editable": true }, @@ -148,7 +148,7 @@ }, { "cell_type": "markdown", - "id": "12ed4396", + "id": "45fef288", "metadata": { "editable": true }, @@ -160,7 +160,7 @@ }, { "cell_type": "markdown", - "id": "c1516595", + "id": "de152393", "metadata": { "editable": true }, @@ -174,7 +174,7 @@ }, { "cell_type": "markdown", - "id": "2949b61a", + "id": "621fb2b9", "metadata": { "editable": true }, @@ -186,7 +186,7 @@ }, { "cell_type": "markdown", - "id": "46e01f35", + "id": "c418fb3c", "metadata": { "editable": true }, @@ -202,7 +202,7 @@ }, { "cell_type": "markdown", - "id": "9c6ea29d", + "id": "800aecac", "metadata": { "editable": true }, @@ -218,7 +218,7 @@ }, { "cell_type": "markdown", - "id": "6c66c7ee", + "id": "2e973cf8", "metadata": { "editable": true }, @@ -230,7 +230,7 @@ }, { "cell_type": "markdown", - "id": "f8846711", + "id": "95e54e97", "metadata": { "editable": true }, @@ -240,7 +240,7 @@ }, { "cell_type": "markdown", - "id": "e4b9e408", + "id": "6fd99322", "metadata": { "editable": true }, @@ -252,7 +252,7 @@ }, { "cell_type": "markdown", - "id": "b8f62859", + "id": "2b2ebf34", "metadata": { "editable": true }, @@ -262,7 +262,7 @@ }, { "cell_type": "markdown", - "id": "f2ec677d", + "id": "d488d19a", "metadata": { "editable": true }, @@ -274,7 +274,7 @@ }, { "cell_type": "markdown", - "id": "31099b72", + "id": "af6e84a4", "metadata": { "editable": true }, @@ -284,7 +284,7 @@ }, { "cell_type": "markdown", - "id": "cc6de265", + "id": "c9407fa7", "metadata": { "editable": true }, @@ -296,7 +296,7 @@ }, { "cell_type": "markdown", - "id": "de05db8e", + "id": "b33546c2", "metadata": { "editable": true }, @@ -312,7 +312,7 @@ }, { "cell_type": "markdown", - "id": "a5fcef62", + "id": "e2b7759f", "metadata": { "editable": true }, @@ -324,7 +324,7 @@ }, { "cell_type": "markdown", - "id": "38f87147", + "id": "5e1062d7", "metadata": { "editable": true }, @@ -336,7 +336,7 @@ }, { "cell_type": "markdown", - "id": "92822d7d", + "id": "2733bbc4", "metadata": { "editable": true }, @@ -346,7 +346,7 @@ }, { "cell_type": "markdown", - "id": "7d08d013", + "id": "b72c5008", "metadata": { "editable": true }, @@ -358,7 +358,7 @@ }, { "cell_type": "markdown", - "id": "8ca04a0a", + "id": "b219dc77", "metadata": { "editable": true }, @@ -368,7 +368,7 @@ }, { "cell_type": "markdown", - "id": "cd2b820f", + "id": "5313f959", "metadata": { "editable": true }, @@ -384,7 +384,7 @@ }, { "cell_type": "markdown", - "id": "74a4bf55", + "id": "fe80b0ed", "metadata": { "editable": true }, @@ -396,7 +396,7 @@ }, { "cell_type": "markdown", - "id": "b6a6ea2c", + "id": "fdd61c89", "metadata": { "editable": true }, @@ -408,7 +408,7 @@ }, { "cell_type": "markdown", - "id": "f569e2c4", + "id": "e73defca", "metadata": { "editable": true }, @@ -420,7 +420,7 @@ }, { "cell_type": "markdown", - "id": "11dd5f28", + "id": "6a10c098", "metadata": { "editable": true }, @@ -432,7 +432,7 @@ }, { "cell_type": "markdown", - "id": "e79bd0c6", + "id": "5e45bea0", "metadata": { "editable": true }, @@ -444,7 +444,7 @@ }, { "cell_type": "markdown", - "id": "0994dc8a", + "id": "345aecc6", "metadata": { "editable": true }, @@ -454,7 +454,7 @@ }, { "cell_type": "markdown", - "id": "a5fb5850", + "id": "65142a0a", "metadata": { "editable": true }, @@ -470,7 +470,7 @@ }, { "cell_type": "markdown", - "id": "75ef56f2", + "id": "5faab1bb", "metadata": { "editable": true }, @@ -482,7 +482,7 @@ }, { "cell_type": "markdown", - "id": "2cba4793", + "id": "476fcd72", "metadata": { "editable": true }, @@ -494,7 +494,7 @@ }, { "cell_type": "markdown", - "id": "002f872b", + "id": "8c8120ee", "metadata": { "editable": true }, @@ -504,7 +504,7 @@ }, { "cell_type": "markdown", - "id": "ca9db79a", + "id": "5f3741cc", "metadata": { "editable": true }, @@ -516,7 +516,7 @@ }, { "cell_type": "markdown", - "id": "91c832ea", + "id": "129deee5", "metadata": { "editable": true }, @@ -530,7 +530,7 @@ }, { "cell_type": "markdown", - "id": "497597b5", + "id": "9991265c", "metadata": { "editable": true }, @@ -548,7 +548,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "31785b91", + "id": "d70cb287", "metadata": { "collapsed": false, "editable": true @@ -621,7 +621,7 @@ }, { "cell_type": "markdown", - "id": "fb3ba38a", + "id": "0c995e02", "metadata": { "editable": true }, @@ -631,7 +631,7 @@ }, { "cell_type": "markdown", - "id": "f39aa22a", + "id": "22b79621", "metadata": { "editable": true }, @@ -648,7 +648,7 @@ }, { "cell_type": "markdown", - "id": "07786f01", + "id": "554f6b51", "metadata": { "editable": true }, @@ -660,7 +660,7 @@ }, { "cell_type": "markdown", - "id": "855ae860", + "id": "5f747812", "metadata": { "editable": true }, @@ -670,7 +670,7 @@ }, { "cell_type": "markdown", - "id": "f982b39a", + "id": "6f9a465a", "metadata": { "editable": true }, @@ -682,7 +682,7 @@ }, { "cell_type": "markdown", - "id": "5b0b9a6a", + "id": "8071b978", "metadata": { "editable": true }, @@ -698,7 +698,7 @@ }, { "cell_type": "markdown", - "id": "f74b7c7d", + "id": "bf0242dd", "metadata": { "editable": true }, @@ -710,7 +710,7 @@ }, { "cell_type": "markdown", - "id": "dbaa42b5", + "id": "5ffe91c1", "metadata": { "editable": true }, @@ -722,7 +722,7 @@ }, { "cell_type": "markdown", - "id": "8f7dc89c", + "id": "f5512cb1", "metadata": { "editable": true }, @@ -733,7 +733,7 @@ }, { "cell_type": "markdown", - "id": "adfec9e2", + "id": "fa7c53f0", "metadata": { "editable": true }, @@ -745,7 +745,7 @@ }, { "cell_type": "markdown", - "id": "1a91248a", + "id": "b500870f", "metadata": { "editable": true }, @@ -758,7 +758,7 @@ }, { "cell_type": "markdown", - "id": "f8dd5dde", + "id": "79cffbd0", "metadata": { "editable": true }, @@ -770,7 +770,7 @@ }, { "cell_type": "markdown", - "id": "806e1970", + "id": "75bca176", "metadata": { "editable": true }, @@ -780,7 +780,7 @@ }, { "cell_type": "markdown", - "id": "887c1cc5", + "id": "e6aa2766", "metadata": { "editable": true }, @@ -792,7 +792,7 @@ }, { "cell_type": "markdown", - "id": "4bf40527", + "id": "47f01305", "metadata": { "editable": true }, @@ -804,7 +804,7 @@ }, { "cell_type": "markdown", - "id": "a5d878ba", + "id": "f242e593", "metadata": { "editable": true }, @@ -816,7 +816,7 @@ }, { "cell_type": "markdown", - "id": "318c3664", + "id": "54f82c8e", "metadata": { "editable": true }, @@ -826,7 +826,7 @@ }, { "cell_type": "markdown", - "id": "c4fc9951", + "id": "04835564", "metadata": { "editable": true }, @@ -838,7 +838,7 @@ }, { "cell_type": "markdown", - "id": "3d187d65", + "id": "b364a288", "metadata": { "editable": true }, @@ -854,7 +854,7 @@ }, { "cell_type": "markdown", - "id": "24ebe80e", + "id": "7ce5168e", "metadata": { "editable": true }, @@ -866,7 +866,7 @@ }, { "cell_type": "markdown", - "id": "640153c7", + "id": "7d7d5872", "metadata": { "editable": true }, @@ -876,7 +876,7 @@ }, { "cell_type": "markdown", - "id": "9f00a103", + "id": "c0e39147", "metadata": { "editable": true }, @@ -888,7 +888,7 @@ }, { "cell_type": "markdown", - "id": "c313a954", + "id": "b086049b", "metadata": { "editable": true }, @@ -898,7 +898,7 @@ }, { "cell_type": "markdown", - "id": "8f9899f9", + "id": "094989b6", "metadata": { "editable": true }, @@ -910,7 +910,7 @@ }, { "cell_type": "markdown", - "id": "fe894efa", + "id": "2742317a", "metadata": { "editable": true }, @@ -922,7 +922,7 @@ }, { "cell_type": "markdown", - "id": "ff3f394f", + "id": "0e2c957d", "metadata": { "editable": true }, @@ -935,7 +935,7 @@ }, { "cell_type": "markdown", - "id": "91783800", + "id": "d81132fa", "metadata": { "editable": true }, @@ -945,7 +945,7 @@ }, { "cell_type": "markdown", - "id": "b9199d44", + "id": "9c779d7e", "metadata": { "editable": true }, @@ -957,7 +957,7 @@ }, { "cell_type": "markdown", - "id": "0cc20449", + "id": "d1318243", "metadata": { "editable": true }, @@ -967,7 +967,7 @@ }, { "cell_type": "markdown", - "id": "a376f16a", + "id": "4f9977f9", "metadata": { "editable": true }, @@ -979,7 +979,7 @@ }, { "cell_type": "markdown", - "id": "9cf47847", + "id": "fdc2aba9", "metadata": { "editable": true }, @@ -990,7 +990,7 @@ }, { "cell_type": "markdown", - "id": "c1bd9901", + "id": "36334b10", "metadata": { "editable": true }, @@ -1002,7 +1002,7 @@ }, { "cell_type": "markdown", - "id": "addef0ec", + "id": "d7c74c36", "metadata": { "editable": true }, @@ -1012,7 +1012,7 @@ }, { "cell_type": "markdown", - "id": "a5b267fc", + "id": "9258fbf2", "metadata": { "editable": true }, @@ -1024,7 +1024,7 @@ }, { "cell_type": "markdown", - "id": "53d64a33", + "id": "4703fbac", "metadata": { "editable": true }, @@ -1034,7 +1034,7 @@ }, { "cell_type": "markdown", - "id": "765d3576", + "id": "0b451589", "metadata": { "editable": true }, @@ -1046,7 +1046,7 @@ }, { "cell_type": "markdown", - "id": "680e6d32", + "id": "75530820", "metadata": { "editable": true }, @@ -1058,7 +1058,7 @@ }, { "cell_type": "markdown", - "id": "6b5a3f94", + "id": "1103fe71", "metadata": { "editable": true }, @@ -1070,7 +1070,7 @@ }, { "cell_type": "markdown", - "id": "c0b94fbc", + "id": "f2ea48ee", "metadata": { "editable": true }, @@ -1080,7 +1080,7 @@ }, { "cell_type": "markdown", - "id": "a8bc0cb4", + "id": "a508fd86", "metadata": { "editable": true }, @@ -1092,7 +1092,7 @@ }, { "cell_type": "markdown", - "id": "fc5ba87a", + "id": "fc4448cf", "metadata": { "editable": true }, @@ -1102,7 +1102,7 @@ }, { "cell_type": "markdown", - "id": "d7928417", + "id": "7585a56f", "metadata": { "editable": true }, @@ -1114,7 +1114,7 @@ }, { "cell_type": "markdown", - "id": "38fcd712", + "id": "b2b6e1f2", "metadata": { "editable": true }, @@ -1126,7 +1126,7 @@ }, { "cell_type": "markdown", - "id": "a68a28b5", + "id": "2e3f4fd2", "metadata": { "editable": true }, @@ -1138,7 +1138,7 @@ }, { "cell_type": "markdown", - "id": "81b5762d", + "id": "7506d23b", "metadata": { "editable": true }, @@ -1148,7 +1148,7 @@ }, { "cell_type": "markdown", - "id": "6485ea35", + "id": "c53a7ad0", "metadata": { "editable": true }, @@ -1160,7 +1160,7 @@ }, { "cell_type": "markdown", - "id": "56f15a69", + "id": "e741bcd1", "metadata": { "editable": true }, @@ -1170,7 +1170,7 @@ }, { "cell_type": "markdown", - "id": "7f3597fa", + "id": "3f40c8b4", "metadata": { "editable": true }, @@ -1183,7 +1183,7 @@ }, { "cell_type": "markdown", - "id": "f3af219f", + "id": "38133e07", "metadata": { "editable": true }, @@ -1195,7 +1195,7 @@ }, { "cell_type": "markdown", - "id": "7a21d5ea", + "id": "a48e0d24", "metadata": { "editable": true }, @@ -1205,7 +1205,7 @@ }, { "cell_type": "markdown", - "id": "d0c3898f", + "id": "c2527d7c", "metadata": { "editable": true }, @@ -1217,7 +1217,7 @@ }, { "cell_type": "markdown", - "id": "34325b5c", + "id": "5b70545f", "metadata": { "editable": true }, @@ -1227,7 +1227,7 @@ }, { "cell_type": "markdown", - "id": "918bb1fa", + "id": "02d7a474", "metadata": { "editable": true }, @@ -1239,7 +1239,7 @@ }, { "cell_type": "markdown", - "id": "894b36e6", + "id": "28f26a48", "metadata": { "editable": true }, @@ -1249,7 +1249,7 @@ }, { "cell_type": "markdown", - "id": "55f20c6b", + "id": "7cc8726f", "metadata": { "editable": true }, @@ -1261,7 +1261,7 @@ }, { "cell_type": "markdown", - "id": "0d783805", + "id": "123de6da", "metadata": { "editable": true }, @@ -1271,7 +1271,7 @@ }, { "cell_type": "markdown", - "id": "f64324de", + "id": "518455da", "metadata": { "editable": true }, @@ -1288,7 +1288,7 @@ }, { "cell_type": "markdown", - "id": "c860274d", + "id": "c6591a2b", "metadata": { "editable": true }, @@ -1300,7 +1300,7 @@ }, { "cell_type": "markdown", - "id": "266ed951", + "id": "e8b6fcf1", "metadata": { "editable": true }, @@ -1312,23 +1312,23 @@ }, { "cell_type": "markdown", - "id": "e3597760", + "id": "ced47734", "metadata": { "editable": true }, "source": [ - "## Layout of a neural network with three hidden layers\n", + "## Layout of a neural network with three hidden layers (last later = $l=L=4$, first layer $l=0$)\n", "\n", - "\n", + "\n", "\n", "\n", - "

    Figure 1:

    \n", + "

    Figure 1:

    \n", "" ] }, { "cell_type": "markdown", - "id": "29ddf471", + "id": "1c43dce0", "metadata": { "editable": true }, @@ -1345,7 +1345,7 @@ }, { "cell_type": "markdown", - "id": "7e290342", + "id": "0e633d9c", "metadata": { "editable": true }, @@ -1357,7 +1357,7 @@ }, { "cell_type": "markdown", - "id": "c16f6b07", + "id": "9b67f305", "metadata": { "editable": true }, @@ -1370,7 +1370,7 @@ }, { "cell_type": "markdown", - "id": "67289b7d", + "id": "d2f1a218", "metadata": { "editable": true }, @@ -1382,7 +1382,7 @@ }, { "cell_type": "markdown", - "id": "035f7467", + "id": "9bb82aca", "metadata": { "editable": true }, @@ -1398,7 +1398,7 @@ }, { "cell_type": "markdown", - "id": "0c867ff6", + "id": "6b8654b2", "metadata": { "editable": true }, @@ -1410,7 +1410,23 @@ }, { "cell_type": "markdown", - "id": "502f4ffa", + "id": "0eb3c9d3", + "metadata": { + "editable": true + }, + "source": [ + "## Layout of input to first hidden layer $l=1$ from input layer $l=0$\n", + "\n", + "\n", + "\n", + "\n", + "

    Figure 1:

    \n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "3d798d71", "metadata": { "editable": true }, @@ -1422,7 +1438,7 @@ }, { "cell_type": "markdown", - "id": "6ccb4bd2", + "id": "7ddc0fc9", "metadata": { "editable": true }, @@ -1434,7 +1450,7 @@ }, { "cell_type": "markdown", - "id": "86661f7f", + "id": "0d8fe9fb", "metadata": { "editable": true }, @@ -1444,7 +1460,7 @@ }, { "cell_type": "markdown", - "id": "b7638311", + "id": "edd7787f", "metadata": { "editable": true }, @@ -1456,7 +1472,7 @@ }, { "cell_type": "markdown", - "id": "45f9847a", + "id": "80f47db9", "metadata": { "editable": true }, @@ -1466,7 +1482,7 @@ }, { "cell_type": "markdown", - "id": "48754bd5", + "id": "a48bab67", "metadata": { "editable": true }, @@ -1478,7 +1494,7 @@ }, { "cell_type": "markdown", - "id": "5c7a69fd", + "id": "94648f68", "metadata": { "editable": true }, @@ -1492,7 +1508,7 @@ }, { "cell_type": "markdown", - "id": "be7536a6", + "id": "140920e5", "metadata": { "editable": true }, @@ -1504,7 +1520,7 @@ }, { "cell_type": "markdown", - "id": "bca9c851", + "id": "dacb70a5", "metadata": { "editable": true }, @@ -1514,7 +1530,7 @@ }, { "cell_type": "markdown", - "id": "86fbee3f", + "id": "ff05f9a6", "metadata": { "editable": true }, @@ -1526,7 +1542,7 @@ }, { "cell_type": "markdown", - "id": "11eaeef8", + "id": "99c45bc0", "metadata": { "editable": true }, @@ -1536,7 +1552,7 @@ }, { "cell_type": "markdown", - "id": "80401b16", + "id": "41ef7064", "metadata": { "editable": true }, @@ -1548,7 +1564,7 @@ }, { "cell_type": "markdown", - "id": "1072ac20", + "id": "f13df94c", "metadata": { "editable": true }, @@ -1560,7 +1576,7 @@ }, { "cell_type": "markdown", - "id": "89ffde64", + "id": "20d5f27f", "metadata": { "editable": true }, @@ -1572,7 +1588,7 @@ }, { "cell_type": "markdown", - "id": "325836cb", + "id": "f68c72b6", "metadata": { "editable": true }, @@ -1582,7 +1598,7 @@ }, { "cell_type": "markdown", - "id": "a30436a3", + "id": "5ad4d7ae", "metadata": { "editable": true }, @@ -1594,7 +1610,7 @@ }, { "cell_type": "markdown", - "id": "641e2b0a", + "id": "623c32d7", "metadata": { "editable": true }, @@ -1604,7 +1620,7 @@ }, { "cell_type": "markdown", - "id": "14f11611", + "id": "9dc972b4", "metadata": { "editable": true }, @@ -1616,7 +1632,7 @@ }, { "cell_type": "markdown", - "id": "49989080", + "id": "af1d6e91", "metadata": { "editable": true }, @@ -1634,7 +1650,7 @@ }, { "cell_type": "markdown", - "id": "f5a2ec44", + "id": "51defa9d", "metadata": { "editable": true }, @@ -1652,7 +1668,7 @@ }, { "cell_type": "markdown", - "id": "deb975c0", + "id": "fff66b0b", "metadata": { "editable": true }, @@ -1664,7 +1680,7 @@ }, { "cell_type": "markdown", - "id": "afc51f7d", + "id": "e646d1f7", "metadata": { "editable": true }, @@ -1674,7 +1690,7 @@ }, { "cell_type": "markdown", - "id": "429f602a", + "id": "b448f6fa", "metadata": { "editable": true }, @@ -1686,7 +1702,7 @@ }, { "cell_type": "markdown", - "id": "e2edd78f", + "id": "bb477975", "metadata": { "editable": true }, @@ -1698,7 +1714,7 @@ }, { "cell_type": "markdown", - "id": "63f7032b", + "id": "00fb4848", "metadata": { "editable": true }, @@ -1710,7 +1726,7 @@ }, { "cell_type": "markdown", - "id": "f02b1a1b", + "id": "fd6ff6f1", "metadata": { "editable": true }, @@ -1720,7 +1736,7 @@ }, { "cell_type": "markdown", - "id": "0574e916", + "id": "8cb4e9d5", "metadata": { "editable": true }, @@ -1732,7 +1748,7 @@ }, { "cell_type": "markdown", - "id": "41bdc15f", + "id": "c9051e99", "metadata": { "editable": true }, @@ -1742,7 +1758,7 @@ }, { "cell_type": "markdown", - "id": "09cc68f7", + "id": "e0675944", "metadata": { "editable": true }, @@ -1754,7 +1770,7 @@ }, { "cell_type": "markdown", - "id": "b40293f4", + "id": "657a96be", "metadata": { "editable": true }, @@ -1772,7 +1788,7 @@ }, { "cell_type": "markdown", - "id": "c3d18ebb", + "id": "0f46875e", "metadata": { "editable": true }, @@ -1782,7 +1798,7 @@ }, { "cell_type": "markdown", - "id": "38775abe", + "id": "88d3dec1", "metadata": { "editable": true }, @@ -1800,7 +1816,7 @@ }, { "cell_type": "markdown", - "id": "5ec01530", + "id": "49bdd416", "metadata": { "editable": true }, @@ -1810,7 +1826,7 @@ }, { "cell_type": "markdown", - "id": "b86f7ca4", + "id": "af9834e8", "metadata": { "editable": true }, @@ -1828,7 +1844,7 @@ }, { "cell_type": "markdown", - "id": "fd42f1c0", + "id": "5b0fecb5", "metadata": { "editable": true }, @@ -1840,7 +1856,7 @@ }, { "cell_type": "markdown", - "id": "9904c840", + "id": "38eac0eb", "metadata": { "editable": true }, @@ -1852,7 +1868,7 @@ }, { "cell_type": "markdown", - "id": "40481edd", + "id": "bfd769fa", "metadata": { "editable": true }, @@ -1862,7 +1878,7 @@ }, { "cell_type": "markdown", - "id": "92d8689b", + "id": "135bd5de", "metadata": { "editable": true }, @@ -1874,7 +1890,7 @@ }, { "cell_type": "markdown", - "id": "22ec4130", + "id": "fd8d7e06", "metadata": { "editable": true }, @@ -1886,7 +1902,7 @@ }, { "cell_type": "markdown", - "id": "8491df20", + "id": "6d077330", "metadata": { "editable": true }, @@ -1896,7 +1912,7 @@ }, { "cell_type": "markdown", - "id": "d4959520", + "id": "0b1deaaf", "metadata": { "editable": true }, @@ -1908,7 +1924,7 @@ }, { "cell_type": "markdown", - "id": "0feb4648", + "id": "986e3cad", "metadata": { "editable": true }, @@ -1918,7 +1934,7 @@ }, { "cell_type": "markdown", - "id": "bd8747a2", + "id": "d35bcc04", "metadata": { "editable": true }, @@ -1930,7 +1946,7 @@ }, { "cell_type": "markdown", - "id": "b13a8b94", + "id": "7f486441", "metadata": { "editable": true }, @@ -1942,7 +1958,7 @@ }, { "cell_type": "markdown", - "id": "5e765c85", + "id": "39d6fda4", "metadata": { "editable": true }, @@ -1965,7 +1981,7 @@ }, { "cell_type": "markdown", - "id": "849bcaca", + "id": "938d24c7", "metadata": { "editable": true }, @@ -1977,7 +1993,7 @@ }, { "cell_type": "markdown", - "id": "12d2661b", + "id": "278750ac", "metadata": { "editable": true }, @@ -1989,7 +2005,7 @@ }, { "cell_type": "markdown", - "id": "c7ebdf39", + "id": "3e126986", "metadata": { "editable": true }, @@ -1999,7 +2015,7 @@ }, { "cell_type": "markdown", - "id": "be4a1d22", + "id": "8f048a4b", "metadata": { "editable": true }, @@ -2011,7 +2027,7 @@ }, { "cell_type": "markdown", - "id": "1f93b3f8", + "id": "4886d206", "metadata": { "editable": true }, @@ -2025,7 +2041,7 @@ }, { "cell_type": "markdown", - "id": "3dd41f12", + "id": "f7f67a4e", "metadata": { "editable": true }, @@ -2037,7 +2053,7 @@ }, { "cell_type": "markdown", - "id": "0e84ce0b", + "id": "f7b054be", "metadata": { "editable": true }, @@ -2049,7 +2065,7 @@ }, { "cell_type": "markdown", - "id": "a9bfd977", + "id": "d8669d04", "metadata": { "editable": true }, @@ -2059,7 +2075,7 @@ }, { "cell_type": "markdown", - "id": "19d2fec5", + "id": "37b39be1", "metadata": { "editable": true }, @@ -2071,7 +2087,7 @@ }, { "cell_type": "markdown", - "id": "0ed61a3b", + "id": "e7cca043", "metadata": { "editable": true }, @@ -2083,7 +2099,7 @@ }, { "cell_type": "markdown", - "id": "0c42324f", + "id": "1bc0321b", "metadata": { "editable": true }, @@ -2093,7 +2109,7 @@ }, { "cell_type": "markdown", - "id": "5509053e", + "id": "e7be68c1", "metadata": { "editable": true }, @@ -2105,7 +2121,7 @@ }, { "cell_type": "markdown", - "id": "8c989c9e", + "id": "f0b0b4e6", "metadata": { "editable": true }, @@ -2117,7 +2133,7 @@ }, { "cell_type": "markdown", - "id": "e49cc8bb", + "id": "5a63acb6", "metadata": { "editable": true }, @@ -2140,7 +2156,7 @@ }, { "cell_type": "markdown", - "id": "c950cb42", + "id": "99f26331", "metadata": { "editable": true }, @@ -2159,7 +2175,7 @@ }, { "cell_type": "markdown", - "id": "b3f1cb1d", + "id": "e0f8236b", "metadata": { "editable": true }, @@ -2171,7 +2187,7 @@ }, { "cell_type": "markdown", - "id": "b09b88af", + "id": "9710090c", "metadata": { "editable": true }, @@ -2181,7 +2197,7 @@ }, { "cell_type": "markdown", - "id": "0580595f", + "id": "6d4a3d43", "metadata": { "editable": true }, @@ -2193,7 +2209,7 @@ }, { "cell_type": "markdown", - "id": "c59b68cf", + "id": "eb7622ca", "metadata": { "editable": true }, @@ -2210,7 +2226,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "c1d4f091", + "id": "baf78bf5", "metadata": { "collapsed": false, "editable": true @@ -2294,7 +2310,7 @@ }, { "cell_type": "markdown", - "id": "ab302230", + "id": "53903197", "metadata": { "editable": true }, @@ -2323,7 +2339,7 @@ }, { "cell_type": "markdown", - "id": "3cde68ea", + "id": "a1ddcdf1", "metadata": { "editable": true }, @@ -2349,7 +2365,7 @@ }, { "cell_type": "markdown", - "id": "e0b04810", + "id": "da7a5bbf", "metadata": { "editable": true }, @@ -2371,7 +2387,7 @@ }, { "cell_type": "markdown", - "id": "dd7968ba", + "id": "fc41c84e", "metadata": { "editable": true }, @@ -2389,7 +2405,7 @@ }, { "cell_type": "markdown", - "id": "6fb372cb", + "id": "a46d2c87", "metadata": { "editable": true }, @@ -2412,7 +2428,7 @@ }, { "cell_type": "markdown", - "id": "f04d9799", + "id": "dd7e5b6d", "metadata": { "editable": true }, @@ -2431,7 +2447,7 @@ }, { "cell_type": "markdown", - "id": "c258546e", + "id": "eef14fe3", "metadata": { "editable": true }, @@ -2459,7 +2475,7 @@ }, { "cell_type": "markdown", - "id": "356cd43c", + "id": "9f725ea6", "metadata": { "editable": true }, @@ -2479,7 +2495,7 @@ }, { "cell_type": "markdown", - "id": "ea800ab5", + "id": "8c93ba64", "metadata": { "editable": true }, @@ -2500,7 +2516,7 @@ }, { "cell_type": "markdown", - "id": "96b69144", + "id": "bc1ed0d3", "metadata": { "editable": true }, @@ -2514,7 +2530,7 @@ }, { "cell_type": "markdown", - "id": "ca2cf253", + "id": "7630316a", "metadata": { "editable": true }, @@ -2526,7 +2542,7 @@ }, { "cell_type": "markdown", - "id": "5cc62a16", + "id": "46d67bfa", "metadata": { "editable": true }, @@ -2548,7 +2564,7 @@ }, { "cell_type": "markdown", - "id": "3afc6427", + "id": "77136736", "metadata": { "editable": true }, @@ -2570,7 +2586,7 @@ }, { "cell_type": "markdown", - "id": "f7bd09b4", + "id": "de9c468d", "metadata": { "editable": true }, @@ -2596,7 +2612,7 @@ }, { "cell_type": "markdown", - "id": "370453da", + "id": "cfb50ca1", "metadata": { "editable": true }, @@ -2616,7 +2632,7 @@ }, { "cell_type": "markdown", - "id": "6dce1d04", + "id": "3f125ef4", "metadata": { "editable": true }, @@ -2636,7 +2652,7 @@ }, { "cell_type": "markdown", - "id": "910f2a34", + "id": "b71d2ee6", "metadata": { "editable": true }, @@ -2662,7 +2678,7 @@ }, { "cell_type": "markdown", - "id": "cfaf1f5f", + "id": "4f52bccf", "metadata": { "editable": true }, @@ -2691,7 +2707,7 @@ }, { "cell_type": "markdown", - "id": "fece4515", + "id": "521d8224", "metadata": { "editable": true }, @@ -2709,7 +2725,7 @@ }, { "cell_type": "markdown", - "id": "5dda98bd", + "id": "7353424c", "metadata": { "editable": true }, @@ -2725,7 +2741,7 @@ }, { "cell_type": "markdown", - "id": "161c505e", + "id": "ac41cf2d", "metadata": { "editable": true }, @@ -2737,7 +2753,7 @@ }, { "cell_type": "markdown", - "id": "d60af03f", + "id": "f5da6f7d", "metadata": { "editable": true }, @@ -2751,7 +2767,7 @@ }, { "cell_type": "markdown", - "id": "d8e4db81", + "id": "e3cc99fa", "metadata": { "editable": true }, @@ -2774,7 +2790,7 @@ }, { "cell_type": "markdown", - "id": "14ab8c21", + "id": "1fb0cd04", "metadata": { "editable": true }, @@ -2786,7 +2802,7 @@ }, { "cell_type": "markdown", - "id": "7356a830", + "id": "ff41762e", "metadata": { "editable": true }, @@ -2796,7 +2812,7 @@ }, { "cell_type": "markdown", - "id": "26cd1439", + "id": "b0f16ac3", "metadata": { "editable": true }, @@ -2808,7 +2824,7 @@ }, { "cell_type": "markdown", - "id": "a8e13b66", + "id": "4006451e", "metadata": { "editable": true }, @@ -2818,7 +2834,7 @@ }, { "cell_type": "markdown", - "id": "205ebbe0", + "id": "b722befc", "metadata": { "editable": true }, @@ -2830,7 +2846,7 @@ }, { "cell_type": "markdown", - "id": "551a96dc", + "id": "996ba49f", "metadata": { "editable": true }, @@ -2842,7 +2858,7 @@ }, { "cell_type": "markdown", - "id": "1c0b3401", + "id": "1839cfa3", "metadata": { "editable": true }, @@ -2853,7 +2869,7 @@ }, { "cell_type": "markdown", - "id": "5ad8c77b", + "id": "b2532482", "metadata": { "editable": true }, @@ -2881,7 +2897,7 @@ }, { "cell_type": "markdown", - "id": "b6533f98", + "id": "dd7f255a", "metadata": { "editable": true }, @@ -2893,7 +2909,7 @@ }, { "cell_type": "markdown", - "id": "bb2d393f", + "id": "77a835f0", "metadata": { "editable": true }, @@ -2903,7 +2919,7 @@ }, { "cell_type": "markdown", - "id": "2e148310", + "id": "3199d594", "metadata": { "editable": true }, @@ -2915,7 +2931,7 @@ }, { "cell_type": "markdown", - "id": "b5d87151", + "id": "616b74f1", "metadata": { "editable": true }, @@ -2926,7 +2942,7 @@ }, { "cell_type": "markdown", - "id": "28e2d06e", + "id": "a7f53917", "metadata": { "editable": true }, @@ -2938,7 +2954,7 @@ }, { "cell_type": "markdown", - "id": "eda4883d", + "id": "6edc54f7", "metadata": { "editable": true }, @@ -2951,7 +2967,7 @@ }, { "cell_type": "markdown", - "id": "71ce4abc", + "id": "5c76b2da", "metadata": { "editable": true }, @@ -2976,7 +2992,7 @@ }, { "cell_type": "markdown", - "id": "4c611560", + "id": "f7310c1c", "metadata": { "editable": true }, @@ -2989,7 +3005,7 @@ }, { "cell_type": "markdown", - "id": "2c0d438a", + "id": "b792a6ec", "metadata": { "editable": true }, @@ -3001,7 +3017,7 @@ }, { "cell_type": "markdown", - "id": "6c98d19b", + "id": "66dc0422", "metadata": { "editable": true }, @@ -3013,7 +3029,7 @@ }, { "cell_type": "markdown", - "id": "4d00e55a", + "id": "520431d8", "metadata": { "editable": true }, @@ -3023,7 +3039,7 @@ }, { "cell_type": "markdown", - "id": "c9aada26", + "id": "ebc9a411", "metadata": { "editable": true }, @@ -3035,7 +3051,7 @@ }, { "cell_type": "markdown", - "id": "a3b4dd2f", + "id": "30ec94ff", "metadata": { "editable": true }, @@ -3047,7 +3063,7 @@ }, { "cell_type": "markdown", - "id": "eec6576a", + "id": "2b35e6dc", "metadata": { "editable": true }, @@ -3059,7 +3075,7 @@ }, { "cell_type": "markdown", - "id": "85f8ed7a", + "id": "293d6eba", "metadata": { "editable": true }, @@ -3071,7 +3087,7 @@ }, { "cell_type": "markdown", - "id": "f2f13ddb", + "id": "48886ac8", "metadata": { "editable": true }, @@ -3081,7 +3097,7 @@ }, { "cell_type": "markdown", - "id": "22b642fc", + "id": "cd441e1c", "metadata": { "editable": true }, @@ -3093,7 +3109,7 @@ }, { "cell_type": "markdown", - "id": "6178c894", + "id": "de1a7b33", "metadata": { "editable": true }, @@ -3103,7 +3119,7 @@ }, { "cell_type": "markdown", - "id": "38e05b57", + "id": "ea69c7ef", "metadata": { "editable": true }, @@ -3115,7 +3131,7 @@ }, { "cell_type": "markdown", - "id": "9a4f4808", + "id": "3ac7275f", "metadata": { "editable": true }, @@ -3128,7 +3144,7 @@ }, { "cell_type": "markdown", - "id": "465d3ffc", + "id": "467a250e", "metadata": { "editable": true }, @@ -3140,7 +3156,7 @@ }, { "cell_type": "markdown", - "id": "d5da2c3b", + "id": "60db33ad", "metadata": { "editable": true }, @@ -3150,7 +3166,7 @@ }, { "cell_type": "markdown", - "id": "d4eb87c0", + "id": "5b06a7b5", "metadata": { "editable": true }, @@ -3162,7 +3178,7 @@ }, { "cell_type": "markdown", - "id": "67788627", + "id": "870fa8cc", "metadata": { "editable": true }, @@ -3173,7 +3189,7 @@ }, { "cell_type": "markdown", - "id": "1fb29f4a", + "id": "5a549f75", "metadata": { "editable": true }, @@ -3185,7 +3201,7 @@ }, { "cell_type": "markdown", - "id": "81955be4", + "id": "90f07923", "metadata": { "editable": true }, @@ -3195,7 +3211,7 @@ }, { "cell_type": "markdown", - "id": "5233b18b", + "id": "79e5a86d", "metadata": { "editable": true }, @@ -3207,7 +3223,7 @@ }, { "cell_type": "markdown", - "id": "326d597b", + "id": "978c6a93", "metadata": { "editable": true }, @@ -3217,7 +3233,7 @@ }, { "cell_type": "markdown", - "id": "5d331cc6", + "id": "5c7a1d26", "metadata": { "editable": true }, @@ -3228,7 +3244,7 @@ }, { "cell_type": "markdown", - "id": "ba07203c", + "id": "6c1e8554", "metadata": { "editable": true }, @@ -3241,7 +3257,7 @@ }, { "cell_type": "markdown", - "id": "4280c0c6", + "id": "cd9d1a87", "metadata": { "editable": true }, @@ -3251,7 +3267,7 @@ }, { "cell_type": "markdown", - "id": "6448b342", + "id": "41775cbb", "metadata": { "editable": true }, @@ -3263,7 +3279,7 @@ }, { "cell_type": "markdown", - "id": "3881713b", + "id": "7e99b816", "metadata": { "editable": true }, @@ -3273,7 +3289,7 @@ }, { "cell_type": "markdown", - "id": "9dc18af4", + "id": "bbccbbaf", "metadata": { "editable": true }, @@ -3285,7 +3301,7 @@ }, { "cell_type": "markdown", - "id": "238cb866", + "id": "3a6bdd01", "metadata": { "editable": true }, @@ -3295,7 +3311,7 @@ }, { "cell_type": "markdown", - "id": "ad0c9102", + "id": "02675ee6", "metadata": { "editable": true }, @@ -3319,7 +3335,7 @@ }, { "cell_type": "markdown", - "id": "41d90e91", + "id": "24906bda", "metadata": { "editable": true }, @@ -3369,7 +3385,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "3f281bbf", + "id": "1b520431", "metadata": { "collapsed": false, "editable": true @@ -3422,7 +3438,7 @@ }, { "cell_type": "markdown", - "id": "8c745ae2", + "id": "b37f42f1", "metadata": { "editable": true }, @@ -3443,7 +3459,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "cec6ceb5", + "id": "4bd1752e", "metadata": { "collapsed": false, "editable": true @@ -3481,7 +3497,7 @@ }, { "cell_type": "markdown", - "id": "67ed0f4e", + "id": "21668c31", "metadata": { "editable": true }, @@ -3525,7 +3541,7 @@ }, { "cell_type": "markdown", - "id": "dddab3f1", + "id": "ebd5fea6", "metadata": { "editable": true }, @@ -3565,7 +3581,7 @@ }, { "cell_type": "markdown", - "id": "6de261cd", + "id": "2838bcc7", "metadata": { "editable": true }, @@ -3586,7 +3602,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "f55b95fc", + "id": "ad2318ba", "metadata": { "collapsed": false, "editable": true @@ -3612,7 +3628,7 @@ }, { "cell_type": "markdown", - "id": "e4048837", + "id": "39eab2f4", "metadata": { "editable": true }, @@ -3640,7 +3656,7 @@ }, { "cell_type": "markdown", - "id": "8d8e15ea", + "id": "13f8824d", "metadata": { "editable": true }, @@ -3677,7 +3693,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "c366cb05", + "id": "a58b391b", "metadata": { "collapsed": false, "editable": true @@ -3723,7 +3739,7 @@ }, { "cell_type": "markdown", - "id": "a329838e", + "id": "97fec3b9", "metadata": { "editable": true }, @@ -3754,7 +3770,7 @@ }, { "cell_type": "markdown", - "id": "acc107d3", + "id": "b383bb8e", "metadata": { "editable": true }, @@ -3792,7 +3808,7 @@ }, { "cell_type": "markdown", - "id": "b7513f34", + "id": "ce251c80", "metadata": { "editable": true }, @@ -3826,7 +3842,7 @@ }, { "cell_type": "markdown", - "id": "80d4c1fa", + "id": "428e9644", "metadata": { "editable": true }, @@ -3867,7 +3883,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "717ac016", + "id": "f1319356", "metadata": { "collapsed": false, "editable": true @@ -3946,7 +3962,7 @@ }, { "cell_type": "markdown", - "id": "c5578ae4", + "id": "69491ed2", "metadata": { "editable": true }, @@ -3967,7 +3983,7 @@ }, { "cell_type": "markdown", - "id": "897b827c", + "id": "81f07a43", "metadata": { "editable": true }, @@ -3981,7 +3997,7 @@ { "cell_type": "code", "execution_count": 8, - "id": "2d81f6b5", + "id": "e176e7cc", "metadata": { "collapsed": false, "editable": true @@ -4091,7 +4107,7 @@ }, { "cell_type": "markdown", - "id": "71093a4f", + "id": "47ae64de", "metadata": { "editable": true }, @@ -4110,7 +4126,7 @@ { "cell_type": "code", "execution_count": 9, - "id": "dbda9e36", + "id": "314354e0", "metadata": { "collapsed": false, "editable": true @@ -4137,7 +4153,7 @@ }, { "cell_type": "markdown", - "id": "d1baf786", + "id": "c030fa58", "metadata": { "editable": true }, @@ -4151,7 +4167,7 @@ { "cell_type": "code", "execution_count": 10, - "id": "07cfe247", + "id": "4a6b0ee5", "metadata": { "collapsed": false, "editable": true @@ -4182,7 +4198,7 @@ }, { "cell_type": "markdown", - "id": "333bada1", + "id": "f9b0c009", "metadata": { "editable": true }, @@ -4193,7 +4209,7 @@ { "cell_type": "code", "execution_count": 11, - "id": "4aa6acfb", + "id": "400a00b1", "metadata": { "collapsed": false, "editable": true @@ -4237,7 +4253,7 @@ }, { "cell_type": "markdown", - "id": "40608240", + "id": "43513585", "metadata": { "editable": true }, @@ -4260,7 +4276,7 @@ { "cell_type": "code", "execution_count": 12, - "id": "ebb62e20", + "id": "a1ea5919", "metadata": { "collapsed": false, "editable": true @@ -4287,7 +4303,7 @@ }, { "cell_type": "markdown", - "id": "893ff59f", + "id": "8f2d540e", "metadata": { "editable": true }, @@ -4298,7 +4314,7 @@ { "cell_type": "code", "execution_count": 13, - "id": "5063e339", + "id": "1e133a00", "metadata": { "collapsed": false, "editable": true @@ -4343,7 +4359,7 @@ }, { "cell_type": "markdown", - "id": "4f48007a", + "id": "4a154c4e", "metadata": { "editable": true }, @@ -4361,7 +4377,7 @@ }, { "cell_type": "markdown", - "id": "093bd249", + "id": "e480555f", "metadata": { "editable": true }, @@ -4396,7 +4412,7 @@ { "cell_type": "code", "execution_count": 14, - "id": "f592148d", + "id": "9e2f7517", "metadata": { "collapsed": false, "editable": true @@ -4408,7 +4424,7 @@ }, { "cell_type": "markdown", - "id": "78c5fc79", + "id": "ce92e9d5", "metadata": { "editable": true }, @@ -4420,7 +4436,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "3a17f898", + "id": "df07f1fd", "metadata": { "collapsed": false, "editable": true @@ -4433,7 +4449,7 @@ }, { "cell_type": "markdown", - "id": "1909eced", + "id": "6d4a3bc4", "metadata": { "editable": true }, @@ -4444,7 +4460,7 @@ { "cell_type": "code", "execution_count": 16, - "id": "5156163c", + "id": "a483bea5", "metadata": { "collapsed": false, "editable": true @@ -4457,7 +4473,7 @@ }, { "cell_type": "markdown", - "id": "230b7faa", + "id": "09e7a7a2", "metadata": { "editable": true }, @@ -4472,7 +4488,7 @@ { "cell_type": "code", "execution_count": 17, - "id": "1eef0b98", + "id": "10a78c23", "metadata": { "collapsed": false, "editable": true @@ -4484,7 +4500,7 @@ }, { "cell_type": "markdown", - "id": "cea9bd1a", + "id": "588a91b0", "metadata": { "editable": true }, @@ -4496,7 +4512,7 @@ }, { "cell_type": "markdown", - "id": "99d7772b", + "id": "b4a2bbe6", "metadata": { "editable": true }, @@ -4509,7 +4525,7 @@ { "cell_type": "code", "execution_count": 18, - "id": "6c6f5bfc", + "id": "a82a877e", "metadata": { "collapsed": false, "editable": true @@ -4564,7 +4580,7 @@ { "cell_type": "code", "execution_count": 19, - "id": "4fc79e1e", + "id": "68750ee2", "metadata": { "collapsed": false, "editable": true @@ -4593,7 +4609,7 @@ { "cell_type": "code", "execution_count": 20, - "id": "241dcbec", + "id": "a4cd8054", "metadata": { "collapsed": false, "editable": true @@ -4623,7 +4639,7 @@ { "cell_type": "code", "execution_count": 21, - "id": "1936206d", + "id": "6de65a81", "metadata": { "collapsed": false, "editable": true @@ -4650,7 +4666,7 @@ { "cell_type": "code", "execution_count": 22, - "id": "e52140a7", + "id": "60f77310", "metadata": { "collapsed": false, "editable": true @@ -4692,7 +4708,7 @@ }, { "cell_type": "markdown", - "id": "665b3ebd", + "id": "d1cf196b", "metadata": { "editable": true }, @@ -4703,7 +4719,7 @@ { "cell_type": "code", "execution_count": 23, - "id": "36caace7", + "id": "f561630c", "metadata": { "collapsed": false, "editable": true @@ -4880,7 +4896,7 @@ }, { "cell_type": "markdown", - "id": "9f85a17f", + "id": "7337285c", "metadata": { "editable": true }, @@ -4899,7 +4915,7 @@ }, { "cell_type": "markdown", - "id": "6d9cfcc8", + "id": "ae88bf09", "metadata": { "editable": true }, @@ -4921,7 +4937,7 @@ { "cell_type": "code", "execution_count": 24, - "id": "be81a71a", + "id": "ce29cadf", "metadata": { "collapsed": false, "editable": true @@ -5062,7 +5078,7 @@ }, { "cell_type": "markdown", - "id": "63283a38", + "id": "72db4c2e", "metadata": { "editable": true }, @@ -5078,7 +5094,7 @@ { "cell_type": "code", "execution_count": 25, - "id": "9584fa14", + "id": "24954087", "metadata": { "collapsed": false, "editable": true @@ -5091,7 +5107,7 @@ }, { "cell_type": "markdown", - "id": "2a5616ae", + "id": "1696f10c", "metadata": { "editable": true }, @@ -5103,7 +5119,7 @@ { "cell_type": "code", "execution_count": 26, - "id": "866b3bfa", + "id": "1b5a99e5", "metadata": { "collapsed": false, "editable": true @@ -5125,7 +5141,7 @@ }, { "cell_type": "markdown", - "id": "756d05d2", + "id": "ee76a451", "metadata": { "editable": true }, @@ -5141,7 +5157,7 @@ { "cell_type": "code", "execution_count": 27, - "id": "d8850104", + "id": "baa5a1c9", "metadata": { "collapsed": false, "editable": true @@ -5179,7 +5195,7 @@ }, { "cell_type": "markdown", - "id": "8edc05c4", + "id": "eaedf4e6", "metadata": { "editable": true }, @@ -5192,7 +5208,7 @@ { "cell_type": "code", "execution_count": 28, - "id": "a9e67ca3", + "id": "077af6de", "metadata": { "collapsed": false, "editable": true @@ -5213,7 +5229,7 @@ }, { "cell_type": "markdown", - "id": "1dd64c49", + "id": "3608c95b", "metadata": { "editable": true }, @@ -5229,7 +5245,7 @@ { "cell_type": "code", "execution_count": 29, - "id": "129aad84", + "id": "3d8748eb", "metadata": { "collapsed": false, "editable": true @@ -5287,7 +5303,7 @@ }, { "cell_type": "markdown", - "id": "e4dfc795", + "id": "165a06a9", "metadata": { "editable": true }, @@ -5302,7 +5318,7 @@ { "cell_type": "code", "execution_count": 30, - "id": "c0e717fc", + "id": "fd978199", "metadata": { "collapsed": false, "editable": true @@ -5323,7 +5339,7 @@ }, { "cell_type": "markdown", - "id": "29b69c91", + "id": "2b0f73af", "metadata": { "editable": true }, @@ -5347,7 +5363,7 @@ { "cell_type": "code", "execution_count": 31, - "id": "904c351d", + "id": "7396831b", "metadata": { "collapsed": false, "editable": true @@ -5819,7 +5835,7 @@ }, { "cell_type": "markdown", - "id": "2673e5bc", + "id": "d1f68897", "metadata": { "editable": true }, @@ -5831,7 +5847,7 @@ { "cell_type": "code", "execution_count": 32, - "id": "10ba6bc4", + "id": "764d64d2", "metadata": { "collapsed": false, "editable": true @@ -5875,7 +5891,7 @@ }, { "cell_type": "markdown", - "id": "d452842e", + "id": "20158cd2", "metadata": { "editable": true }, @@ -5891,7 +5907,7 @@ { "cell_type": "code", "execution_count": 33, - "id": "2871b76c", + "id": "138d2e76", "metadata": { "collapsed": false, "editable": true @@ -5906,7 +5922,7 @@ }, { "cell_type": "markdown", - "id": "9e2a0f1d", + "id": "6b8812f9", "metadata": { "editable": true }, @@ -5917,7 +5933,7 @@ { "cell_type": "code", "execution_count": 34, - "id": "95caf15c", + "id": "36f68eca", "metadata": { "collapsed": false, "editable": true @@ -5932,7 +5948,7 @@ }, { "cell_type": "markdown", - "id": "33f4287b", + "id": "1e2278c3", "metadata": { "editable": true }, @@ -5948,7 +5964,7 @@ { "cell_type": "code", "execution_count": 35, - "id": "19065db0", + "id": "82d5f7ed", "metadata": { "collapsed": false, "editable": true @@ -5962,7 +5978,7 @@ }, { "cell_type": "markdown", - "id": "6715b1fc", + "id": "7c82e420", "metadata": { "editable": true }, @@ -5977,7 +5993,7 @@ { "cell_type": "code", "execution_count": 36, - "id": "1958e0dd", + "id": "d69bc831", "metadata": { "collapsed": false, "editable": true @@ -6003,7 +6019,7 @@ { "cell_type": "code", "execution_count": 37, - "id": "d33c99ee", + "id": "b31e282b", "metadata": { "collapsed": false, "editable": true @@ -6018,7 +6034,7 @@ }, { "cell_type": "markdown", - "id": "2763a4de", + "id": "d9fb4833", "metadata": { "editable": true }, @@ -6029,7 +6045,7 @@ { "cell_type": "code", "execution_count": 38, - "id": "04cbbddf", + "id": "bd849a1e", "metadata": { "collapsed": false, "editable": true @@ -6044,7 +6060,7 @@ }, { "cell_type": "markdown", - "id": "f405b627", + "id": "59e4d897", "metadata": { "editable": true }, @@ -6055,7 +6071,7 @@ { "cell_type": "code", "execution_count": 39, - "id": "efbfba8c", + "id": "9a3e6ff5", "metadata": { "collapsed": false, "editable": true @@ -6075,7 +6091,7 @@ { "cell_type": "code", "execution_count": 40, - "id": "bf066770", + "id": "85cc07cf", "metadata": { "collapsed": false, "editable": true @@ -6090,7 +6106,7 @@ }, { "cell_type": "markdown", - "id": "3b99ffc1", + "id": "a4d3c4d4", "metadata": { "editable": true }, @@ -6105,7 +6121,7 @@ { "cell_type": "code", "execution_count": 41, - "id": "f85167ba", + "id": "53a180f6", "metadata": { "collapsed": false, "editable": true @@ -6142,7 +6158,7 @@ }, { "cell_type": "markdown", - "id": "d614b967", + "id": "41c67355", "metadata": { "editable": true }, @@ -6155,7 +6171,7 @@ { "cell_type": "code", "execution_count": 42, - "id": "f4d5609a", + "id": "ca3a2127", "metadata": { "collapsed": false, "editable": true @@ -6178,7 +6194,7 @@ }, { "cell_type": "markdown", - "id": "efce242a", + "id": "b3dc22c1", "metadata": { "editable": true }, diff --git a/doc/src/week42/exercisesweek42.ipynb b/doc/src/week42/exercisesweek42.ipynb deleted file mode 100644 index 846880178..000000000 --- a/doc/src/week42/exercisesweek42.ipynb +++ /dev/null @@ -1,467 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4b4c06bc", - "metadata": {}, - "source": [ - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "bcb25e64", - "metadata": {}, - "source": [ - "# Exercises week 42\n", - "\n", - "**October 14-18, 2024**\n", - "\n", - "Date: **Deadline is Friday October 18 at midnight**\n" - ] - }, - { - "cell_type": "markdown", - "id": "bb01f126", - "metadata": {}, - "source": [ - "# Overarching aims of the exercises this week\n", - "\n", - "The aim of the exercises this week is to get started with implementing a neural network. There are a lot of technical and finicky parts of implementing a neutal network, so take your time.\n", - "\n", - "This week, you will implement only the feed-forward pass. Next week, you will implement backpropagation. We recommend that you do the exercises this week by editing and running this notebook file, as it includes several checks along the way that you have implemented the pieces of the feed-forward pass correctly. If you have trouble running a notebook, or importing pytorch, you can run this notebook in google colab instead: (LINK TO COLAB), though we recommend that you set up VSCode and your python environment to run code like this locally.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c6f61b09", - "metadata": {}, - "outputs": [], - "source": [ - "import autograd.numpy as np\n", - "from autograd import grad" - ] - }, - { - "cell_type": "markdown", - "id": "6248ec53", - "metadata": {}, - "source": [ - "# Exercise 1\n", - "\n", - "Complete the following parts to compute the activation of the first layer.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "37f30740", - "metadata": {}, - "outputs": [], - "source": [ - "np.random.seed(2024)\n", - "\n", - "\n", - "def ReLU(z):\n", - " return np.where(z > 0, z, 0)\n", - "\n", - "\n", - "x = np.random.randn(2) # network input\n", - "W1 = np.random.randn(4, 2) # first layer weights" - ] - }, - { - "cell_type": "markdown", - "id": "edf7217b", - "metadata": {}, - "source": [ - "**a)** Define the bias of the first layer, `b1`with the correct shape\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2129c19f", - "metadata": {}, - "outputs": [], - "source": [ - "b1 = np.random.randn(4)" - ] - }, - { - "cell_type": "markdown", - "id": "09e8d453", - "metadata": {}, - "source": [ - "**b)** Compute the intermediary `z1` for the first layer\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6837119b", - "metadata": {}, - "outputs": [], - "source": [ - "z1 = W1 @ x + b1" - ] - }, - { - "cell_type": "markdown", - "id": "6f71374e", - "metadata": {}, - "source": [ - "**c)** Compute the activation `a1` for the first layer using the ReLU activation function defined earlier.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8d41ed19", - "metadata": {}, - "outputs": [], - "source": [ - "a1 = ReLU(z1)" - ] - }, - { - "cell_type": "markdown", - "id": "088710c0", - "metadata": {}, - "source": [ - "Confirm that you got the correct activation with the test below.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4d2f54b4", - "metadata": {}, - "outputs": [], - "source": [ - "sol1 = np.array([0.60610368, 4.0076268, 0.0, 0.56469864])\n", - "\n", - "print(np.allclose(a1, sol1))" - ] - }, - { - "cell_type": "markdown", - "id": "7fb0cf46", - "metadata": {}, - "source": [ - "# Exercise 2\n", - "\n", - "Compute the activation of the second layer with an output of length 8 and ReLU activation.\n", - "\n", - "**a)** Define the weight and bias of the second layer with the right shapes.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "00063acf", - "metadata": {}, - "outputs": [], - "source": [ - "W2 = np.random.randn(8, 4)\n", - "b2 = np.random.randn(8)" - ] - }, - { - "cell_type": "markdown", - "id": "5bd7d84b", - "metadata": {}, - "source": [ - "**b)** Compute intermediary `z2` and activation `a2` for the second layer.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2fd0383d", - "metadata": {}, - "outputs": [], - "source": [ - "z2 = W2 @ a1\n", - "a2 = ReLU(z2)" - ] - }, - { - "cell_type": "markdown", - "id": "1b5daae5", - "metadata": {}, - "source": [ - "Confirm that you got the correct activation shape with the test below.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f7f2f8a1", - "metadata": {}, - "outputs": [], - "source": [ - "print(a2.shape == (8,))" - ] - }, - { - "cell_type": "markdown", - "id": "3759620d", - "metadata": {}, - "source": [ - "# Exercise 3\n", - "\n", - "We often want our neural networks to have many layers of varying sizes. To avoid writing very long and error-prone code where we explicitly define and evaluate each layer we should keep all our layers in a single variable which is easy to create and use.\n", - "\n", - "**a)** Complete the function below so that it returns a list `layers` of weight and bias tuples `(W, b)` for each layer, in order, with the correct shapes that we can use later as our network parameters.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c58f10f9", - "metadata": {}, - "outputs": [], - "source": [ - "def create_layers(network_input_size, output_sizes):\n", - " layers = []\n", - "\n", - " i_size = network_input_size\n", - " for output_size in output_sizes:\n", - " W = np.random.rand(output_size, i_size)\n", - " b = np.random.rand(output_size)\n", - " layers.append((W, b))\n", - "\n", - " i_size = output_size\n", - " return layers" - ] - }, - { - "cell_type": "markdown", - "id": "bdc0cda2", - "metadata": {}, - "source": [ - "**b)** Comple the function below so that it evaluates the intermediate `z` and activation `a` for each layer, and returns the final activation `a`. This is the complete feed-forward pass, a full neural network!\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5262df05", - "metadata": {}, - "outputs": [], - "source": [ - "def feed_forward(layers, input):\n", - " a = input\n", - " for W, b in layers:\n", - " z = W @ a + b\n", - " a = ReLU(z)\n", - " return a" - ] - }, - { - "cell_type": "markdown", - "id": "245adbcb", - "metadata": {}, - "source": [ - "**c)** Create a network with input size 8 and layers with output sizes 10, 16, 6, 2. Evaluate it and make sure that you get the correct size vectors along the way.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "89a8f70d", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "306d8b7c", - "metadata": {}, - "source": [ - "# Exercise 4\n" - ] - }, - { - "cell_type": "markdown", - "id": "221c7b6c", - "metadata": {}, - "source": [ - "So far, every layer has used the same activation, ReLU. We often want to use other types of activation however, so we need to update our code to support multiple types of activation. Make sure that you have completed every previous exercise before trying this one.\n", - "\n", - "**a)** Make the `create_layers` function also accept a list of activation functions, which is used to add activation functions to each of the tuples in `layers`. Make new functions to not mess with the old ones.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9df82312", - "metadata": {}, - "outputs": [], - "source": [ - "def create_layers_4(network_input_size, output_sizes, activation_funcs):\n", - " layers = []\n", - "\n", - " i_size = network_input_size\n", - " for output_size, activation in zip(output_sizes, activation_funcs):\n", - " W = np.random.rand(output_size, i_size)\n", - " b = np.random.rand(output_size)\n", - " layers.append((W, b, activation))\n", - "\n", - " i_size = output_size\n", - " return layers" - ] - }, - { - "cell_type": "markdown", - "id": "10896d06", - "metadata": {}, - "source": [ - "**b)** Update the `feed_forward` function to support this change.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "de062369", - "metadata": {}, - "outputs": [], - "source": [ - "def feed_forward_4(layers, input):\n", - " a = input\n", - " for W, b, activation in layers:\n", - " z = W @ a + b\n", - " a = activation(z)\n", - " return a" - ] - }, - { - "cell_type": "markdown", - "id": "efd07b4e", - "metadata": {}, - "source": [ - "**c)** Create and evaluate a neural network with 4 inputs and layers with output sizes 12, 10, 3 and activations ReLU, ReLU, softmax.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce6fcc2f", - "metadata": {}, - "outputs": [], - "source": [ - "from scipy.special import softmax\n", - "\n", - "network_input_size = 4\n", - "output_sizes = [12, 10, 3]\n", - "activation_funcs = [ReLU, ReLU, softmax]\n", - "layers = create_layers_4(network_input_size, output_sizes, activation_funcs)\n", - "\n", - "x = np.random.randn(network_input_size)\n", - "predict = feed_forward_4(layers, x)" - ] - }, - { - "cell_type": "markdown", - "id": "54d5fde7", - "metadata": {}, - "source": [ - "The final exercise will hopefully be very simple if everything has worked so far. You will evaluate your neural network on the iris data set (https://scikit-learn.org/1.5/auto_examples/datasets/plot_iris_dataset.html).\n", - "\n", - "This dataset contains data on 150 flowers of 3 different types which can be separated pretty well using the four features given for each flower, which includes the width and length of their leaves. You are not expected to do any training of the network or actual classification, unless you feel like it, in that case you can do exercise 5.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bd4c148", - "metadata": {}, - "outputs": [], - "source": [ - "# Loading and plotting iris dataset\n", - "from sklearn import datasets\n", - "import matplotlib.pyplot as plt\n", - "\n", - "iris = datasets.load_iris()\n", - "\n", - "_, ax = plt.subplots()\n", - "scatter = ax.scatter(iris.data[:, 0], iris.data[:, 1], c=iris.target)\n", - "ax.set(xlabel=iris.feature_names[0], ylabel=iris.feature_names[1])\n", - "_ = ax.legend(\n", - " scatter.legend_elements()[0], iris.target_names, loc=\"lower right\", title=\"Classes\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "c528846f", - "metadata": {}, - "source": [ - "**c)** Loop over the iris dataset(`iris.data`) and evaluate the network for each data point.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2efc507d", - "metadata": {}, - "outputs": [], - "source": [ - "# No need to change this cell! Just make sure it works!\n", - "for x in iris.data:\n", - " prediction = feed_forward_4(layers, x)" - ] - }, - { - "cell_type": "markdown", - "id": "334560b6", - "metadata": {}, - "source": [ - "# Exercise 5 (Very optional and very hard :)\n" - ] - }, - { - "cell_type": "markdown", - "id": "ea0a8fe0", - "metadata": {}, - "source": [ - "**a)** Make the iris target values into one-hot vectors.\n", - "\n", - "**b)** Define the cross-entropy loss function to evaluate the performance of your network on the data set.\n", - "\n", - "**c)** Use the autograd package to take the gradient of the cross entropy wrt. the weights and biases of the network.\n", - "\n", - "**d)** Use gradient descent of some sort to optimize the parameters.\n", - "\n", - "**e)** Evaluate the accuracy of the network.\n", - "\n", - "**e)** Show off how you did in a group session!\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/doc/src/week42/figures/nn2.pdf b/doc/src/week42/figures/nn2.pdf new file mode 100644 index 0000000000000000000000000000000000000000..7b62d8ff7e8041ca714b58182a0415744f59cf06 GIT binary patch literal 35328 zcma(2V~j3L)HMjVZ5vnHwr$(CecHBd+qQYyw!2T;Hs*fbd{6StWPVKj+AFEl-l=3& zRo2=?rYI^-$3)KtLpHxWx(>rm#7N{|Yz@Q93&S95W^dtYNyN&=#76Xg1`LC^m949p zGZBNht&yvlsF{g_sTmAEKa7j3vzd_{%%9C`J(>8UHsrqB`uPQZ#e+{LJtC%rEpBfY zA1HVHt-v)YJY1n4->4)_wUmmRG*-Ly5ug`W1-HgGr0NTj%WQovAD;1pXpV>3>Y#jQ z#oz|_7IVeP82!SfWqeUMQiR2>OA0AVuA5%Cg$yD&pUc7&I$0alp@cq9} z*NVG-@1vw@O44lK+-&Yd(#|Ny27cQA@s{)@BUsk1Fot~?w|$tROcJm$-R+?4eY!tS zWAL%Un@=f_lFO5rK@ZZ6ZPIc<&Zu!V$P4oe(k5}O{t=JRBAOW~G728aKxrrTFFmFU zf3PW-ByvK=R_~Ct^9gcDOBTSOd77M2{*)Il#pQtQVwI!YtpoPyl}%gj7m!d>31|=_ z?OSGTP)FnnI}V+|8op0rNF_IMdm& zg$YyiOH8p&J8Z)9h2{B8_qJy51i3yqhu77X7sOIt%}`e8X3#b7aRCe&HAtS^19sT` zVMd`spP-1sO@by$^js7EB6lA!M8enuCPJ;}-iM?n*`U?)zI@FKy4!uc#^fjc(-KqN zkc9Mh8bMxs6;>`)&*MQ6f2`dvv&M{l&l7xy5+x*{sq6Bo=4#WsW_yDOjWBP`X9J z!gia6F4!G``cx+l=3sT)SWbYLi_CM@kpcTew0jyRtrUHvs_kR#J~ZYdGnMg+`AJ4i z<{5Y}m}Mg{lV~WDV?v3Pi&26Z&$ZhaCZ=-VJR>Z6Uj8GA=^xKe+x`mO={IXPxi7`B z@`Y9u{@R65$L+W=V-CD##vaT#vG{u|)ILjdpT_(LLP5!*r5pvDRnb&qLUsOtWYP2v z6eE(e;HO*-(I7?i%>*^C^RO|FuuzaW?p!S$*Bz>9ozOiOD{0E1f2g@u@ z6jk+r642^l=J9qY4f|JiKoJtqteh?FdG^R|ix*fVeiNMWe7LI-=f}^2yG1T(v=Q$= zhPdx9ZF>6LPA`}HYnI833GwEM5htF@LccLgCXXfoqd?7mc07u65kq;x;{*%-!)zVpspeH=OFl^`|ytocX@#eZ+x8B_jw4VhM2U zYeV2BaxJ|VZ**r;cqhPAI!AO*3#-gjf|m|Mrbk-7L#U7zojlYJnrlu+d;>xy`J-p@ zLyNJi5zgWNS+#qyqnTOvM3Cy#5EFWwed0)DrJT!z`W6WCL_pTP({^Tc`VYsu8M8Ca z;*<7BX(gSUfzrZMQBf3TpmGrk(HYSgFi2!nRsdrd8uC>F7n)iPR1&}+WQKSOr2E(w z4v(N}VS0Ymy~#$>H4Lti^uywj5E^9%APAuL3;}*;Bwd6gUO)hE5-g%Ylazf6a4l^k zUxqdSC1sZywHvaC^kX=fi}j8kjUcAMl+cY>gk&UrmrKWDUTSL^w3eWfxtz5dn=+mD2hJzOgdvbipVWr!K}FOQLyD^7a)$lm0qvB13m5Sdtg zW`BUxf_U;FR1RA2zlj%Yu`5Dlf`F8wB^4Oq$&CIMa?)-X{Q@8$%1qRPrk2)vqVrH8 z-`HBrwzT28D~$@grKg?)}$|!;)sS* z`j_J1;4L!A#{KN?P<>aeoRM!$diNL-(z`{ES?!WuaLnNCn_B#F+Nb?u{Gkl#tM&eV zPm6|?074d~rqGR8Wn%PZT(U8Ka}H^-#C^mK6Xscdb4K7*X6sl%i zQs%9UZwUI_LP8U&^NlAH>h7I=LX(3_IAARU5~BK#-9edjtZ~&@$bphv{aJI(nK@eH zh;4q`sdv@CdsL_CZGCGrzhAvbK+Y^O*k+nVL}^+1OGLhj0!pHpWNxIlb) zz?m{z_JslA1qcn>bp?Wxpm$RFpqVn4_DtrfiO}ptZ^$uHp4aH~dhKO=yro2LGk?ga z^hYdD4fh~MIXAfd2L-M)R2Ev?ewxK^-_u*HQ6+2)@iV#5?rreN>DClX5gaNj4E*xN zKuN{2@UpgMJ;o$VpGNGG+F+W*|M1K42>Gxb5XgBia7!$)V_I!i&uY9?)GV`QxyJzm zCGUlveYHm}czTDG>EUGSTWwB&Y_l|#S=oRp4QA~l zF15(WY|+Sh_rW^Vm04N*%8w0#lAY;bi_l84hnH*;Oy)uy2C^% zA|%Nc1aiw_6VAxxx(kw2SnXUA%X%+-5K;7stkOf~sE|_b6y{r~4ql}K1mUzgIS+?E zd$Pjp^Rbc^TC5~;3#^vnE>dd5kdjbAphXButx&_Rk`W}4l9pPag$c^7P=kC*tpvak zBiV@n+H67yjiKZ@WpYUWof&70-mEO5+I<)y{wFyqKQ)0y+9Y5c4`>!OJ^}6eSh!$BE z`#(hc!OmjO{p(d~WfEYo9*eNlqUCziYOks&*v;x0RyLz3SU)N?l71c4&Z+!>k-OGI zlf}NJMe$!2i{8bSe6Tc5s+efl+N?cFx+@#MVo_E*zhYH1=$`~WauYl&zl9JVYxAUx zwXcitrIroB|1Jk1Ky&Pd!~P<47JF(^p8V2S(-vKpJC#pvp;dPD|0%7tP_zLlEgN@}8nx##MfKal zx?*fuXj>agZX$A~?%Gz?r4}d&s3*TLkWWr*NLm$wJ2#0gumnx=B0jtF0{36>rgrGf z8DM6^mK&eYm`Q0pW0GTj-nZPkbIJF3R6EzxGfA0$^5>G9plMHj$NSvJui5JF1MhF4(#BKW_kBLwt3C2R)(|D2E36hLwAtyMJH2{)E;v{Oh zP+4~K-ZAk*5KeAE@vewC`^3bkMJnc`-kD`TG~Ct+NweI#ghWd6c991oW|UFW1uI2* ztNzqyg~mk2YmZ5B04>#~p)#IOa*;RZ27hr*5y`^p#bb(aMAA~C<{P)Hgz+Js_QzYF2gcUL!W+jzDKR_y^&}!lq=? zP0cjNMHMt{3;6v0>-T&2l!A%Vc(&w=>QN)vYD**#do1RAzsLsYajQj<;TP9fCr}Z zPs+FEi!V@1FBSQ}e*cZ%`+DnS9GeG)x&!(dJ`>xj;~KOybN?1gnBs=gju1x!u8wF) zdz}(lf45ZVI}9uZ3LH9O0FQKuyrowbQ*qnt#%FGUxwiw1clr=DT!$;Rpn=tWA@(XD z9_06b>28A*Xq7d|!n^MnTTr4i>va`2s4DM7ek<At6|M3?2nRuz0Zn zLVAJRiZ!6y;OWQ^ZV))cMo^rqmxRO}CHM`RDWp(G2M5GJZi7%n3olLEK7xo~tH72} z|Gn#O9Ch~R(gM2_G2il6!GQ1tO(4Ne+6SSm!k+_*4g_`F=37q~FJZJ}St%JaRsR-az)L1};9SFZq;emz7l(gbN!Hz#0Bxe>_ z+icYxo7BX5y8CsSaQha%lTfU9Vci}K^Z5tdod7;v-=DNAq`EeDfvpDL2$0MUz&61U zTgtDYBlSSP+lwQuA{Z?_IVn9zNH_379wX%Z9N3sX6I9x9fDhOo4MnM zV;Szp+rw?YEh3!$*KTD%*y*xh-#WgSRX|h|i1te-exL%td(id)MLT`KVdr-#=SQEz zGK5|iKSJckZ|taVc;~<2r~~0L#3#&hFK2g<*S`)A2HgQ@+Lg8EPaf0NK$ z7h%Oof5jwpfg|{u5?K`0p_1?be-D44m1@|b zE>?$i63@g=)~;6-@DNg^!{2g6--&B77r6PLRcl`U15J3>Tv4~#ygnkUOQ4G#|H@_B z!y>dB@VZSam8jK(v=duChbFZkPi>j zunO(2l;E)YS2m@>%6b&dRsKdgIHiYA7X!g(iVu^v#al1o-LG_ah90nYDxl0CD=ZD7 z)udn7E(RvC1j8D-N0qC6<<&H&8$@Ultx=M#;?smF`B(?J5kmpy$N&?=y>Z9CH=qD| z0iwvZD_Z~LAe9-{wh{f(cSb&({ypvSF49J&#UQA%E(Xl|$DUG^V_{rm=X7cEGpNie zQaZ?)80_BBQy_UahijE17YrF!_8aJIxYKEZ2Hn;|0SU>x+8n#cV}PuvwShYH?V%!P=)BMRR` zv9R7Pgm3eQEeyaEp7j3y4bR0JC%K{U~3I^c_QCyK=)&d+tveMaiGua0k6! zTs8S)-yP^bJadM&v&A|X7po>DU2u`BoK|>?Mf-4x!TrCESTv8OJ9UlnURz3+SLayW zD#4F~wBajxu-)KdRU-g@4aM09JBo^z0g{jNe`x=Vd?V&Q2eO}*>$naS1WYUCKi09@ z;n=s3!~Nd#R4!2K___w)OM3C={W(;O8~Y7^jvSla<<;r$PL7mgoo37VkID)qHrInl ziDJ0$cRRp=t`ovIYMGhb=NJFv`Z;tQVCJBP>1FnO0xHhqGza2v+5ha`jMV2^n~?@~ z15|V7#~rWii`ML@&*5Za3rvdg%pQdmQNblXTP?^l<%l2ooBk~n@8+?$}gq4I=L^ZZQKjSf21!$W-&ndW*-~tFUh(3g!o}J&EUof(sNp(c zb2Z~mq&r{4hf0q_h9xY&nNd?P;HJJS(b;u$e#$I{xD~?$I|;dd6Ip%Qv(^iA08aX+C=?NUJY2S z#-n?y)kOO;F7I_Li&O)zgaO2DC`Tb?1^ZK1f}v-zd!L+-FWn*o34G{02Ff%7X)2)} z9JXN_;ARy%*v;f9-X6n}$S-sq3@zHnt_nmN$gsjw4&t94s`J8dAmTd($vi)ynWE@L*^%?~Btilt!v&ekAMZC%y(L2BannClemEDDyb zJI~1Kl@>sH^Liu@aA#vPt1W7gp67WSj$hyR>0u0LNuJI;(7GDxamO85k3w>e(~yqmOa|yDFuHY|2uAv9Hp;dckE#YLxt=XKilmFo3V?V z1k_5>Z?XRTrw)qRg^Z8Db*3^aXFNk#R-zw=z!N*PbN9WIi~DRU638p9Bq+dB<9JZU zFDU8p!{L)dJva)aS0^bgzi=vIeC9racX@ql#*{9=;>e=* z0qsZoeci9$swOuoBz^};{(`ygE1-b2ix1~mph2$oFC?T5kuf)KVjZ2k*AEP2bur z1uXGkZ>-zkn|A+Us3aixR5WGqYso(Ma8mfow8r)Kqsb4ACu*RPwV>BxrWh4>D4H@$ zEalj$4`;4Q02i_?MoZ7wN8U z)TR=QY;kT1TYD{h&UX*$jdYlLC?|eE!6n}jeNmQQ86b7A% zTT*|x?txlfjc*e?-uvdoHV;V&+I`j$JMm^3;pEQMm`_7G6)r*Z)0%DMcXMvSz+3d^ z@->NL4RYDEi#8gczt)8(wVVSWIchg%9Dl@=)*jEhLJw{%A~nuo`HUd4bR%KUoyJh5 zjH|%W-#_~oVF#v&Bd3aUaGh-V=){-V13G|9C?U0nCSGy~LgNGqlv{Om*tQMuo-(=Z z&ch`#ce}eiKQ4ERtw1c=o}sYP>-qKaFjYy6XLU``J|EK78=LgW&a1DUXY?v$b%x<| zJQXR)v7PL-!aP89#2)p3F}lh zwHqA`WV)BjV9YQ#qmz;Wc_J%_HGM5E9EU;&%Xp$M(YrnMS8Q&{$r6A*SFn#E0^?aq<%;TFz zF3d5tvuHi`+K%Co8gYYW(A6%+e#N7&cqleiYML*LNXZzxQXc_@W<;t(=Y?Ie3_c~S zU3HYRWnE>stSW=~=i@!xyT&nTyVA{%_UN1fiLZiVRh*lZ$?s$P9IfSwMWTm2DL|HR zvJ?C3N)K`Hu-(Ur4Pnx8S@CfzHqR@UFv%p6qVCjT-AEi57Yp;2Sg4_OD5lfwe+0O0^|b%!g%@@Kr;B zua6fmgpcHMTMu2$i>l20w}t8WjV_qYVqvsPU7+7s2wuca=yP z<4@%B#XxL;QnP&^K<8Ja6}CZoatMh^f47F$w31$66IBV>=wtF)`^r$n(@B_W6zrnf zH}3`=ks-Ph`Z2~TrXYLjBvdzTpxmdnD;bn3|J(PU_`zSa!s`u(JoJmcb)51H3VXNB zjN7K%*Q>cvrq1$oeHJ_|D@!q15IeU9X`4gf zvl!>Lh~yqzsZWwwGA*T5!CH(jYs&#?u=pm^;E4$(A^0a~iI-n(7xal))!sU|P5_G8 zT&T*Odx}OA@@BPWx^eTO*)1?h&HV3}V2-!~QLIO|-kj%-JR^0Gl;{xq+z@cy9uxy> zSa(Hx%G+_Vhl7W7O?dn2lKl6tR4qwvtp~b!D9x~NkF(G*>Cf2kwB=F1I%nhGpQ~0k zDd<%G5w#gReS}ujh^9oDR!jJsuR_Jb#6;2ZcQ==)h@1G+^(Z#4d41Xq>Di0S)oygO zVg=DFC?_HZN`(X1lgQ27x}LA*_W6A@YyBliV$FYAN9V(82YjSBgtPMaAsK2jvtZSz z&b`w&ZbfFlczivgE>9)@QaRjU2hZD`g3})B!054G&h&P&mCMYM;mz)P@3U42oY?d< z^(ECcq`!EuY%SYtu6d%j6ux1@uZ^g zvG2YxRyQ2Qut0_Gm3?^LFI#TNqhPoe6)fzVCy<5h4=^udx`*#jHL~uH@{Y~kZCJkVeK7K~2m3+9 zT#DqXL%D7#5o7rrPuHNI7hx`teqfi6sA|lQ8jUwEJaq{b$BW6~hSy){d`)>kk+h4$ z#DyvHltSmVw>kzv&+@Af`Ord_*)o4VJSz4yz3P6~fpAbkjunFjpeOdvEC^=S@84SRG$R%|j+Q_{7RrNCM#>EdAdz znQSux9BczyG*CiYRsz`-c=x)!stV8WM0UG}0h;sB7;-#*ivuIiGHJ|~9sq8=+42| zr3XuxQoDx?<-62)pgl9?i43i9ZkWxx{a%7^h~++$r#Mf1;-c}(X!&>qB#jNGPuUF;}? z|2uw8%3Q@yWTwV0{Pbb0Q7_;DQ>l+v@Md&WenbX{ZaRE?uG z?8EDF>nDLhqG>@maT0*Tr2p@1Q>$;tA!C>Z?=7)i zUmt^hKp@hG?Zx(4II=D#E{E~lQz5`bU{rlBqEjH3_?FK7Ro*no+3%vn_Y7|$RAc|% zYnD1CQ3_6FWmpb$Yq#eZtaC6sn_}MSC%Ciz*z7}z6<4mF7oJw5`wb^(uu)IC9boCE zFSU?j2wHYp8==jlJtl!BHUpwJ99^lC8O)V*~P|Uk`mf+=xP8(G{Z4Vxwvk1y(t>I%jbMxd_?BiAL*v{NyPJV#}BPKps z-WKz@^^B0qP0EL9zTHoUD%V9QH8cO(%a)!_K>7C=f9J4xaPf|He8smrT~m8RG~Qwn zuFPuOV!wn4kfi3e#}o$IJ6XD3wlH989TgE@%3YuhCi)<&l zW(oPSxORP^ zzR(@SF~$^cRRXVi5xzkUFBxxcp@YH}SFafnstSIaqQa{JR5wW5 z1M%o=9`*2E6|p?AKhSE`AM;LG`$(B0`&9|_<_*40uFhKle&XewwU1cVIQDtFq)ycD zE>{X^EM*1;cA{QP3;Y*R%%(~G|Hb;__el!|=?NGjuvy>Cf!-GLR>ax(L zwzvQJ`Zu4;f;qWo&2Elv?Ry#*2Zl`!!-*yWToM8#tO-bK0~{!65=YP-h=+%QhKC0d z2R=3pe;N2G9xGNH^61bS5*YStN^lX}6u~!9DpSPZDuviEASfFYXdD{o0Sd?ye0>8L z@9OI7*96K58Avj-drJ#AMGF`PiE$h~RvhK<=-Al81lE`9#Ge58DGd&YNI-z$(clKi zNg^RSx0DO3z!squoU1R#(&`3iK16dfut5J8JxG5X1I7)-(B$#%&XCQ+)hx7QJ%W)D zl=rHj4G2DgW_%Ri6#PwvQE+q{`N_kq#*!6?PKRf$|2PZDrXsu(7U~Y**w73XsAs^t zJ)lm6IWTnqB}%--G-!pWT(+FES#nU0I|24BSbB#Amrw{itxs69eQis6kPj#37pi8HB zl~YhxQ|FH0FKzmXdQv*IXhkL6sks$c zTl0%%bbK;W2H)o54mu(8ZSnvR{*0f700VbyZf<^jZ~+mV1xhF{rQ6EC)7FQ9N7Nirh5^A`3AOT$*LOeXaw;%Q+;~+sbG&P5S&9a*a(caullJ(d4>gR@r zj-VI_4~@nj7#f^G0Qw=uzMTr*^AA^IY5$~%^?#oXt|bQg>2LP6TKYA;`auLf`_)2# z81VnA9N5!<0aoyfI1#(vxj|>p7x0^3`IA2ND-iY@`S2_L@SB|wlN~&4VExhm^ebTP zBEbV$?+>Rhc?sdG1!Oipg%j{wUrB#e$(`<&|ZkT{GvT`%m3S#JqJ@dQJTa$L&onAfs9Gc``DaerhxF;bU zOh>HluO1Ve4QY6DXnYg;!hD1|0)GeGcU6Qw{>Xp=x~@5hbQTS}U&<>Gz%~MYJtz9^ z32b}fFX~??fZd?=4GRKnTi`E550rh~Ps9;0b7PQzIB;@D_%-!J>Uc{7UM^8;1-(h+fY`zG*)n`M2J%DKy z(M=1}f{R>T3UAt5kL-ChT*bgBEL@$HF*m%?cC>Ww7!gshs9HeOcIrk_Ai5jTIY=!T zMA0P)e0RNSgRQWgoM$Tf{vO5lQHC@dXmna&_70fe+9 zm9U&dPFR$MPo7`I!sf@kbpoKCo(BMs{ca)T>%Lb-1IbpyDDW4jCN6Da?OJ zUX@cK&Az9g(Z9}kv%{h%o%Wdjv`hz$cFiR!*_9)M+h5hW5$c}-*`Jbw3ZL!19N1H= zrm-Jpb}3dJYP4sG9|+rU?FCA{^3;&2Z`Qf?>1_D}w2tz>9^f+kY8^gV3eLOde0v`! z_Z11J+}6v}vIoZsYj6F!MvHojNZAPl#jSLP$B)SZ;T9Dl%;u3hLzo~I&Rk>Ho8&C0g9bN}Gu*LJ!iNC)^cCGPpyA1IwjJP{uu5Ejh0@SLOqjjl_lD&U41l)+zdu(54 zbyo~aloHav>HL8&BjrooZFoHbxkq1=(5A-yMD_zj9W4=mQ&)aY)I2*wpnrfxPhV;l zNra6V-9@4~NN+{L#RcWdtF5zY{-EhDJg>@fRDrKFriJi?nwa2~)ur9-tV&ue1Vg%H z_MpM*9RxYo9|kIR+y>qEs2XuN$pv5yhEI(F#dW!_;Cf|fWNOkhvSxP6+^Lpb2gh>v z=j}@NtSom;lt1W)pXX)cZ4|vRHHuyTQ?yc@UReS|I*OAI_IQ(&VQNQD`0UvwpMMA@}NS~rngEek1H)o+c6_(2@8=;8%mpI36)yBHD4d<4n&0-rm?Z*l$oMKP=7nk_;wfC>I-Ig- zK-$fzyNIk(BZ4U6;8bmbhs%gt1Dmt6SEE4(Nu%-)S38?$T^NkF2UNMDmJlIkmc zs)8QhgTvN3{5_~AhAIO%=rW>Mn6sK6H(Zwc#YHn7J>b$tz{w_!w!Xcy2{k%oHRl-j zK8A@S0NkpzhqP?pSrLpqsgD({_3G76QRKb%$EMXpzMxT(BWp4vePea+GIBRh1gbAa zrP+_hJ?m}dpmi^4(8qI2_PQ&%J};5`0Wr1E~&iR86)kPn79QGV=1rpLR2vOqNTQf z2cEa(QylyO1dCG61C0n~Dl`gH;olv0B3$U|f+tM&&_fd|gSgn_dC#XcedSbD&Za&0 zQ5^`7q7W3wtd|k`*(UopsC8E7KX0gy0vncq9Heg3S>W9xU*kk^TR~E$#v|Z+kKnjj z3{gk&YjD%+%r`KU0GM|(OMgXgCS-JFOq=_zW}Pj|m6A`!QHhq2E#^}cjknOLNjMM2 z=#r-~7&NIu3J{e);Cqq@Pf#KhF&%DGQz$ZJ-TSOem!U-z`!F!VZ`{dqpAW0NOr^T zYp9p~vkhJAsm$yCwhrq&l1tFi(qXnCrvKJRcbOS-hCHxQ5I>FCa7AG1MGn{} zH+RsZryU8h3Rb;Sm8zCn^V)UB*4x{p#Y9ki*RZgib4YB++9;*j>Mfd?eai#wjf#N# z8Q2*2JkPqS@0@l^Moo-Xq2(q+wp8Z3QOo@wo7qwX(i&_(M9R}W*HtQ8Y65yN(vGYi zo>`C9s2L!WKmK2FLAqKEuUdv_qEuVi>zo*$HPhgHO^p4t>w02UB+^=G>G|6F6E~6U zN7`iveh3_#I)^gx_P?E`jv_IZO!Jop$(?qR1?7@4}nPY>lzTG>CQ0tQy zED#lp@xJA>Sd-E1&#MxTzeRoUlJJ&}PtZ2PE@PX6fxMXSH4tJuP$>kP&G!-stbO2Vf>L+%EuO!qEV}%`VlPCwG>UDc zQ?96|WbmxUziBOL(W;mg>bv&(G(?2&^E#}MB^IsiwQvF9SPo3{0QS*yF4Z@dRDZ<+>MBcF%$Hx1I-UeW)Z?b&W zUXBoeb;n6#ynC5<=RMkt3u@M1K_NU_lKby3k?+dIk68KXp+Ili7z8^w z#V`%!F>*oie0fCS%JuONmV!jJts?QWQi_$P4mXQux>4gMIKPjAqivb0o?%Rr>5mo< zVq53hDKW@j-$zv|yG}EXe42ZZ4C(9m%R|Z^pTn$}wUyUTJlh6{xr|k6T`#^PFHeYx zfo7z~_cONWIALNcDS~4W%gCXsONYDXvlQF(T%?#j`vWxVe^qJLVl&J}u4;VTxkA;X zM^Es;<c`5a>tHs~D`eZkQN4RBfD7$JR2V6#&cttl$c9fEi z;r(0mr$gP(hBL5~ZxI&!JVm9np{N^wXcdWy3%BoX#tSZ@J*{y1f6=o*)73pMff6(L zDkMe86egU@4f}T-Vp61Fyjq*wJ(Q_?&F!&}O?`_`CeVp6v!ZXkBWgYs89GR1*rUTs z<7FnOoCbUEK8;?@MP?(MtG1EOAu^b=+-)anoreuhNQVZi=c(`Cx(8F?e@Gvy;lITy zDf+lsAXG;htdku`l~!BZsT2crgR}NBAmiB&XBVoY2I~Sj7k%#r`Yxs0Nu?_#M213C zMTWKywiMv_RapCr4MHY`Q6A{*)(Co&Bhh{$E3hA5m#sN(=DPz?(-T0y?Fod z&FuLwLCum}>>K8AD~~sFn0)*$$KoFDGv8rH%WEEm0@$D|^)g%ap{<7jD3>>*@ee)I z{GrpzEYk5&@azT+ls&?t>8=&L%Ei`v(qhRn=+JJzFnIf~F`(QOlfcL?o`E_fc723h zemK|u@oHgFcHy(#z^5#mUXW5Lg1YxpfWi-1ImZ;Zq|ua6G9ZMK^34BcSr>>C+W^B? z-8OJ>S31%>3|;UC0erE$(oZyINRE}M*Wk0tt2(!hTfY0AU#{E?Qiu+$K*=fnO3o+< z@rSmRWrrt=WiM}2WRH<(7O!+Lq%}D36L&12WN6{0aQorsMM3mIEBq8%4Y!QpaI3

    cM-IVs632=n4wkE3pb2#-2*g4(7w$(lI%b@XyMUT#!YibUdx?4;RJ=XZ_0X1FNx z2EB}R#(0Kn8vB)4{bhDh#AQ|D*#DFB8tD$efR7Nu4u;{Ulb``in~Rln|E|96^r_?z zj#t{q?rfr<8Mcy&1|H#<$i|Ij->qJTA+lQuPqn4T<_N{UbyAv2sn%1yT~{aTP{)^! zyt+2y{SYAu)&8F-e@GCF%2F!;5w?G$)YG#}KM2|5)fI)5XT6w@&XO{9M zHuri^hPvpbM7!HR#cd3y-RZPs&-PtWZS|L=iJLp2Ezb zBq%YO{l_IAZ38e8POo=YaH9yz5p1;OK~wDZDUZ>Zeyoo}I%4%tT=;zfl;Zq`?fV3k ztZMO#i(b<-k^<3wXrNgn%TdOXy^XxQEf@csH|^`aULyGAZ1kjrM@OK)ggR~q(sVU_ z`N}>RL@k6f@s8;4PmTA~#smXuA3OwX*&g-D4yVisbHZI#+FwyOO!>C+i)RX#OoN#g zmY2E&)~ANq&TuHT%E!IPw^se1(=bNcS}4h+&2=mQ*Pjmr4#kYp~@0YH;jZ zA&=4CmsRPb_&%5Ld(DE2LufP zuC*3dE)L{(n_j`eKfq;=b^=xDn=$_)YJ-qS66eqL8yawHG9Xbdgo;ZQ53ZnALChygmGH)p=P^ zLv;h;MYEKIfQARVxn|j>;V%iB+=%%>h!N1wb&03^7-TeLTM?}J_PX6XwaFz7kxp$qBkXNJnUxmwN__B#HJ&xd?SS~t z==Mj~#M{FeX+lXo^NQ#D+n3rJ7!>3YFR{Ov@BBaVi6EmEgUv~*k&nvwuabYPAnTn3 z36gz280_^tB7tvq?^i)XkDiO!^!QDtV{GId=v z^QLxyKRhE!pepW=(&bDk6`tNm6PLs+*y^Iz7TZd&%H=|g9$8O)f}H|IR_J&+ia;hE z860%Q5u>#Jw1mK~*03&=sf&GeeztD`#)dh)O3PCE>esrasP=OvShyQXc3#k)wez7v z)i>R?wTk-O=`YQ(%EdR^$X4DMv>jo|Gs015P4f*AdL_&OkRZwAR-aB%Yf%?Au6_Va zI8nu5k5#}jmc?cf(#ERS+5KYTH+ZWSZvK?dBJAufQ~%w@dXJK;Hx%60K2d{l+D4-$ z@57n+krk28qy{kpa4>hti(@b`i`ZxM!<9d%3Y>Tn|KlrVY1T&6j$beMc8eKv^?n!z zf0%wnbuuiI4_y%cDPP!yTz z2mU#BHdL*M!ixr-I$`g3E1W;Ts4bkT@cTghA;DpKjTrcAPV)+=j(0~~0C$EdjDUm% zt$l~_FDP{b;%xO9y7f(&Sd3~!>i7cK3nGAa7s($h7dM6fIE^4x12EQrSqWg1pOdcPF zsEE5!F9+2+5A1?gc>rU+!>sz`8_d6f}qcyXlwRg|_^~A8PE-;rP{xxY^i_ z3SP4+bw8YBhat-66aMR{%&v0cEfvvP?G#(oq9A&N+0d2-q8n>ww)1B#%$_&mdwSx( z6G|Qt(}LGgKN<9mf9_5>UVz-OlC(2gmf=eqL7Bfps!PIs%G0a;AZJBHdxeE<)VakS zv8Cj_-0CN(Gp5OPMkn%v!=5XY8X64imrQu*uW{IyINvO|JX$K@Wi{BZ%_bu4jT1fV zN0w*4$nQ%b#sedBW97w2E*7hE5zUVRNdG~Pj`SAO3F$wVQ!(;R9fhk0Y9GlLjV{hk z*sflQ=@c&`sEhMH<~b**T#qZk8>T_n7*_JS?wQ}>{bu#Z zgaO?=cmAFPX-GO~)asFDCFM2o`xEy~;@CKRooX zxp-?M^Y}^V?gAM6?l4aHg)yqrqscX^tIm)u*5=Gs*di$?`;1uE!1d<&O4%BZ91?13 z5y%hT1NCb#SdO2d*=InMh|Hn;s_wXY|+!Ty`>zQ z{amSn|0Hb9AHCo!6bH0*5*ayguJZSiCy#5k*gJUqM}%QAfg4gL$WUR_>iwMw@#ql~ z;}|i294Cz1_@Lo~xWm#4dj1E-eHiW_!z7aq)jeD~b&_(#>8TeRN&(GkC#P0}8YIAt z()|zR%i*@K#Bs*g1m8zLP=RgC3-HxG+1aeUfH-jS4wjTB_N+h+gq{UhVOe8>SwOu) zd|yNwL%h}$5_&Y2q~&A!c~K%VjI!jnbOO!&&VD}Jn83c_2b%_RMg?YhAcNOD;@t#< z%wEvRezK2ZczJk;+aW4m+|v$7@?MWG3;vH1%NPeJQll|TWh-Mn{$R40XZg&MGie8g zTr)?a9BM-xa0q`G6soGxOsAEy$6}$Yh+;eN?+9aCrlL4sw}}wu&)j+hPZFh!(aL!r ziQ5kB-GcQbgh6{5=IdwS5PC@IB^qoG}@3AigoJrT^>?J1oY6%q62&JWTloX#4 z0p-=t$#tHk-yBS$RRSWm{%pp2o(G$$acDy&Af7NQ(2h#baO&pq!*e=LKOcpZbvu49 zyuMrDcXaf6G5>d-8f2_i&DhiP>&O0s{;~UJn>AHvJCich0hJ9CaPwIzAq=u9KdntG z3aSUtqtr{BOTAIF51SNI8WRQ#HY}iEgr8f3m(v9{-SH%wdMaPycJ<*zFnl?aC0AK1 zX|sL{CO8seK}%|}=b+}KmEHDjfNNiz(ex&a*aMxT8Z1yb;!you_g3NU7qkPm#gm>d z?~dX(KUkx3%0-%(8iQ&T`iGQjpD~s?mafbE%{bD7$$_XmIW{P0Jc-&95)a67TtYa) z3{*RvxkAVj99(*xqXU$8`3%iG6{W?3^f&K-!PT}7XDFCBTnc=WcH#`w$R?fc0_J1- zpemDWEs%4>uI2sYFwAHHJydd*D5doTJ?$9Y@u&Z)Az$(%3>MXIjx%nC1XAkvz#!^b zIX<$sJ?)y};sRf)!hNa;jzMPXxqhx@&>Nd0Gbs2-+V&&fNvWKu=+M&b{xHx$XG>dM zXqOAy=SVpvZTtSPG?%VM+fa*ZrZ;*}cr#JJq*r>+ok=OpSO2ON4rA^7E!z`P+NpS( z=g0F4R@FneQI%SWha3x1Mee%gJ*LaR{F*Ch-7frMF`kE0R=jDd=t#3M`yg3SOv)V) z^96SD{O-jGuwMxugKh1o1CEVB7@De1Nx5p}`pL<3t-{0}zHR!a;EY8B z|GQIJtaS?sR%-0W)d)=Fud(S_)73eS0@nkwn0uopLyq>?vdP?;^aDlUtYUvWNUS2I zWUh#s14lp_vm)4uuV=QA_NJ#x@9m`gaWS?Hb$~Z=9x(5L9~w)|_gwD6(|m+kUXwkK z_Eo%z*ZvXd-?1RcqzcKfuVqm`&-qHl8lMyHCP_rmf(1iA!mlT@nD314-oKeL`7?0X zNQ2PLtG7Ya{7dZ(ux8-^+4hvC@a}}z-ev}Uf5_iqOCHH${*t-(wJD@}n(EVZ%cipB z-~@@-X70xK3YKk5v9yC%h2Ea>hU<^Z=A>yR$m!w3ZmWs4yihs$Hbua*ks1{IBgbV1 zeFcyd7&K}YAd$s*ZoBs@1?~R~0O?OWO0j^q^qr`gH!;0G)Zk@{5O;|Ij+A%>CH9QW zqpcdou@!qGn-aV2T@RInuWEh?H-08M{AkpZc@9q6ys%Od&?mTZq}zuuro6cj|25OQiA*RYsj3cFZFAHY*K!MY z-5Eo+N8L3zv9$dq0NP{hKp$CrW_4TDposiQ0Tw{ad6q?F@U;h21$m<5H(#0DQk#K+ z>?OhdGz2Vk>o&Z9Q6VE|)UY^pU2G#&K{f2zO99)1F#U>n`$Z)2 zv3Vc;x{!rX8^yhHU8D~kYe0JY(&X<(Tg(|vyj~hT)6YNy%%oj`G*@HXmGRLF2@)R7 zTdP(`r)4K|!XMKobS>E}ML&Vl5BCrPCix#1y!oPEyexP@5;1kLEEikmpuya=`_uhM zdbOJ|A5sBSbiFz*6~;_Y(E}PP8qIIyYiin1P1y^C4{!I{E$})=uYwy{e zRgb)E*o9q#2Hs4RNYUkAjSBq_h@4*Z(sLcHzgk2qS0j9d)M%FgB556J))SEf#>^Fr z3B;j#CEpO_Hq5~qtF?G9{DV-zMkqVckmK7A;(0vS9ud~5Hs~@#^9VxV0G-UDz3t{a zA+PH1BCK-+XUU^fWlVL&($*df(eG zh5{3!S&og6*y2>b?M!YNxWB}RmeYKJqO%=LrXzXKRxZvvcT#2}pD<)qLPNbWhdj%`xzYCtIJloKoPk6~@5p-DlCC6&gZ=op zgagPeiqGyX7^!3IQeg9Tb!D7u`)6?25*e&OeZ9;N<9U=So8+#1XYekOr5TG)`r$-( zNO|e)_6)VtC)#UJ{Ai}fh!@x~%(`NQ^deQFywCnJgtV%hjJ0B(CJS_n5`B&AJf_!8 zS1yn+q~Q9hLWEsZN0Y@vQ)t_R<5Zr|kp)qO4wm7pv@tmpsaRHotkaEcFG3C*LC|Z` zfs;~ah^Mfu-DIbXl8Y{q?)8=eA7haj{ET^X1pkr}F!72hrOoq)TCzCK65jrn&T@AJ z!s)jc*<%~iR}JNEw`dq_ZZiC#7n|jHIk-FUV`5ggo(+yF8ms_AQsqs62V*~49bc=c zD{xlfM6x-dM5+g4l^JpUfB+P*-}oGFdO~>m)>cH}9#5MdnQy`{xOn>enVfEC{x;Hx zwdYZ|8P5P(Tg;{=_>PaAdD^!xKFmjbmy6ef-9Z{e#QslMnn!f8`kUZ(;+tCuXyLn z1t!?cOFhEltPsADDPK?#dCV63lta3tgP3C4?t`ydRQQfL5?q8=wQnv@6uh-y2h6*z zUT2%Q#DaeawbOV|9g1BAL+KBQeA4Tm5?1SZ^(hX_*)}igw%c?)t6AX8NDvD_S(m-F zbrimrw7NcoKF>bu++Y22!DM+z#bZ*4-tHaG*y(&?`$gN%vU4!jt8ugGuFa(31UYBQ z&?FDZM*}S(v;PQlyoo{r3ajku!p|QjK?+^dYD5Gru#Rr>cdyn5s1$^bb#_$a)hx9B z66n$t9!hsPX{gFakY$T*`!I$k)Tf^_ql7@WjwZpNPO{;SnZjI z-7$Emwz4*dG(nN*Uo2dLE+!$FYP5HQa|G{JWpoNv2|XG4lh*=O*z2v2P~FX8QJ`^s zjMq^CR-mmIfKS`F+4`_tp)bbH(NOhd1An&11rq9>exX5D>1}jdQthGtgH3=4gu%{r zi~tes6>D=4cLU+7k)E-iMcLWr|IdgrV$ycq0B7sYet(uba^(t@&azA_!I$q7~YXHTBi zPk#X+p-{e_ih*~dUevj9q4HRD!vJxTOO@(DF}lg%{4WpGS!mcIj!7YoP4`(9x5ffm zv|LLShJ=Y3-_>LtPUQe~o?b3aIj{QVmvEE1JCiGT%+RroEk%gn%Eu2HpCs<{CFu&w`dixv#Sddj3$K&WY@MwE##frM5 zu~zqs?4{;YOshkE!}VZrc9umn@7(yxgMRlNggotWBb#JYhN*b%avFXy7M61)5+B9U zuvkwqFUuKRTNpN>*SfNOAIq?Pl8GA6%0w9@c5YCCm5dck?E2HpIbS`F@ZsZSLaaR) z{Ck%dz9dh!v!W|Lst)Qz=I1JTxc0kPuds)C1vW~Q3GMaz>%zfq+2-eARtO-C*zPh~ zT)7BF{73EgJP^ERZMSok1BF19FCMt#^f0#<3K(sYc!y7O-`Sj;3NG83M!*Uh6 zveC%pLBD&|^qsTou$Nozk*;vi2@?EEOMSB<{)F?P9hnw0Z(p;T2NTkFLZ&sEn{8H8 z7M*c?-TKNdlmqfc_@Ea(gV=&m)Un2`ry=WufKa_hRmTSYebJBNn0R=Ipl%lkrYF(%Il)gAVdRQux3HAS?(o>wMf26Hvwr7jPH{HsvdX zV0VZfm8HO5tSl}*xp)1cPGAQo%l1r5y?e~E=!vqbO-`El>LtFyUP|IFV@rLK{&vbL zu3kADV$AJn4ZdN>pJFx$0*#|cRB|=p;5U=uH<xPx-yynni()prhwr_FgGmL zgd9DEVuGmk_8b5;xUT#8(X4;Q&flAA=*;77C#g73m+^<}i#o1{_vrR{2Zszsh%Zg| zfK%?50U=DTBX@@JCTN(@2sI((M1;1_8u;P@5`{s;PUEE9Q)cz`Lg8czW$dWWIifr1 zc0D1orSqcIu8XK|qS5OIYM@P~4-Qkh)jC^j*u!9ZZO(qaP*m^uyq=r**N^=A zewU5qalAsl-+_$Q$81SM4&I3aNc{xI(W0R8zlwVX#D#-hL`kv9_OgEGsF8c2L<^dB zA1ZXa%dB!fA7<0esOH*c$Eq>j`v{{W4lWdVnxlp3XP20fn03+1bz3c+@kpK4O4$>+ zm_iAjTH|GdX%dabMVlXU@@-hkh)x* zOFCx2hMadIEH$7BkUQ73d#!^8ARN2khC0kriG!P($%cKmGLAU9b5pdbUz5$$ly@v~ zab?03@Za8kfMzjii~pO-G}nJunP%nW_|J|s^FJ#R^Z%vD6C`m|z|*5+(|7v&dtoQwSkrW0Fu` z@DS8KgNlw8l)z;OVx5Cs^%UG9-GL>N7M!qnpA69aTpp|&66m1Nu&}5>-X9T!O7_s7 z!hM3A8+ew$kHPuChV*>~$G~uTJ9WE%B7S)Wn%_9~3e-_8py5G^^#^M9!Q8-#yc&Ug zBGEIVBKok;sp=qDo`HybI<0&h5`unSS_dK`jqSPyi2V@%Ch><5$eypd`8%Wh#2Ykb`1ISAYy!&52V^H_$Kyn)D6)*@m#8 zFobaPR4F{ifPF<)NusM7zq7*#x`A*~e-hpOWH@)*;!PYE1UZ%4OoO|B@Lmau z5kL^f1$=Xfu&Dkq2?7K3-M?jMnFopqo00$oV$>t38P~{jhS9E7hj7)3^Z+w*LJ)c{ z^!`h)o~&B;NdV4wH;5piAVwPt_8F*4Fv+iA)bw@G?q8=uXHFjwmG5Zw>Ul0OMw)8p*t=T+{kiKyC=g_XYp1L2y69Cr2s}a-}9_ z+$}-S=5MmTnKg#7pVp~>u+QD4uc+f6qPw3NiNW2;%hu%E)VH5H;qHC?{($v*?<{^OKA8*CL0Yt{vUfZ>D=jS1QY=2F@Z}$E=k;hw-WuzI&RD(q_Y9gqR0>wGQ zUoFFH*`SCZ{eSaeXwO=I2N5EHBFDf18BfB1bPy{A8hj-og8bIq*Bm#V7YL+87kn@L z1|KpDB)l4Yn#G3&eKX933Bi~``$B+zgZWy#?p?2A+`t3-F!)?^bR~Sq6|^Fx1Py%$ z`K)BM2tZ9i_|oxS>-cH>`HJi_6k@dKyJB5Kd&zDbH1n(&0uPJ}TqnWNZFm~5pMHyx-#ca(A)8{13jCN@HWjC&y}4!iq)^bB!vMUa3_dNHndF< zBgmHfL(9v8@^oo%Ljm-{ry2*CQUB_dZRQ#<${e?QM! znz76ntvtlH}_ zhCbPcW+BFv#cpHZ*+E9f&Zoj#di?SHQ@Q*kIf&u#s{CA#`Wf3SD#+o)g?OsTHPr8} zBtSXDRTG{pN!4pHDFR>TBFEt(?hF0~Z$R<}9Je%UF%g9v2Bx~K4PnnOXCQpaIH2%zL{>Qr zVT+}GeE!hz!g0Hvq%>cpdm@WKs_Un{{~>NkWVr81=+sw2C22lTYXH{Bn#K9J^K(ryvr=9&m)y*ClX%P}q=2T( zL&NHR`pB2QdOG26k7<1&|*$8mc`aXgW*R)u>FY0;vq;3F{}=8Cz`xWA}YTiB-B z+9b3*)>@DIl`TM+40Lf)^(d00*LC9zIO3-(FMT@dJknXM;4k z5+OXA+%IWYYX7jF!2v8n+m&_xk)xT3@9p!SgDKnL+;R3JFMYl_J^l;e+6_dw3=v=r zUatXOcXgxHy#TK~g7a#wmHU%a_A{C!JeOsyVfOKNh*L=QVjf7sa6ja$6t#h+BRwY} zJy4bZv(U+jX#YS`b2XI%g>HB#k}$EYc^h_E!rHBead!qle?fszsTqD50s}J<5xn&B z9JDn`z(TMiNsWvs(xX*A6Wx@93TgrhC+e@uE()NI9vOL4*l z2Fa2wR+9+w1wF8R+ZGx^E0?d;#XiIBd{1G2w6pjX2pqtuo%d8{avBU>pt|?f09=lMp=xE8Wk!-3W4NqVFbM zf_L{itYejp*UAvN>biu4NgNzAaxOobhNm$q0a?=4y_Vc4AMi#a>wJ|np4Jyrjj$^- zmFE&u@5L9Ot*cBTdNc7Ti6pdECL8y%;jb0E^0)X!NMVv*_?=FRc{p1JVR%W8tzu4+ z@d7#c8mYBWxu<%^ueH}%WL8Jtf9@d>SW4$3+kzsef$H>WHRE{VJw|`;&SLAmZiBsO zjIuu-{@z^f@z7W;=K|)J{>ej?2HuPDu({I`??JM-@m1b^#kLLizHR}TJmZn_rpcV} zITj45?JudXkZx8^=J=jZ;F6juzQ^K3!r*UaReKbA@IC@xpx3^~M$S=79;H`C27??m zt|Sn#jh4<1^}vp3Vf(DRaa1@6RW$8&O$NRqP*^eKFq^xpTdy`6VNoRlRMT>Qks99k z*N^@sbl#^QhE!H&j8FOx%P;zpD~cte!(p1m#J@AreRj=N)bRj6KpVRodGNdFCI1bf zRo3=)SWUM;3oXBfLa{2Hx1^RJ-oL}*?q9AyOu9ZOK7dTKsR54wU5KnP`+80+^QqnM z0fp|`r%$-~q6V~`tao`C>shsR50_S7u37M1W?a6J8}8(a>2nWY{8{ph@@=Y;W)#oy zw%TT@>2dFq=W*AGVF>!p}M4CW+_MY5CFW%y@9y zd|1;jIT%T`^MBM%Ds02dHle<%&?Z z--QqLbTF+em|mORJcXS2f-iMNF4s_JpT5f$5uXn~GTtXg_7fjsqrnlD#Je3=VgY5- zzVB7L?`KunaZDrJoDA3k@!JLi_mXc->OG`9PavGi$r^|f&3u+LL^LL5{P2MAQI>Hn z=feh`zD9uE7+;+gwSvA)#Mlk4sH`k%*@3`1Kjg_&6v>X)c|X}&>&B`>CR>aYmV7r# zR*PD?Ime2$m8bRuD?u5&&y|R+)Auzm>MJoocyngtcwL>oEgxvT<=i-buv!(*`!2AW zErUNubj3X9O_kOBhghF`*^BT2QEM~-fR_#)`=0gEvTG7C|s zZZ**Y{=<h76K^!32RQbSqrFV%TST%D;5~i&;(N#O_L3<8UXnc|`yuhrK zR75xuh^05QVGg3?K6E7*te_Ae!?r2can6Qd6pCCs?D7BdbQeV&@#Bze#Lh z=8w*;?xsrC6|9wOA?(ZanzGaa4mUiVcz1(T1y(tL6$1UJ5tBs_mh5hyr_^&{X6mGi zdtC_an7-w4kUt z$R^Ofy7*4+um~k1?GKa7-(%1tG7eY=yVC1H9z7Z&$4+}p-BeRrk_2}3ej567ZgFBY zq!k>RLLTAiJbbtDJgW069Xy9cDfj*?fk?YiWR2}io8v%ndzF~{xyfpTEvOpYIoIu2 zG_JX%yD4EMpK$AhF5rs_Q5p(WiPTTZ>WT7AI<(G(D>V80#TR=&j5n}$`ti_4J?1El z3%6ks5C~ThPF${ytIClxT`^WYq0xSQhOI0pD3Di;>x0XbNag#NJ9t)raUy&OkdBIw z0wEI48!p}JNyH4e=;8Tx@N zu4h}tMhAP@C(K_1Ey=Mkb}s1Ef4TIY4PLup#b+L|)t*CQukuCSpoZ~^M^OtbZh*t3nxzMp$l+Pk4fV5!@G&xvZwgz zj@fk>xt?#$QrnX$qV(``lLGW_(UlI-sS0-27cXj+ud`RbomhVZSZ?0-1?D0RmbX2B zgdnzFk=Yw8`f0wS1Q@NvV>iVj~Fd}H))fF_k?e;ufy zNml)jMONY`^@Ho@d`yY+X;{0{TyCITQhc++A&vS5y~fGq8;kKt-2$(PRP3nex9W!z zF5J<0@fi=!inp-tMQ#AND4e_jyA+MMB6ZBTUqR1?E9FjUjzM423QQS?7=_gQhm%YeGEQ7do9f`2l>mH<=~pOr~M zYEUg|_Rew>=e&!6#3SUQ=eSDn74`&VZcqG?5*Pp5#fuQe2PhHw@cr;+t&WOqa^D67 zz~S|&OCEippNLg76H${vGMs+66k*q&&WEPop=UWJT2ni8fy&v%7Xy19vz$sc)n*k- zq;fZS+3Pun<4LfC;9z?Fokz64u^)FuCB0k)ZqW4n>aq%p(mt9jtz2Sgr+3qb(<;t< zCytQxtQJz%RsJc;RQ7%k09o@~kW%va7GBcB2i`uX;FW4o4FU zH}B$Z!lhzBy!~#$BphPGnD)O@2C=8e1%$uw*B(t}|LxZW8AsE`<_WWK>+&OTsy;_C z0K`Hi!=ayLsAJlDM8WOlT4Ep)8W*J2m3>vikC8Z#`Z%toy zAc0i$J68mtn4-tvb`shCtBQCe_bAU(?rw77c@i_H?5%E^H4tabVB^4^V^(VEIws>B zVIWYtP+2Hw+ukk_%BD@P@pW~3tetSeXrgCApK;2yq5YBCH8Y1io!SUe@XvMA-@-DV zdvMhm|72!Bz}8hFk>ECRY;$;DhvDSq-mC%PbIfhkYjf(8yY*Wf+1{g{rz_|XukkjM zlR_I7f8OCooJO0<3BEC>`W}uq%n}*Cu>>Eka;C8_Rku3rS;w2G6|BII;U2AxpSD`& z@8crShHeRCuALK}gFbz0Ff2$PJy)5HJmCdi{e@vmymZ<)h@4p@a+XYdyWg%?+fNtp z=xo8N=iACy=AkGy5St_>OX*jnzUXD{l7RJAg5wv=^sX%Ig&Ri36e3h7?g&=lpIs#Y zwh-Iv>zr3s;d{2^m`yqsi2;n0pi38#7RR)*N!@h<#SKx_gPQX1ZOnumBqz%$=H3%T|xpu>mO$Vc-vF@^vo*uPib{uGUUpgtD5@Yr7m%?iSA! zo_i)Io>*5^nJPkhqKh||G8owxIxaI#;{>hbBk~8qDYYqJ{D@algqO_9M?m|#`H02T z?v3TW)jSsx$xo~H0=FqLsn9ZPMv1D}gS z-ahqk>a)vFA3{owF@vEM~V=xnia0)T$LkKg@wRy*Ck9NO;FbTT?H*-c(zg(nK}qD+UwORb{|fa}~`Uv)a*kUi}-LYvad+OC!< zDtr*zX-j6Nx}Vwow}TeG9i^m?w<}@;uF{HuO6D%OyK??}i4`NdEh9)k>p}ch-2*z+ zTB->GPJh=s1MOb4C46>5KhAo-eO)A1hMrgHY>Vy>Nfd=xHGKP-5%1EccswR)ctyrL2L$593LV2Ryst*dZZrX~0iV|=r=0?j!@aUhXS#@1Xt5eyJU+2Q~}=!z1_5>KDsbNZIJ1y z$7D+O7Vc}(lFAP5phQaU77_lrz?ir~Iz}8X4f%FbO6wg~zZHV8uA`?jkc2XHF+7@s zpIOL5(2bOn6`L*~ndFrfD2g(WTg}BM$*W%5!|+I#QmI0+z3JU!OvAe*IDP6BiWQY! z1UH-d$V7*1Q4yuzByqfoSpT`jwRtHRq*}p^*p^-_Iui2aKC?+&e?v9b@JMmLem>D{ zin~1=h4Jj>A*nHx1@EbvmefY`q=)U%Ui{|%sf>@=-RMf++{2xOCWph(_O_gwVfb3; zdE$iZL6o;Oz@1lfS*=d8Z{w|ENLol9sjA3SUm@&lsAsf{;NtnH=Jp4IqXxA7}@5%vqk&B;XcJ~G-bdzS>K7S_D3+h7I$U{ zCYD5jVZd5?5pwI6b1q=ntZSW#$dPp=xyG!Ok8dkiCdv`4lK^80nB=IUw&)!CHW5N% zx(jS<2!5l7Nt=69Ey~~SrwgH=6%wn%!Y3CetU|n5Zt#3BGqs1weoO@E`J{h&K;-%j zgOhDRt_H|dZJ?24(v#Q4i!9WGLUoE3A7#!)ZAxbm#{KmyBpo;fT;Kbn>6|fAEEPd> zw>Jd+EuQYk1Bn$f4O{1utrC-K>+eznsbc;)le^0{C^vpfeF%$6s=!7!9xH2XE+lau z%(E5h0J=IW_5$%S(6eq9119DogpPN^|k*7vM!d$#;tSoROXEr&xY84spkRy->Sf;7X|#H z|EN8{E3Joxo{0K4|EVHFwW!yJ6UXcYG1eBkwe@>*`c-qRb#-uqxa3`{C3L)Q1H{`w z+0+_u@qLGVb@kv8;GQ%-?S%JJtFbCj)Z7wR~L z*UkKl@)>0YclQh5!Kfhgrh%e^Ip-{*HXi$?bgwd3efP>?<>NajRYO!@d0`@bA?@?8 zjz_a21t)2Ul3w}u-t~=5O{hO3!pu-ByP%^zh`2|n{il(oM=#1XZ~dlp#zNr*zf!xP z#OGhy7rnptqM|GDJykthT=AwHrG0|ZmM~?0J4*+y+a`s+jKrNec+MvA*Xv<@VnIVw z@gG6!USha(eE3Row*wikEfI;NySz?#F4!e4)S*^!8%c{s52Sk%3{@0<>8gRFSi8+l zS?CgP{|vKq^npSN?G7N3qtTB5gO1rkc}h|i(~NTjH^^qdI|G_+0t>+fxJel_xd_8B z2nGOjEH|NV$CQOMDlX~%HQ9oMe>*NTiOP*KV9W8xygS~z>A)0;KZ0w~&E5NVSb_;0 zh35RDFRA-{m1*kp>v0xV(t4Ji!d$aa$o<%Qr&%6(W7&t-s1K^~VZ#;tq2a^+o6x|E zDEyZUHtT5B5w2i10B`aPk>M(WBne^~6#hZ60c&9=a_rfSsRcoY+O!nnz#DPd;Bpa4 zYC$irUg=oWoJJ4K*RBN1i@s0G=EEp4fWLdpAG!BJkK?UJWBS_b=car@r`qShf2Mpy zX(oLg>^9-imEYTb3HaRr)0R_vpw?j>D=YhW^uB4*jdIKmGshu%TA$qUWv)IkS`#b{ zHa+Om_MK*pib6=%3g=ZC7p*8ZeWv$J;PGS44P?!$1f{Y8PjQQttY=Sxxb?~)w5DQ4 z$U4fvl46eKnZkbp%QyWR0)-P#|o*3d-z#=gQIdoIDaTHNBxWLnwR#rI!T-VrU zzeS_HFSB-mu`a!w4qo!U+&kS=S#u%(lHhXzhWGR~g729-O=-mn(<9nh>ApJ1!b;_3 zzbfF6a&xLHfx2Ns4kx;|uUytwRS~_c!2A8T9P{yH7QRobor-*-R4>YD<(ZDdG_Vu0 ziq)}7;UN7W->uJ_st^q?`&UKp8GVOvwG5w0mL+|;R*#KB#|#dr`rMhVHAvdTP*fo) z6!D6|O!g+}Ft%g%u>RXfhQUej#ovVCvQOPxaBc>E*;-=P|HY=qsXo^RYiqxJ`N}-)lR< zeGZ4qt~NQvr>BK6Bq0W>^*m#R&tDOy-1akO(SI`z9~ZjQ+*Ts|yc$ulTNHR4*O}d*d0r5%O%#5ew2Ax2-^0H}vDXdaw{g7@jULm6Hh({}u(iCr>NkA8fw1?B zxDX>xa5-?=d+Tv{4p6jp$w{jeDBBQI42v`b{tBs4WQ?{BWS;$ul*uN99tm7cZZShw)Jnfr4c3eVLCje&^7)PfF6 zcd+AhkVAO_IzYt@U?fMdV5#qx*Ap`v9jktE-YS@Ps<&h->cBsCZ6Won-4~4M|xJWGO|+uf3K zvGq(okJO%~#H*JZ)a~tKJufR;9$p=tb!%Qef%?@RnpStimv!>prl-5TvCI};eQKTt ze{!r3rPiR|{8!)G-`PoWDlq)(1{a5lttjvhK76=%=c6tsg3oBCNr2Jk(+->5G|!_f z9>Z;QtyRNl7uIfRqeZ2yeEZ7pim>Z^dgOn;J1*VbGcEQT{Vrox=-?o3^iP)!ZRR)UVa(&VSA*h`L{SP-C3GmJr(^Py+{sl|2dF0!2G?5v`ejNv4MWs>q*H59)6RW)$(3 z)q8~R;>!a)gjxxkGJ+lRF_<%La^ z`gOsc)%kSqZSi>DF&WVLeF3WhQ{*%jmrzKPil0mJoQw{BKQ(GIv;zzWM;~Ea;gTKe ze2)I7C;!%2G}QRk?yu&1Pl(Je$7r+UE{6o(`3HK)M~}TRMkzlzy!FIdT=S7z zR#0sh7V>-YJ=2zY`b+Y`)*;wI^Js*E8|iYcQ-H=9jtT=AlaKBzrl&A2QDEtk45g)H zd8+DM%1KUer*t{|Y|w12JMik(Wj~({(&qy6Yq8p$%EVsAY~_h$C4ve>L3%j-O2mv4Dl#nU%Pm8S1ncV1?@+T<31G}sb{4mo~N zf=y5IqK9{7WO)Dn$eK1-vF#?5(Quxb%?tNcs`82`Yw*4Sko)MraY$NEe|q!W!YMQ4 z)uh+5b8pFi`tIVwWdEAkOMdwFVwLTvCsb3@$Xs+%N%vp$Pa{0`9bahB1uv=o$&|4i zcIQw=kvVqp{X?w{UJgr_!UVl#tkY*&R?gV^r>=>!;Paj;hadEN<#kmMAYLL-XmGBb8$zi5CTO0 zfBB)#T|LS<&xgFI_gxB$W@DiJ?3boa#l`qJsGr-FB4%Th`8gsR3i2fMx_JC=9*ZJ3 zdCMQz6*6{Dq@rrQ79MPJT+behEcg|;AG{XoXA|Xpa{ybXQVhO1P5%ZqPNn{1>cwl} z&uk*SUk>xlV-i$usq8ziS>t7PyFt%K@uA%SkGJ1bh0E?H#5`ZiP5vOtYSYO(K0zmm zdV8$T*k~_YwEn~%!~J8K;(vq8zhW2RfBc@u%ln_jFG3Dh#{UasVE-RvHY-#syM>KA z;t3;!IRU%1yFO6#oA`oR$fR=b}vU%3<=zS%zvgPRBFvX zWc+Y1%twOW)e$5l-po_lkw0{e+xA+oXWce=neNB9JpOc*FFb>S&-DAE_MaqHiEOKW zm4#djX%LcDUN~b)yz1p_N%b9Ek9yUW%DGqulNvp0aCqUrBP_ijPUU_~Ji9&4+z7z9 zplr&l;?afwL#9^C4523X@yo`e!S*|*ljiWcuu=9vGp?ke`l@|1xn-B_mR*w5ps|Q@ zCFp-FcNOryskJxc)m9aS>}y#c;Qm@+;=;p5ZA|4iL=%imLLzoKMOerL9(<_K*?QP3 z8At~jtPILH!usXH1?Tnac3Z?wX4*POWNLl+-xYJ)wIhx#+&IOC4VY!PDm-p5#0i{Z zbl<|vE+c%*?(w-DVkKUR8`WoaB>y;I0o-8GV{Gr2IH&%R|C~qL*lZf*TsYDd4>VhQ zvPKArf8qRc;4G6t)dbC=3(aR7RxilDP;kO2FqS#)Qq+?L=JOQW-zl)`Gwto<_{04v zRAIi&(i0~;l^G}V-wEg1m3u*7H8zxdE^|06 z0{!Y7okJ$*zmP0`)XX+DQ1(aH56LfqON8GC6<^|fzOwZ^a1*?_)t8+Yx#vsE{=DD78ZSgUs4rK4ZZeVE?A zzocUJ1JS*9RZo|`zgykmuWMKG;mFnAz59;E-ae@H=Yg@^-{b=?c5?hIgE=T%o4GLlex$XDVdJ(HwUKAfeVy~MB8-`twU9-o<${8P2iqh+-Z^>; zS@aw~O6*`d`$DYYnM?(X;`Mn60@{p<1#*);_60FK4O+n|%T>U4M> zlx{MR`}S$8X?C=ugFu@cpYPQCQ&&P-mWRd6jGR{TnSF*qP>05+rF@PCZ?z@;ZM}kn z_N;zP^NbJC5*#?>HvIt+Uao zTe>DEkmtVB$4{SA|0g}sXwx@da(`vwiPj(bJcqAKp4*(y_I!(@LARTU-wx^Tnn3hXWp9WHx#tBUk5n( z`0n!grnmj(89nXo5!vSp>o%XeRFmfLVf*4rv%K>MT=ugbX1%{qkUitb?TcH~7xTJp zc%C3~n|bk5zHXZXO8gslTwhXj{^T)RHo3YpX`()_>Pv5Zd!uR+{PR$@(>J-k`4

    p4iRV;2`31ac1oP*XalCj+{w*bKuAw2LX{c$CyM*;tF=0 zVfZ1_B&^>V;NY@is!7o2n4ImCywohMZ-sBDU#q(Opwte#2hR@HZjr4#^Pu!Gd+86( dyO=>&T#{H+Qc(m9EMpT(OG7SIRabvEE&vQMJKX>P literal 0 HcmV?d00001 diff --git a/doc/src/week42/figures/nn2.png b/doc/src/week42/figures/nn2.png new file mode 100644 index 0000000000000000000000000000000000000000..c402d795be94abe50e4699beedc0cde7c5d9eb96 GIT binary patch literal 41803 zcmZ^qWmFtp@aB7P39ccy2DjiE+&w@D8a!xlXRrXl-Q6KraCZ$3!QEYh1>4U1Kf8PO z!@_}Kdb*prx2kT{t>-tPDoWC5C`2d#0HDdrNT>k-gdF^P6bS+RO~_Ji0X~o$%cv;= zfCnuAKm!5bFL(&L4*ORkRm3KGT4c*ME^Uj-$DWa4@6A7&clNIfu|7D+{XNwtF!l@ zAf_2_)&8mCY3%qRN4N(mA{r|KDVr5ZwRsV>v*b#h4v5+nsO&NgZzntC44*(cP&@wb z|LIejNYX+T0DU#5@^t!M}B=^XYuF=tHTVvpiBr@ems%_ewpUrZ3ru^Z;#(`1x~!lVI}-G z0(>qxR9!2S1agzl&Qf>nn4)xZO8iaVa1Z{Xe5Ke-sC{#O)k?Hs;zGs>B|!REO`<@K zHY7-#QxgZyT?dA<)P*-)GUho@%*Lmcg!EC>KRVivTAQH@PQU2)gG#dpDIh6Ml*5Yu z-((@LAsZZT+L;d)aeTo6l5Sk9f-3)#hTPMC zqA5e(4tI8ZkMme)`=~O##e@mVN*s{*33JPZs+D7x&HVAHVwY*Gzn?KF7}Iq-aUHV8 zLEDrdNW!4IH;+W@)~$? zf$ZmxtFgYc1THJ__uZSP-ed85OB@_daX!7Ga=MAhL6FEQh``KQHZQS7()`FT2}B|2bELiUdENYE{TWDV5llKq-qr z00%Wp#5^c#O9vkKNi2#s7ufnpT(mnc1B5QyZRZy_j`}FC7?ClEQGS+m>H}G2_8nVGM9@wB!h$9|<9KT8>kR~L7H6J+vHS|n;Ig6xvg=)V zRz1g;)+VY%T&D6v+Y1ZD8+>DUveYHTXQ9s|A<>UwX`?qI;jaydT0ZyP#>B+fx*hM1 zR&G`gWRl9rg!mQ}6kPX@Uwx*g7IEwjyx+-iki@JVDtM<4ywy>D@ z%`anAxpSWj!N#s|&o3%6|M>MQ*7vZmb$jZ4QUWBWywXyl4BXPvH_E*O10pMn&G%!Rqtdk!GCsR^ z7m+7lzit)&&~0L(Wwk3TxbkZE)~@?v^O7TO^TO{y%@#G4K{Y{vKNySf=WKruGS8Bn zoLu#1i0+s1NG&PVV(a|}!N;nWhRVvb-0|tae^asCSl#|`%J|y2@R~4YZ7qCiYKJb8 zRH(kN>?i?o{GG_8irSQy-TN*)gSNfQzQmvk1*=)XZ#-*WMb$NqOEa9KzxMocu|);b zUaOdkmAx;_EFkqsKlFTVkg*p$3I&}JxK~a)D;pYa<+BJj(*F4oIBHV2jUYOyH)*=N zUWhO&jVJXjh|SH-a}mkLF+&cdm;~H>%L_jcQervM+>?dkgypU7?65m=g?Ym4AKj## zzkZD=IWH+IlV&57Ajb?GxWa#Yru}UPoOiCA3wXK6#Gb2#DG zXAcjL#0y31K!w4dz@~pMqb1M8_9N0puV|qg3E?=&mwzh_fta7<_H{SrDe`3CQMbEm%<{}(SZ!U zu~)uXg6^?$Z|wPTjFE;!mL>sR0$qEbtOd&WdlnXbxZoaia0Ckr)7^J2yx44ZtINr? zwKHYo?~G%LTORhUldK|X+Vv_JA%L^9{*GOW9iUycPE=c4J0>*km2m<_4n5u(+$)ps z-spY`U$Y|P5y8nyM$kaoeHnTbOfYbAaT)!N`*i-0*6jr`bLzuu%2QX1wd0>|O__|R zkE?uIoMWkz>}gnA3v6xGRmc)nyuJHaW41Y8y;?T*&dv-pmy)%$g0;&{P~zRqLt1o( z5E_dXq_i}1PDq%Fip0{ggiaVSZR~YH2uw(tlKKJXpRZsXZ)Jt?E`NJleJ(TLI%RGN zU+Gb)F=kG?{82e@d#|@Q{5yMEI1rtbgbEHGa1{INg;rdC%~0x2j;UcO$>Ii-j2Zh`{wD+W<#ICM(AS_fj2nf4<*C0H}uy zW`F;tqB6!lc4_&Evv_js8{8AQxhYv1Wz^}(@&UNk21vfL?6=DhuNb^P59Bw1PJfE6N$@{I?rWrcqY6@NsU z9J4oqXDlh#V8d(a1UZ?F;_G9InOn#0V!9w@UK0|MmWCA?6_x8t zF(c~40f+&8`1OLB6B_CcHl6l5v*Mu#VMa@MRW%DDE%n7g5%Y?|S~GYf2=%k)5K6Zw zT-OTyIuKY!kIL|o&z0-<0Px0Aflw)e%zGx=LL`mHuJ6ymM|<95B!qy$U%%ixU0R!) z|5iyNiO%tzT%Cw3k52*x;~991loSPc@y#FH*1WyF2f&P_sfO{#Eh_O$H2Y1K&)*c- z&fxgwaW`R1ih?wV#+S^3>Aj^QFGBQoNMiKD^75j6q5CX00k#qgo17-*RAZ{zIEvUX z)|57imby;8fAmgS$>!R11$jP^C=0Y7FHobhu-1aBKp%DQa7sq&6`E>u&E!Uv>&)*H zgihL+WG8Up>>|Iqun<3s8Qfb6UR`akp{0{z^a9pD z*mbq!2q!Bzz_sF|wE!57G4oD>nR-3Lx1F6sel@jX?Kp=@HCXDV3UG?O5q%0WPfMwW zf${uZkK{?fAFIbBj#C`B%RYjJQG=@WRPBX&f?ciJTH6%_OfmlPsi9)b0s*0|0R!J3 zrKO9Sy1Icmm#4Qpd4;<>IMPw14?QWn%-9Ro^{H#mF4H6|bLmE&d+5PWuI8z%u%&>? zV4en_VF@R|S&d8n{@HU2MQq!aR_y32q&UV)KywT&&zF%GT}+O%%OOtpF>60q_)Vv^ z+{&;h4c%Im_uiu($% z&Z@LszU#GUgV8PXg6+(ebiE}LUcR2$%VEfj_A2TzT65isn4>=U-6`4Q`r!jtru%Lm zm7bp8$T%l)7%3Rm4`2Abf({JaAk@eiVr-TMu(WPp-a7MOtEIwr%9bqU3A3)HB=1{1 z-Za>e+Tnc@kLsngwvxw+Jo#DE_1cO}iA8QDp`-*gGo6EG%kyYXE1dNBdbWHC%KSjsFf{I7s#DRt4J*RS5-%5oQd-gpl%JXE^^DHC+KT*sO=4AEh z7WU+XHDf?gm?wGn!~2ql@ZkJ=ULn+VFTl7f&={PivB1E)Zar zF*F|P%s7=eLR(@R|8(#%&;<<3aY5IiSE3~g8$?OLA2g+w}Y~YjSvnfv(uA z+tESMfM*<0R%n<>xrUVZy8!AK%avxd;?hznIk`)Z@}VIdHu+9$Va728$d7^z?(!dM zGcXonU4f>cPe1M4^z)6OAHgtE+%?dK*7?dE3v@l%k`{LMi)O?q zG7(jmlRB*h-3q?ub7~lj2QXf*qe!flCMVf_xY%wefUDZDhz?GkcTVVccpzbs0g zOpThOEb{cDvW!tiT|Lgl_vYi7nc-$oe6u_r4du0>Y@FxG)9}&xiKklaqWq{-|STX>3NVXh3r_%o7^ggK=DJI-*p>clYvgn{Ep7 zU6LwEZFE|%HvVzdZr|H=rb|Q=2sNc{8GZS>Y;XZkdT&RGvxuBHsNLj`dHZFUfl?Fm zCz&W#KGQ+SSsym#=BrranuXOS!L!S1k;YrZDmv{l7|<~NJyAm8ZCS@BV7Ao4=x2%x zjGS<4MY?zVHtJ<`OVI(@Zvg(N9=cq5dIB0(FBGRkW$7761sR_7D+_}J`UKE4Kw)QG z4~=|o-@WO3J28`>hn$Afm^>euxE>iUMF9p(Bf$u(8VarG%3~9M%CxY-pKmx`t_~tm z6HJ{^m5?eD1uAAvw?Z}}-es?=hvdK6cQuu8_UuqsdS`?8G9_tO6GQ5p5{uuKN#?0? zb#wcp)NTcz8&LU)vj4D*G5BWtA=IX$lE&;X4KtVa&D)bQG~wgcy(V}a?}PA_pE$Dr zlWn`{V|Q*)^8j2+fRUK&^XbL_X@7;Q3m55vb1$}1#rsBG3;v#V<8$Qd4DatvuLH?ph{7fdy#@~}PEk_HH0`2wmwZ=fstAo ztzaSH$OiNFGwq+=nNVdt-fJ=OjjgB7u+vjpaERwV>wT-rH>EzX(AYj2LLPAU5y*&5 z%I^L;1zd>f@@q+tC3m4G+UoSqduoRjyw}}`S8u&H)$6HZnAkR8fha(x%E8z63m@dt z7f5uxDtu*qp_P_CH#nMr0@#YlmEFS-q1dH(Q8ac1b_zO@?kfV^0eoh@7=|cNd5BkE zu1Yb)La@V?&p^dAovg=NTUfZs@T^rI6^BD+^;g$!41vOV8nP1sfU+(GT}3y9`2IB2 zJR2A&!nV$V8NvOULIK#v2_%UaKc$GFVm`mI{X9g2p8+@oib$ihB|+@JDwaf}l6{RR zmg7|50#^v!{=UAp^O-H>J{0MiMDF%REL1!FkO)@W-s`nSQlRSO+D6;$O3d2RJMA*V zwSnOF<<18f#4DV)5Bh~DztK(Sq%8dU8CDcu`}*BmXPmG(JRg$tE4ll&Lq0oGtTD2m z-&s;fxC~pYs81x>-`r`P0fPwSZkdJ~M7moPVUT8s_0@j3Ni3n@ZMN7=tN~iOopRuo zF&l~;4777 zO?-1(e6wR0cla0mBwh=)Dx>^MnH^kl3UP>p8K$|p`Cj}{W!vicc3~oI-#8Uq%xk<6 z`p(HbX>%%*E2VYR?OF>#6ua1ScJ1sPLxHSfTpxe6+j31CC!b72UUx3)y5&+?O1&Zk z{0~l+kl7kfs!f>fy&ZcWQ@W}&q1&MpL!zbLw=O1rSo#oJ!9GfCT3HEuAKOS2i0&AC z)dtKiOdF|!j8X(;DQ(~f_zopxfR618PO_9RxWts>SJGs*sWm6wF$Ky=_~o)o-%2n3 zP~s{ttaail;L_|loQzFD&~WRTLH#>Qsn3_R{Z=?D{WahG#TdGq$a(I%6+Dnk<)Nm7 z#7J}0&D0BiZEgUQq8`42y^fwI?%}#t;U?p+xUIXjzg{~cIV*ymDr^`D;JIbQ}WEm8jh5raZPft(TM)yno%RXbt5et@6OPYRp z6{t+xNZ;Lo?amM|IrgRVil!4TlH)#~gg;l7GGi-7pWn7#U2Uf`2T3nbhsN50o7NKGp3%*L4oYU01WTpTT6lgdYh^Hg<|NQ?q7B$i}%yD zHu4h*DkprPDko|)y#y|Fdu^{`@q0_cMS@MyqR9-BM>hCHi%e0#Lb|Y(^LV{hFyl?r zVas(Jqy2Hhh_*-rUyWO%Tr0RX(k-VebvN*Jj7j8fcPH7MJC!&D7YRYX;%fFfY+Nxo zRiixv%xrzzU%pg-i+gj7z8UwF)w`*(YM_kh=;eUAVa#1J+?77HnY{COWsfv02hi|` zegWdEYD#23M)Ul)f2{!s5SS7`pIfY2USzKQCh`#>(VeTZr9w(9+Re zR9JP^Wo^j0?TnXB`ST{g*rRKP@r8-ssK(_0xt_(YhKAW|EotRlKT1AYe6dthV7>5t z_vdD;VZ(_ZV|{|AV{=M?;fjiAGfwsRPXg^gC=utvyEnD>IpSOZ$2}#EPP9wTek%23 z;4JgE<6rmHi-%qtOy%oa8^jE{->B{7shNv)9m8{EKXcXE(wK}_))5oM#jUCH{(^96 zKiJ1);?IOmd}H5-jx6B+Bzu+e7kcdSa5gJa*G>kYm=R*cym z$upP!EF%L}_kr}VVVECq)I5@-)o$(4vZ=k~n3<@tx^3dePKYbsK^9(*zRZ5Ja8``% z3Y;AkyIB@wlz2P+kybSr{#6`CV4Qt-fWew%cXM?qc*eKc2r$bHm1rF^)RK+cBkO1E zP5t)I9YyABdfwwVAuGY{v20kn)$<*_mZb^nsarC1#az-~`(~(&iE5`~ap>f8R~>w@ z*YSA&)MH16{>EBS_8YVsv#MmWa1*sLAS7EWR{~mAv)S@wSU6avC?VWbM(?c5)`CdT z`P=YXTkCU3!c-2W@}&SeC$=Uw`FC^oi{iI5$vg_XMc=iWK7%tT^9l_n*^h#fPEIlt z0&oK@#lp04x`KxYOK#sth$I+>l1uRlC*w=B8^=FtnUaNqMMQJ33qnBGCvHu<-c@hU z9-juf`ReoCpttla?0CWgE+?HgA~t6yf>R}B8}@1S;XxnSF_`?-VAk1^_GvV&b!G2H zYRJ7fx-z1sT#FnCj^h<7*_dT=^?@}&N{?#>xfg%BI0evyTYsDX_uW{fedq9^@)nzx z>93ehHfcdRI;amDTjQG48=bk0wBTb%l}Ei&#y!=e59tuD0L|M7FTtKi-$$NbrVRL} zS52r3(yw}^g=XT?(_5brO+^SXpR+A2mdcjSkEDH8L{gR9aLE!S(luh)h9zYR@VhMp zDydM_+3%G;Bf%Aj+g9m`%_42H{jrbxiCGu=CAIY04%Q8x_x@<3>13z~c*O5y#jMC> zMO?ZLCidXTA+k5G&7{&V)mekA`v7Ysk&-=%WAuhql%))w^cx!1!aZ2mVG>4>ZlO#` zCE7XT>VvBsl%F?`dl4TZZE|6UiFVdua16+?*Pfm`G9=L^>u-XE@ zBV6U)hsAx(!V4#_Le@hH>z1C1FO*BE5>w><@fSZ^L%mR;y}qXAD9uk)#d0G(HTBU* zdAyp()59Zk`sk(4b)n}mGh&KU5Mlixx^yV+fZfL6 zENKEvwGB=<5(ca!(4H(uecw~(AE+!_<67a5J?bRZ25Sna! zsz@j6)t1mR%^J7&(H)|v*Mc;*x0T!cytCQNmu)|-sNeyW1N{ZQ-cv$VBeA&G)YE*c zAHjW@nJ(Y$aI!xi{9v<6M&SO+Nf216vf6l-teMrE?0cOZMNO|N`o`-+P2U@F6zjzI zLZE`E6E6`?dP6r_ekLi3=r^A2zszddyvF$A2ItycN|*&6#2R%S0XDXq#l@B4vZV;u zwQC36?J>%e&+$*MsNs-&VVDf0j?${(c1^pBtRp?RIzn}mAb?hYm6*pPs{LP3lprwh zc7A{S=YYU)_D9O#po}#6L_~A*$~o8`8~1&-GO)7pbS1}Zm0WrWU#rx;?%if ze$NDT|LE3XB#hY}S`J(0W&lB`aR55MY-GjBLZ?w**@I$%Kl~}!i`84P-Dw*b7^pls z^*L!r8c!U`Qfx4h@-T1eDUZOyV|y-E>>T5E2^DM;0@>1 zW%Rmz_QKh~>7tI;<;B4KMgN;8W9xd1C}LsVM#t#B(tyM7@6DoP9ci;TJL<}>MLtXQ z^`?8CHbH>9#*HoNmX%d39+uM+@fVpA@o7B$z}(lln{L{H(Yrc$hVaoGO109;i#Q}d zK3jg?=S9BC0KM^9(eq(r2_Mw^xYeTACL_(~D)`&RFj)c(m6BlR;nv>(9M^T|#uV5_ zx(Y!A2KfUxPPqZ<>J?x(p{g{5SMe_UqNQ7$An3Iuv6FK$1Bz)L!IgX%K z+aYs;29d;suUTR#{oxJuVm|n@Wnr2-$PK67)YsEU_VsDEI+Yw;H^V{%o6XP;T}}>3 zjB7r$PW`?%oEd*N({5<;Z7LEEAUYsr2(Af=n?tK>;xwV4*2nFx_A+^^RUiZwMnqT< z{${}&ag^!;mXf|dSiR{PP5Zu|iC3x@W4Q~qoQu3%$@n^42JQ(z{Ruj|d}!=+d7%Y( zR1uI^4c~|Vtj*(){gG^^rlwC1jp~dhqle1(J;*3`++(b)2=V;WKBX)?GzZV@`h8z+ zOMN;z3Tj$f%!vvFf(~v8s)AnBGwt#v)?Q_@e~F`>(!RJ@LpNj)%|zTceM};FtJe|I zan=2Gt6x}4i-=n(ezOXoRPlKD5I{y#b zJT<|XR@1ra{PyJODU7a=!reYGWAH5a1kGPC8ylsK9JNQXk+Vs zkJ7V;4CilT=R8k(?vBFpBOyVd_I65GY9elkPVQK406E_oGnm%9EIZ5svJ zuX@PRcHMgAlNKNQv-tCqM~`h|mnh&8EPmT^Dhl*n?P2uNQ7>P7cDy?KA*`&?R~$eN zQTl+Oh)twb{;M;a=11;R=n6&d`NnN@_rrFSVVE^a{<$!l3M7#+H?{V^SO5)r1Bmv8~^;b7jo3EFj?*{MAx#>7PSh1rEFDmcvmPh!cO#hz z4TZY%`|k|9$Dz316E;m{MG1@|o{Pjc+vU%k$bvo9@#G5{@eQi0lUtsH%QnNVx@1l| zJ8QMr>Eo1$DJ;0PfYS5^F)u~8x6X4a&CkMeD+Tv~jpz*xYxT!;BJ8-r*UrvhD{Xjh zudGy~gNB#*pA#u)neWBjzxuR@o$&>t0f}S`z?^SRkJU$}4Lqd6qrt{`+?DCor+h+( z9yxbENCN#3H)X*$6!$3^8G6bUU@Y`2z~9~zMaQrsqX%bL2`T`5#4f~wgU>81EZw!1 z>H~B%b*!5o%OcpcL!@5+&}7(CWLVxYmbm+PQ&!9t6&suW`yYD1WhKdcxBAnR@$%9E zRH3U)e`(^aiQO$)yJ4Rrk50iNqsqxiv!UXoX^y@5(0CM$+3GN(z8)O_c1}S#${&xh zoh;^mWagNA1SleDBf-C!{b$O7ZSTWfc-eth%?`pu!*sm|G73MMOfNR)|AfEl*&^~1 zR@m&|x%0BPd4PZ}1gnUM0R!wd>JFxHnjhaG<1isADwP0J@e~?RzK}Zlg9Jmf)Fl}t zG|4-A?4U$_pK+;Y!Ji%i-$caCz*IiP=XlwldM^xW`z9s|9FN|Y+p-Kc*?D=>eLZ*U z#+0lQ7?jE~J;UyKD6S+^0fee3%E&|G$NT$EDVE+GYkhqnO0HLagg%pQ#p#ecWJ7C17?1^ zCv=#$lSINTd!CV0(FBC{)QiJ#Q-M(g(02Sa_TO3^K%bu>Z6b39ULqKfUb~o zh|?)%hW~Bm2Rhp@1flJ#tCicIjw8R8a*>N>KtG_=PnVI7DIj6toVzjhj^c0ZO-zYF ze-zCqzIDzT=d^!=gM(8MiF^M)GPzVd7$ug|g*Q~&sOtO`MB#BP2Oq|yi|vyJC=?!I z4p#%eZ4UoRvpjVBeF?fV1uHATbdmD3kPn#6WfW?C_5R9@h1Z+-6Qxk&D=I;4M0HH$ z;avivz0B38bB<-+M6QF-gP6D&~K>DM!R3AD}Bfjud=hdR?c)nn+m?WQ6EYaAn z4#8m#xhmq24QE)c>)ng}3Ay;;NZ}S#zu&y>fAuhuo6fOTS9g7WPlOw%5M>uk^}gJ&QyGxMIz zT_`FX7gbu;YL+psnJAxVm{ zNk+OG6*MOZe_m-XW$^5{KG#+1eW@9~^@qQT0oNOzRo=B|JE#r+zfzod*W@r{&kI9( z#|s_$PuIiof14GZuFB5Nz8LN4tw#b{y>B_z9T6(~`t_afxNw-|qhL9ZnvxV;2tMiM zfvOXK(^?4$5*5PURnQhz+MlNh1OC2(V$eH}hu`Qw`s9*)ofo2{CT(VEwpc7IEtO8- z2>X9AgA&>dl>yVshcv^6r4NO2U6}Lr1?s4 z;c;e5h^QTs-FUYe86|vJ3Z!@NPltaAVYSXGYizW*pDG9oPB?Qu;$CmNZA~L0>RAkC-3i{5%(CM6Fu$q01^t&~P_^_P+8RB9q>86i`8(KSNTEK)r+Vy3Uci z&-3(vuQZT~$8Poe!E_C{Dq+1sh)E)lQ!$;av|kHOanyV=8ryc52t>*2t)YW+tPgJK(eW6@=sY;a2?3uB=X>U5vR;aAtZ4nOOqIP(I4kvZ8^_*cQ+k=b%Gpvh&0OR&+OZDD+#KP5vIaXa= zq}Jis5=S)MLuU(LpuAX!fppMS9>r^T))98;gR!xE>OBZ3zXId} z0;oQ|ahoRP<)$xIj)s(z(qvrey!PwatA3@$m~=f7T=YbgcZ~2?B?(8wjIg7>G>~Rh z#U-=6XaB@9LH|Qgz%84MqT*~?csC;LUeIg)%>t7i%1kMv`E^yqJsB?p7!c9J`DTD(uzt2Pb7!y7jxZ~+NG`m>5q>KM8N&xzYRQ(Aw76bjdT6?EgRx1IuW#@P z6>2Hs65P`al$MDb*qz}s4rD6fyjqV7i&i5T@5g=4CICkKl@9qTlZN4cCD(tBL{2B8 z+RL}VquOVqD3I>_+ra}A%gBbhpKL{me1r*f5C)1zDnSql$R74%&qa!>#(p)f^)=pM z|5Y`nDURg3nY1FldJv@4s;T)UdbLSJ`L{qT;KSC%F#`r%^M*Y&;iYYY9rSna_03I? zqSUj#GAGwA$qQSDAz>#Vqfesnf;9Ut!PZ_)o4_x<=?1vmFEvRcuPgs8t0c z_PJB0srgnQ^46L`&F-Yp#?sb8kdWu|!)ja7apYwlQE2hqb${jjf^ToWeGRB|lYoUn!L9h!vV5x#e>RfzD=6Um{z1jT= zleMR~xfJE=#l_3KY)Qf2v1uEBh}vBWYb>5&73Lb7fGjpBHng;9p5SAaHQ$Rp-ey5f zjmZ$w^7VaQq^e5SrCcgABDj~5zZ*Pm#^6ZdsIekSy)^30*OgT?*`7;S`%I7es2^grg4hh(cNmr^G9B7rf-A=dy%0jYA?-k)AStC?w$+8pxQ}lr@PvuO(CWoWhQo=HO zTpb3+PZ1q2Z5t1YF( z!J0q{Y_W2%7BUMHYNsL*?Gzee}Ax!j=c0x!D{vqMj|cjQ zY-pkq@$$mo5`a_)CdmfQ>{&)M&6&Ru=LguT8itFE|7_0!&M=srCHizojNlqtrG;+^?ot+~j8l-_pXI ztJme)%j)^f<9#2jlv$_(*m?MeYCW8KRuK?hsmRMukxV&;_05OaSDjY{4HAqo0l|i` zQ2GPtzzpg#q4Z-E(%Xi^U8egda}q28f*h-AjcQS-Q^9dD;>Ts zMVamG;YVydDtn=#=7@L4oXkqLlO0^5-2tl>uA~@!u#cBJf4LXf)R z%*-+cxIt1Lg-p9&%4aJAtX;@No*R0Cqi|ssXxK#esd837eE&W+S|m5@VL@NYB|`H# zH#NN%iF-0Jz}){TV_dStleD9OcLHreL5A zO}`!+XeRznD`O2jg++cu=-gA3mDQh~oo(;kNeyUiO+mru&hCf2{4zMiBh(>pAwob( zl7J7lTfH4QO#hv3j3iambn9r?y{CELN=NwCz*Lwbh&$~hWoR#hAkA^Xw3^d=)W4YE z2pz7fr2_rIOZeDrD*~sjtt|^al9>MIp%c0%OBqbah#`iv`|6Idul*SYrAwa$i2hX0 zJ|Ffz_dod=vbmvx4fzo0cOkpzlF~H1VMLkJ?~};&;;S%;)0b%b4=NffZ$}K)V}pUe z;?Js1m4h3sh2dJ<1YZQG0a1{7dk-uvE-tQZE6pi@9TDB){n;AXvCkW9wh;R_$xzKM zhsz^3%1))?=$nv0){y}Zl4NB`ll&uQYyxO7q;()mBp7+%u{%C~yA04>o1Lw2lBrs- zZnLzc_x;SLv^NX6@CBAG;HfL$C!rx|U}R&FmqGsGCr7{CIO9nwB8caZN1oR`wS&&*UC+wjzbA3b5p%9}%$j2qXq zJpTd1KRaP3P6^DvNvC?u8y1Yo)^RWl$I|a!yT@pnU`Y>@VsTYpeSy|p=xK}KZ zL`F&!UF`%QqPI53O;I>c)=@*TD`eaP)jY2!Sehi5@v}ov~p%StZw#!x0dUsbeaf6CuN|zQ+g7 zQJ_m*{E1$3hnsV{$}<$HT|M88o#1o-=wj_v>=GAHK*V&aQ~`3FP61%*{tin_jtq{C zTaC@4@og?f$Ot)~kdV+XpNlvHLX<2+*NO9%AW77VJhDd%CxdQ8)5b)Ai!EjeZjsQ(tyzF z2E|n^56TV_XP@cWzZ#H2rH$ce4%Fw}fQ%|QtaRH2xGelp505n&ACslqRqJ1$LFZFobvc?r1W8^e_Cw~@5XmIzHp z@JrT_uz!SYdP7pTc99&_GRgS7SZD%}6Z?@QGi}*y?BJmfm>gXQin;)IH+3;3vLU2W zjnz&K{|E+e6Klf(x{6ruPV67bwxKhx3&n*E1hj!!N>3S~!#oOgNW8sdHx)fS{azvy zccy@+$U@kkS|<>DvCZV1I$vO z!sIK<;-ohH!Bw#6zAxL`ijgmQa7g}>o;32zAvY(2F=2J5931Rt_sfI<`uX$c_L}`f z$#2)6|A*yL^=o^$?anj|7V3&L#PD=IusryVZ=zgfJan7biz)V;_JAPK=!K^B8bgln zy+2%F8Fpkm3KJT9KzyO_!aiYp+95w`DBPA@o^^N;#mP21r>sO!OK0g=pwFN_yrJ?t zl``rXtCcX7yN1}*byRsQ>WxMw$aHUeu4H(v*kZ04XIx+)!*bZ2zylE(-G{-IRQOmWVGCw|*AjLr5|T2@KnFgGan68B^OS$%M7b2wh1!k4 zf##Te%%Gg*X;^i7NKF?U6{n2SOX#y3SQDzt$6=%4_z#9VnC)iosK|2@zvzgUc*2;n zm2rW+`~7VpnR_(b(awa)!Sf#}hBTxeywfyO&VPQ!CVgrRrk9N?()~+p78`wzaw~V< z|9zQBL&K>`Pcbw4*ZrQtbIbhbuRFrG8D!b&eB)Vpm|rZ^jb@$Djb3T&M|#jABUjod zZFn+bG9;Jpg~@a}^NKGjk9e?%1|iT9&!fz@E2VtLT>h;Bd(Y@oo73)Ti9YD;fpz84 zC}K}42Gc?m@)3gs@D!$fxF66xF_Vp8+zXo4kBp~Jo43EP4SEG1-E(?prf<7yQ4{CL zGNkGy%*W|a1+p?AWhvP;hXSxk|H-e#)K>TK;O7`0Kkjj_EVCCW`TCwVn`M|O*e9Xp z`(5uC#Gdh;;ce_WJy&Do>+mDPVwjOhmha*pz<2+lk|~bF-ZbCr6Ja;A@~KpI5|Z7QOlsCxOJ&=M@kH?IRx$qbIJ5MH?M2SwACAAJF1m z`x(mog)A%_26f7-YSLPHjey8dchLnk8ABtIJBvcp3hU z8BVq!SM5N+DE2YtrBI78pJ6nbII#(6!xaUxKi$6&KqcMd_h>2B>)70I#R>V-+N`?18>?9Bbx~JBYTs93&Z-mb$?`krwZ!#^Rm7^yCK5$B$Km zV{&a+*U3q~nuEwtnGB%^Djb(+a?8mA?Fd8L+gs@Y>eO!KDjEfZ7$VPX;1bpS%dYju zoy+c6W|_tNZbUe?mx<*CM`nYx4mUrHyr*&mrltGo!J)v76Z5-!WvHcuG9eO~(dvg+ zw@QYXeIm*2KTFqQ%^yCycUJVx)5FtJ1{$LbjG?=LQSv5axMx-Lm} zJJ!dq_@Rz|s#dR*(8T5A;m*|!pXp(u&xn#IjQq7cGUug5Scg(r$`Ith@#2YB z3y`qAWlNDB16~a(MF8sw_I=msH{UtqY9Q6<5 z5L}%O2$R}0#so8U>tGG$vCubU{r06w!?~g;**OFBfZ{{@*nCK;BwXD%li1Ox&dXjM z;4zrnge*l#7@j-pC$(#!eoz0j+&7qgkm_pPu{zJd%t}M&Dp6pJd8h&74CE;zW&9;` z7MmA^3Agk1Em5XU|G?Ty=T6pzDs|Dgs<0rkMFqWK#~7KJ<`-A)+nnCA;X~${`2^U; z5=zLL98>evK6e29Kwibh*|{%8(D8lpLGBAyNf0Nr4D)*|5{^lumw~^XBr4?>6)sit zqB$z1uTo{=N9Rx)DPbV7s|9O=!qcX!eEpLl!v`7Jvfr`&UOF7Ud4tH)sO3ozH%w6` zWzH%MQ`XVzmYj-oj2={x6_Y}UyYT!~mj#9P&qA}Dk^))wwpKE?Y7j*pYzoB7v-R}; z)SGs)(m>#62;J<#E(-lKTe{ZGgniANK}#f#Of8^aDJafiWK#7Y!&cEFX$oliFZ_{Q z{(-5~UG+A!7BefDzt)EjDkqn9hihG}hfZ%Y9)*#neYlL#pcNSlrNB*>J7FalL8J&M z4c*Vol--1!+Z2l(?Zer}e&au23FJ5FAnAmkxP7q*ljwU{vQH(cdN;E%%SJcywpIxb z@LWVh{cyARwg~s5PfC_=e=S+31TZO7%g#rCzrG}}h5+D>u!wdY!))uu24wv&y52IX zs_1XmUYqWe?(QxTM7pI*5ZDMvhja;>l7sS&C}F*T&bBzh)F|#1 z**}b`UPT|N6Yvtkd?7mX85^+^ez+Wr4qk^zB-M<6UtVWK7cwSBiW3%$Ht9LkuH?BE z$KRI#7~xGZUrY`^kw#HLnOMp!@Li;S&`7rIvx=~5)6va_M*zEC8OjWduVS+87(@33 zL}})o-9;a3bnaExKSA-RQPq%_@fBtFb+T-VM_-g-3>U#fHKqS$UK0{;thb~E*4#5# zY28BTpA{vN74vd)ABMgjv5yq0^e4gRW5|okZ@v9j=w`zP3>Nv8k7}$-Ys0vtabc7& zRitg~(TbHRj&TMpGY%37(eP1~+;}k_h}r`YoFcjD^Q#iX8g8ssgcoupe|qQ&bzD+f zB($92U)?o3{ou4{!7g(Yg>@0kL{n!%l8bf?@HG76BZ5~;siF1&Q~ zP~{fj8Y#bswW@!{cWa*gsw0Wc#nKB4Qcdp^D>H zv-?F38#G;ikAf3-L3pKv%46$a_>ly8c5ECm?O$Ji_za|D>jji`oJNS|ZFPpMBAje_ z_M6L0>i%mTl@g%U@-*QMyx`(pZGX@aD{}ANBI6x;{lj)~MZ731sj&Zn(cstKiyYU> zGhgqHrE3+STYI-5dPCg)PJ!SlL^f~VQ0l9EMCN<{Jy@H7T}0>IZC(S7z3;aJN)=o= zFl!smjJ|3?fIV^+MOYtFmV;vmKH>4W$ zfFwjqZs+Eb2mvuU!McGbqn@tG9vb4O62v?#dhp3*_S*F12003U1EP>WxGSE(yFg8W4n}u$1KiIC zew`Thv?A%A?IhEoaetcLBvXXJ0B_6jeqM%%bzHxX(RedzTvh^ zl8|Y&^B3x-44>EOjA;GkeQpcFbmW^Eg`V7;YSd%h@*-15KuURW2d$$%{MPo~n!B!r6}W2Z z6HB{pz)C0(5q?`mG`FV)&~zYBehIVKCj4+9Z8F0AmSUdW_jg?P$ll7bx^QIa24P}` zdbrAs718a>Qv=z+@brwt?5|AB%$tR`X^S-GNF#LCJ+)L`)(0|(=qKn$l(sM37{Fl& z`6o^E5K_1cFE-<#4qX8^N$1;J3K(X``5&)?K>4g$nv-SZeynil3&F!`XsXuX6y5YPL`+7fI||Rd^f8y3-VB+@5*AAc^!lD z4m~KygHfhs5P;RaEc18@gP2KqEw5{olxgUQ1{#{DeeGVtO=p2E$&-)q^=4qTT*h_^Do2b5~>O61icn0goapuh*-8U4sK96JZwEZtfRHPS)Y> z%uH0_WJ0^&K|sy)sMu|^BdUDilyO+ij~5T?Za=7kRRH-1*{4! z5<{fwzb^N_2Dp%p(Mt{oW-sj_)GvL`W3!yvBs`ul#{unTMT9^{Fukt{My;DYrY>CU zEi{vgQG3{#f1?7m{Ku{i3anQB$cI&9M&aYXP>Op=&riy8pg}xItJGshQfoByls$@2 zvyqG>k?Dg;Kj0J{WAC5oiqkwu3kCom8$pqMYq4dw}oc1~9&je+vxM8F6!nVxn!jqu~ zOcqKI$wyq(2_xK<7s`uf+dy*DOH3{8JbNcQBQ2e`SY)_{@$k^4os)dWnc5{jVI#Tv zMc!mzoKhZAr}nkX^cN?oMLn^@*eJnDKzw$xA0E;IIKo{Xq3#GwsLOFAY>6aUxeJ7UI>XfrR`9bJWTVra#0|j5?_Kkf)*FYZq68vY<=RA-M^@ zoYVTuY)Gg7!O;+bJ(pnC3SnE$NM5f#JUHOE>Qg_A`aaGw0A}lTtET^ni2U$CM(8jK z5oT=VLC2#OiE>byTXZEdjvnrUc_yw3vPKwJMZ_szUk zZ8_`!r4DzE<<<}BfB(|U?#2-RPh(7`Z$IqV^1Y}6qmG-y&kcT(O76OG3Wh7Agu5H7 zI{RY->pSD?kafng;yb<%ogww0_oLe+J%{RKsac%5+W5OI^gNx!a{{V6a{~Oen_-poe=^hB zO%H1H?pR~T^aNNOXlxEBcnTF_V=yOC-8CS+iyGRSMS>_P-6<3|dvN0UcwTlHGj01b z;pv#^DVgp3_ZR%p&SBAy3s{{g#ZpO*%);KE3a6^wu#lcqvn%tyTnv(WBXns&>que zHgnI7x83dVWAWwa)KqJ2+18)u#}?1Yad@f-MqjkFLuhgB(>{llM!rq5*2mQ|_YgW4 zLMb|Sz@cf6{O?%%*~lwLGx#~H78-8Dq zrzlO)$!MR@-=00W71@)@G`6wwfPZuDW}B(HZQ!v(0Pw^9p&Kjk)(%FG40Z+n&H$^*}!Pzs_ZkUA7kB9W`*D`)&xh|ps z<_eX=dv|ckpLu>hDJ!t5t*;*kV`61ZM@q4+s;1^+ADH^b3XMy30zU|(KQ}}G zgC;k=#3UAr;^O+D5I$NzgJ-B3#PaAcn!vOQplBCwNx{UT4b$P0xo&#F2YIOJxq>)` zAt+D~^KU6?YHaM)xa#%0y?@>0XJuuI_P$|G1oBdUth{QPDAGXmbF9lajU;MQO8N@r z$!Tf79RUF~83~CJv&MEtO8OuX3dT8T zke=HE-u;lk^<&AnknWZ8FX@YKpk>0M0nM`Qhy7e6D8YG7=>b|RbbpHgPv3n5_;sSM z2<&2+O@7#>2 z7EsD0@8e_pPPt!m;5D*)cenCJ3qqy5d%k+{7r|em{Va^%z+dg$*AxzO0>SwhRqzy^ z+0ueFAnM=H|He7T1^DZZeU~X5Jf91{**L35rz?kZIeI2@ym=#>_;=;A&{>ebgvO9?Ew!b}h50=N6Bn zfCqXXaOke(9lXz(<1nlx1nGA8Lrx2U2DD}Q=aq%ju-5TM<0DbDz7aA~Qr@W>i?*mJ zl8)87t}Q!BkAL;NZQs7}{0c&zp=V;cR$$hanF=#dR}34(+Xb9VheV<@vz@rgGxdt1 z!(At5X7sNXkCxswuJ(e|VD)Q5;~f2KqfH8+?RL@V-P0K+W|-t;u}iMCa|=_O7$ z2r`tnAeo(J(hQX{C{`(DZa2l309ge9`s35!s8SLOhoOe%2R<;UELv?VRi=%Yh^nX- zVplwQ|M6JoV-4P4LI*LJDbSndAyCw(gP~_($hs&)(w%aYz6Hu2a>p+`T@HE|=H@P5 z3#=m6p;ufWr;k19**u=_FW2)5a~C33cIWBq3~xcMi0M3Ydg7W1Z0*gjGY8Ovw^^k^ z3Ky=ft`@*kOt{Y8#Qz0?N`TTUwOq=^?rk3(L@2I#csTv=mQska2S7!WSd5&al9gQ` zEDegVXz^6}qt5%D?yTP7T$HpbKv#EUkNU`ENJ?KH+A>yZo>H#1AwJe~GQH6I%YeMJ zgprZi{WChae1b1aqJ?!}AkaovHuPVXyCgcBK$TT-cLeDqmm~P4O znmU1|7!$vPWQG~|0*5C-nTYqWk*Y^uY;0#KHen7ZtE;Lu`rMpja-KaIz$zD|><^b4_ zjb6My)a*dXdLXVQX3kU{8jRx3oQ4RG5mkczGw{xtJ@Bw(d1hPQWv1}qgnmly90^>1 za@`~Xv4|PGar2jv&o2I`MUTgPq%+>&N0O|A$2C$%E#-l|y)SKJo8oNU}#(^euB;@pxJXas1ea5!fnH4nR09-{bN1 zOMxfCIt#(_GUBnTbm|>(zWT{^T1D9j`3Z>D`+<-AKVEBomfYLu`O(pCs#(Gp1O#$c z+Jrvj@Xr9}s`-@7>2Lt*C=t|>lQKVc5NZ1}8rw%2Nl}qEu8CS&v87fPzxu3ZW~R)v zRe0Zj%nhQ}>7`0) z_NTWg=13W$-@d!eBja5<-I2h^VG>xxVg;LcwUeXdX;g{A|D6oKsGgF)u z1FDj&Ji%4tb+O;O8()vjWkYY#c_9|5N?iJVGs+QpM!+eXWB@+D!kKXgR8f?IyA85( zyjshU>}V2Br^PxGC5}CM4b<}CFef{rR^9evOr51fBFX-C8(>Mvz^3_FY<*)S;v_$P zz5~AYJh{J8C?~+hX~GRDs`?sj9z0}x|LwwXbs1?xVUof_yd-RKk3-FU zi7?A8`c_UY;Z2#6Ncw>?5dsbD=wTks5qbbeZ?u zJUmqB}>oW^@Rg4l5t$rywKe~Ui#`2(N)NFKJ-E#}Q zq9%PV3e{owou_?OeOvx) z{k%g`GGh;1g`B*_2O&d6(D(HNYf{RFhQ&W|Rs=C=D`cff50vT2H3NI9?xahA7C0P- zcaUJ!fQzs36T%w1Xlqwf?WuWANOGz*i=VCS_+itF2E1?IzO7t}u&GVY8~$q-CSfFy z%ZzrpdZV3NSYiv%AY7$uT_!;>zu@0ob@Gq;ke02Yz)|hN;`i?Wta{%bTw0VBTXO^2-3N%*H#a&CouPZFT=;oE zSBv7*D8-$jRZI_R3T!AeipJI$65&yBADtlqKBgx49hGYb@gM(#lkt~2OjK_hEq99x z#X5+ONgq;BCD~RuvV|RX*}rKa!I(-N=1%#ky;l6Sm-co&)@qymv6MXtl1}*8=TqpI zWMee2U@UAh*l%O;!ZQ?JnCf=Y$GFFSiAVy+Ewk0n=s*e``f&F*5R{sFYBo*uf>sB; zcXYtlXKo5gc$i79R51Jge5;=ka?^8N z(5lsPy(xC|uxFZ88}3yH=?g~_qc4A01E1RifOiT*wP$A%_RFyv{Yzp5)a&_323j5gZ+q{V-)JaV;Z%_9 z-ur`E29HN+rM(K2J)WtqxH;B}@|2;W8soib2VRSAWo|UM->WLU8mcEts-0VOaz%|p zY5D#it-~H3cwul*k8&MOK|8EztHlJx-k|%tYVzTJH27DWZKn*os@*`5nms-^d)bFY z9HS3tq^dUEQ_qF>_QOpJ4~S6C<|m=**+* z^*368Fj*9wq#*f(b1;=7fB4I|iiM!y2PAjTc8}#JlPf<#o8EP8mHCyVLhP^mWYWvp z@z(?;bOyQ!sLZ2S^>assv~Pr!WVdh+5Hu{`+C_}#m~ewq)WBnV$diJuQrGSl)Nain z&M`4oZi0b#kgydRsopL9iuFQNM^AMWp1^R{pr=nm(nibGx^c1+TFJhD?DgzSHBPwN z=+pG(W-T~tL*+d4&a7P3B^4(Sjz$oL3lV5cV;V{5tJmm|@0p`ligY)42pmqVP< zB_CvWd{m~I3fb@aSUBEY(6WW7!&w>TpXk-cyVbRx*b9@6j=uhaju1NLe7n2LAG59; zvFUxY0u zi-4l&qrUWeJO?^!(OH$W&=HT2wVdR@6b+=n31f3I8WN*a!$LQ^G7dX}r%Wdu0S^ah8qG46O0i)qvC zw9zd?hPmN+~ee=Ei zPaNK(xwY&4x2gIcKVHFbRdzc(C6um!8;t^XjjxKsG6X?h$gto$CN%fOB3!qUI10vl z?tGJ|jsmU!%f1|gG|D7dadV3*Ve1g{D1ay{b^qWg4Xbc5k9s5_U zKd{7$ChV?3qgtxRpw}us^Kr1v8rj|>cZ#W=G~(gy1;`C)ReG6eAS9inC+jHbtiFr8k)y?IHHKxt9QY?zfiwMA{L8Uz1q5! zAe9-5T|J!jH&O-XxNVD{(N9sL#x}nfVBt7e}?=VkS zI^O%;ov?Uq~;lzH}8o_BziWBHJhLaNyW-nfs%? zz%0Pt&~W$Z&_m>_#ru!E{{Hs|(9E6%G*oNtz;9>Q@sJfO=7@|b=pra6~#6VI=Y9@!^7NXX?8eYzwzi42LHG5Tv}S%&&bnrq=p}*3 z9qK0dd5iFF4-{btTCgPsc?KHYO+MaTot*u;GX*S0>oQICY95zX(py`qu!CgNulf2aC8*-YWl57+(jUBQMu{S?Ye7Ia3kBsS z<@x^q^2skE!HYWDh{b9#b~MKgYLT|3Lsad}3Y|~|1fs6Mf59COofpRN=?OtlpmKX)U47zx-2(R0=U5rXqBG77mgaA!lwyaKK81r#=b@iG=)Mx+`Q}P zVdI-62a2ky*|#f*xa{Bvb-vk&$G`R8OsTP^J4yp3mh^X2tPTx8Jz_J&M05+)o}`_q zI=6hV5Kn94mm=W`Iv`M?rqCya%fFC7Q6guVT=b0+tGo@!Oa*tl)R2jP(Sw?FI*%YW z9(mqMAxSMMu~JjJ@tfO=7@N=&6s!iCS?hAXet2usY3J(3SfKdO{aV36ZR9(UE5gTe ziI|SGk~Qwt|5N6NRO{?_-gbIKnibN?u8YIh-SWhdBM<`&DuL&nLg(|kt}4NIFOjiS7Wx8XDxhh^;0u`(1#i%LJ1By|0@Pz$&irO#e6AxIAzDH zg1CeDY+|{(I0kKvr7krq^us?$(+GH11i#@mIlsTO>%6B1OgBS+{ye4M3~JJB+N6LU zQ5Z2v23>wo+2zrmJrYY+|v938k?iO{luLzA5&+XPq2+M$tiU4U)DHP+CWa zRN{uT`sy4eHxi5J)>}eI+!JQ!ZNX;{dRumv!G+HkXt>e2B?ipHmwpP9REL03F1`}# z{hT9FDnGkE9Uy^>@oamnsJ$g%?w;=-(#NCrOGKWg=H@r1b+|203<_v{CzJ-=Go6k< zoc?lsTn`*%#43`DeVc<697l>=da`|yO0XFvUQmmKJ11ZHnAk5!y~<;`yFj!gQh5%Wj(R(31$r>aCO~_Ss9Bzc%VGT-sPWz-blVc8 z&xyTd^nB#)6zxY6faskWH|O#~8whw^_m$Gk--3BIHoAh^qFeAgFoOIka?MqoF+gFa zr9pKy$w?H2s-8R6Wj8Flw$*@sD2I1`)RSj?&otlZ>STuT_WmaS5-hr|pKH=amVuEU zQ9&s3HrMld7F;01diL)vc*ZBj{{t5_(PDyp>Fm{Cd62{bbzXY09B-gMFr7qv-W@!}9VBQ)Hkc*@})jF0AA-*D>?Zzx% z@G7}{)AwDS$pV-7o&YX@(NyGCiy%1Cv-z>^#ZPI8SWAHT%zVH-w+f*;|3`w|OpM+= zPnVR|y2}SaI^UXqU7@uUbkW;yYd=O#qSkC0K&Cf$xSdIt(RT34qE(Sc3#MgURfs8! z1pY(VuF+lH`%`8>ON(d{s*u{0z}>hbZxh_NhDCRu1K*sUomE_sV9c zOw?_a^{cx}+d9g8a*0}gc{kEF-bXQHpo166vV$KW^0-d(J&LaS53JS5g~7lF-U{xU zHH4aqDx+n_ecCW+1GO%7D!a#tk^AQRveCnMF8V807NWx|JB_^+=&h73L=^gIFOT>3 zw_2=U+b85#7$astFG3#0rzo5l`JX*45wBHqygCPW{!ES|nPs9&S;&KAIUW{DlJf^k zrjkP~tM-0EQ6ZwqFFr=u`HXfPj%(o1saQdf7UVb@Sd zh1&{>HoQZHlM$iqEfAHK&q=EL{FM`@5lNLU62%`9FMwgu*A$;m3`pfVv>q5X`UXz9 zjZV9c9!Jj+C`|u^hl=oofcJLLqLdtFKX`$QEzJemkf>^-m)!>Vy=rN0Epfx0QX!jp zEzVe=mdh<`U8UMt)i3vukmmc~?#dPZJ9Q|S+S;A)C5$VRKmF2~2Wg-OKjm+ys>-{^ zUTvTinX}*vVj_y1LHl_fKdOzmI5bMXUT8Q(&9;;Gq_%}H_ZvgK={M@b{xcO&J_Fz! zhc5*LiK_90B|FKgz8E5KAGsFkU?`RMHY*DXR|;+JIH_zHzB?j=(z;SB=4hza7U&G0 z@wsV*u|gwbonQ27Ad=@j_ixEvo4XAp0k2@?WEILJ(76cO+oV9}VlVh#1V$GZsY=Pf zM>(=Jr8xnaDBEwidWiQNLVgEEA|+fjB>4fcx+uh?Fb&R3u0?`KsO7bGcM|5?h@5$d zLO>+VAH|B8(vR$FFc0zJF~FHT`nkx%W3h^PnzOHG7qwVCj9?Gy9Vk zEJSQc7YUB{&q!G10On&fbp(Ta=_@fEzr74@zmCT!Oqw@q&tkm9C}2z^oFWgA!SD`L7UvJ>uKoBr~>8UWSK z3m4gk7Ld?P7j`Fi<@ZEQa7EXC7V@*C1WYq*+h~A_ZeG%s4XAC!QZXlLS7QtlbB!%i z;)z8_zZme#FXR3*u37)cK)qL-wIVV;9TBA-;PvUTri8#+3s11%f|}Q_xJU!EyRpWa z;+`ik{6O@==2TKY>C?b$6dl7`M&hSKH*ut=l0zGi#~sorxjs3Pvwleo&+Bl}cLbLx zJI9)+^u8sIR){Jh>ll)V>Ic^_GtPC^GRswbj2d%^+xL@}(2ZxQGXGrSgX3DUhA5_$ z{<$R<0Jcs}0cS&0PVBTBPcFsMl-_$AN1N_gPmgV8hTXAPV=>rqpJn;IqPuC7*{IJO z6+ev}@h$nvSH6F>Lp<_$=@^gDx`gnV2xawt%Hdw}4{MhWTM*nY{dNyY`bh{k2ZBI# z4k`!t9rCMc69(}v$%W$EWO7-hWZCa#&;Zfmaf(96@*l*1u0NC6rL1gROJ3}YyNJOC zfICV~hZR$_&tFOWs&0DE+C%$(5|Q)r*g{0C>SR$gS8K6r5RVbd^UwQmuY3t1#2j?K zqP+z(t|mO+*XrY+l#JFx;7U;RvpeX&jf_XACcjTOB^2m2+2U|jloiYS$nk3Bux(s`tP}$qS_ZL zbVmM5m-qzML_q3kd@GuW)0VjrqC3qsfwkdt8A7In`h2>KLYG*aN6Kjp_0!;9b7Qo8 zd5jv&PX{`%VTh4p=iA}cBfhk_9e?VtB;SLW?v|u6xs&dc`P0+V5z9J}klQ_DOw4LK zW-Fxkpl6y<>1=m_LeR5WR*TP_nu(iPUmu2xiJ!6IpB$u_`jP<}yv+deH_RtLtAE3t zRpSH2Oc7{(1mMl!0M|i&!DKNkU6b#z2X(R!)$tXD!D-?!n~&%#jF!r=kF!zv$WR9=rw2C83=+rFMssj z+O8*1;@gshXZ-b%4!3dbku6r`)G9!(`5OPr)*Wa2!dc{np758A(I(Ai+lmkc5sS~( z*_p2AE}gPX17EdC!_l|qs!3D8P#LgGbF(4SG)h5_ubB+xY?*7kA8XM6u^O=WxBwO` zi9{S0xLn5X$|@6&;=Q`bh`p3L8YC>t}R-k>ZAm}a62fc!nUwPGHfN? zz^M=D!u$AyBKk;B(|O5?hT#i|h^+OM*R zlHZ`HcdMh21)W#PEMAphtbYzKT>@d5J>R?H58Rn>MH~zX*g=PgU2UFAB;a>{?+++I zpX))mdLc>8XV~&psq-Y`D=(HT{iU;hY|?A{oYAI&e??tUXmfEWb;xi#d?Y0@7^!rs zAC?}w>pa^#(zaWk#Y=1~EI*x}1^k0!e)l4^3ZU1zpK?O0PAnXCMjS0`s3G{Uz;NrE zVrSI~A}i%LA?Sh2(hAQCuP1-MCdO*Khtdo|LqnbC5WN&eMd)avoS&5!UUe-RuYHNV zGXH1`g(@PiO8zsCR}jqawnAdx1d-w5k*=iG>(BtCq<$O}I|$4VwL1NbQdR+Dw8i9S zAkK25=J8hLa})sGP%jAsF&QF~2+KP)lclg~QVbz=E= zzigN%Q+w(i4fTkE$p_)rr>Nwb{E}s08YQfAG`wTzQYyhx@@w*XG8HJrAz5tSe;{RG zl;t{@{f>xR^U<1?PGU7j#>u#u0W9uSw<8H zfhz;ahsk`Ik$C(C3>n$dMmNNy#oZ3^qgW-ER@U8k*mbWGG>9P2J7{gY>OM>!!S2H2 z?&*{lw@&PQWHz1r%|I?lmynq;78M3c!j}Fc)x^fjI;S2~RU_i16mnEx(qLlvYh=@{ zGb}`KrjF}Af!$Rh;=mxC;fJ%`SI0zxm1ca&P&3s$)R>Z%$~)jSb$#px$= z{m#zLn}#}Tx;F125$uz|%&aWUp9KU*)2#~>BqWz>U1B(&Q$7j^B86G#c#K)E|40bI z_niLwJbn6@)>3!bXDd9Gtdu9Q4(A#2-P%WSox!7>JykG(O^AlnPDnvP=zB4R{Kdn! zSLl390h?ua?=o6VNl9XMdRiYx)p6xjMhZpwzP}HJ2oT>VuKP_saqNJ%cPR@sf)g*HXMzxW3TEPb3=SYCcLnitRe z5E_fqvkaSMPfw;$d|+cD57*F-Qyp^65}r<~t>DztJnR_2UXP`lKN&_MeCFbxDrtC3 zkeHW|n%Y!QQsV9ij=CbMK68_DEuTFZIG0)S?j}4y=jYP>{irKD z{b=N0Y<@gTa_yvreP0_IFseWD6P-oT#{ixPPb?IKYJ~LPU@%yflmT*{t1>m8Hr2o#$X`$fo}5MiYM4J?1ibuEelfZY z?njp;CVi~=7G*3~3nwiDl^#{S(?-EFZ_?u<7#(RW%C}41@`zyX3{{OpG2jLpFts!U z&*oR_ocjCw-JbkuMKMibC*#d#pS%l< z^(%}BiB9qPKF4iW0+`~A?kB(#u6uQiI;`#Y7o zEn^|a7%4uTCu;&!vBArSdtz;OsYZ^Z&Cm{^-rmt=`}>FY_Dgrfb=><@usZv+v z1Q4h@K5)Ojm!7wZJs&O+E!j=3bd3h9FE}K55O^~x_vmt5H8OTom2bL2*4LxO4qHV$ zN|i_^^+lYzC*ocTJ!P18P$b zDMzY$rwGIXah+Is=27#;glH_BccqXaY_{)xAUXq=V?E3{=3nRw2rW~|kc%gh6nGWt zefG!l{dZUN)C-t}z+%`8(3PNtQ&O?J`>*u0W3O=388EWjqy~V8&tEo0rbv(XZXwZ# zd+aVBw!Kd=9YDLH-iyHAr{*RdU0ntcLjP#Dw<%WQiy1hB#zq(Ss>>|Iu5$xOMJU#C z#K8;nBuka17#ke3_gH|dzLvDy-4p|-#YbcU?+5k$ZzgW#v)mC9loqi|S|!sh$-%AA zpN8uDcvg(J=xjN-#8il|A@IeLpxrnL&Y2Br^f*myv1#)`!$c7XKs?dN;I!1~sJOm+ zJR&=6k|g@DK*%A$w%K)g^CRY^!_tqs19b<_OmrH_4~eZv`MCKUObU69X{y0eIw82P z+7POxXO+kXdysgwW!A~5P*fJ%7xUW;Jyk4Q|E zGY7wdfsl2vl4Kuba@}uLQK?+>Uv9)GI1EBxv#$3%8gGd>1loebQe5Ws?!YWq$hs96 zHL+4wCXV6aF*SR01;9bq*vbC4J?H(|FOSL~MJ#n_UnAd}P%W3s8R(B5OCrfB$@9g< zv~5%yuB=l(qoqM#BJeM81LGQKVWs>|$Q*s?cTr_}y^F~s{`7g0AoVSN`Dcyqo7I-F zxLK#ym(||6J#DUG2d3-k(;+S-QO2*3StMfUb(8bba(3=$hjUh3m^6T*^YgmG z7Rr)WcdN5s*1BJ&sVnLzWtWu**v`J5Q)7a6LgICSMNl73&^fGo!2EubLZr$Oaa*g- zFg<3awF|z=Yi)pAbvFp==5DC{udK`mZ_SXqGN<~MQ&MWWpY$glpy$LF=1){#EMd1$ z;S6R<7IjGS~QOZiIUG zU{B?iht?6N6Q6VAr$&K!=^<*bo}*fZw57G`t?KqwbR(X_$#_;e{y{%A;*pVz;C>Knns$2}36SuJ z9q`Hnb#8TbHz9q?4hBDPKDVE1tu+j*4ft10j8m9&)hUjdV*b`v7l9PBW2_3qgwn^D&OsX4JuYh-BLfrHp7QmR`vd`45%z)-^@!@}t~vbKW3UDS zd?zg>^<2bHMnhgJwWV2f!luJD!;0MFJ}IwuPn+24cHEoLBE!T1?C5cPSsL$`ZDJ6b zo10i3)1NLG1;+_b5;eDLu*C6rB5z3qtcNAEgni~zUL|CklJ;$hYByJ@qeN-()t(q{ zSRcwTXZQOrFm~KIH?65G(*Fe12a|%(>_%oth(h(!BS=;$mZH!I;=%yQt@I7!c#XYh z0>@WctTmqQ6$EgEI%75xPD93)sJP>|3@l)C<&(Jh8A+D;w_88#-rn@$`h6mV-qpIb z6{!NDn;uKr)8DL~lv)cXoD#+uZ6=;vn86j}Kkexidbu1`9SlGu^Tq2|+y@k~sji%(Wm z_n(`x>&>;bCQM%Sejt$h>qnc=aw@psLdFtOJf?zO-BoR4$iZ1hCL zVwC1^SeM6-Q>}V>OI0II{MtNO&VCYB4rhsoblc_WE_F|HL&pz^i_hrmt4`HuoCj0 zSP1yKF!kbSQt`Ml(x9Zhi+wAisndA6t`pc7C9{=UTvV{ia(K?a%uS*aTv>I-9x&h+ z<+(IIZcB#8L4NQlM86R?lUY6E>vmI?Vo>|`ND1AzU1@nkP-PA%jxpX81e z6QvhjcyUUv%Zw1&1AR0jvN1JZtRfFgrq)_N$rM zxjCT$dlsbGkjWc_!R<&67lksg?zs3&TlPTV@9WPlCZ9Wv<1yeS0w`vkydbq(`T664 zmXY9p5Y&MIu;OvTEiMl%4F+>Ar9bd6JN)=v>Xmi73s#MFMb+Kd|CE@=j8Y*R7Ao`N zC+0o9=-DNk6m0xILes0KzGAQo%&Kl@JHhe*H522qKm!zeitr2$3%@MwJJP=s2l*+G zCIelMV=HGn)Ok`UmL&N1cVVl%4R^lg~W#~yW9gEF_$R1=w^ z^0imFsj76M?fm~@eu&uh?vD9(uORq;w~7vUg3o>=gbv1BIWBF3Np0&w1z7GOAtZ&w=bU6QCdXbM^LB5_t|LGaaEjz zP9-;4sMk-L|0$LArWO|f39A3;G-YBR&KAP#iKYHteE-`})H+?ULpD`9#;jR*1Odgs z(v5+WnaQ?xm9DrR!T!f&6!RqfDpYLb)#D$)K(|5K-TeZ*w2_(@YxhH>vE85VvvQjh zH-&phtL0|nJe8M*XdpnQqmK}I&Dd~qAfV3M17G`>*eSYX04h`@J=I);M#dh>fmw?_5n=LvJO;cpU>zr@J~fT> z4?T1E!2<0J`E|m`@3J~|_Xcw)=K%cBBUtE|O0lSUXz4Ku$}&^2n*fZ<6*g=`Vmz6! z;#|Jz1L{ygMjr0?gCjGpf4#pK{^DZb{DO--KDPBJEX2lK(gBL| zw7Z zuA1B?j-uD(idPVQSAI<#_-XrU`eV23L-di{^Wkpf-q>Dq3uOp+)^qfHexit-$J=^{ zTs2FAipOcG`pO+xQBG5tS+_mC{ZPW}Gd*-KtGDfz=PFAE0CW#ZvQj!WlAUcAi5f;5 zZx5srjd6L?W2JdrId>cSszVBOWYyWR?-+dEORj-dOxE%L!JGEHAItOk4dGB65&d`! z74Y$iMPdD@M(qDur*nRFnW}X1V+@&-p8^%)FI7~OB~;gU`stKByPf1rtC+wtLispR z1NeG7+Yvm3r)CX$QCG$AwozzC+BKRkL?hIHuFF1$Mh*JWz8&Cz)S)=;*h z5v-X@4v)gFm@6noccB$Br5jr_9xV-76Sn=-eJGkLiT2&tn1L3mK2LVE$cmqVV>No_V;RRAoOL% z<%3L-qMDMZ^~O)NG~99|$&%7pb)yI_%sg+(nGyoJ^@j3Q`U=sON;tMh#mL;;w)3pm z?yd(SG)Yml5$vh%4+Wp`wCZmKo)U%New%At--n9FjjAs50|E9$BxCfROVT7F!=G&a zx_=Up{Q+RGYJn!;F2W@jAMB3JNlf&MAI~XV(XRg!5$;;NY9#B=e(jsG*E^_4YuzO6 zH}M$wLchElG&xp-{8aOi&n+T2A7B@Yoxm~q^TGM&WO4&1=N=IoAH*V6u_ zIk~4IW&z?cC2j3guVDiIfqCY*Fvo~H=z11h4OwQg|1nJ{F%CzkTc?i33qtzL9?*9v zRSDQLTJ$l5m|>^jX=M_Uh#Vfw8#2Sb{uf_r>*e6kJBLy3U@nlzQncz!jMF|I4~w}+zA5hUt@pe2y9F*RbkI%uAQWtO`(ep_oN`E* zLz|&she;Hn+|F?KFWDaaFFRNy%lG$)_sNHX6MvB3CzEvszDaiNeI{T#X_x0W*aEzn zn|pHm(rJUXR3I{9qoxW|;K#Xj4~0{mJNI*C=ab!h@8HyHbZSl%Kqr`sR;xgfEk49S zQ}T6K`ZO*)r`maDhbH(~E=X7V!5v|}?Cn4WiZdygJx~-&7w|N(_FQ};;8E?+1A=91!$|$|4GWREcr6#-zk`_- zow-q;Nh-9JgvaYVe-k1nB_pwt-{p*x6Rm{yg?;4V82Ox$cJfG`Gn721aUo|5sQDOg z5ffI>naeSy&T@paYAT8?#%2|n$lljKX%;`>dcK^RPEGvPvuX^yDzH6oII4 z2FcwRdDGH4ni_U*2IEB8wgz&Gi&J+iv`1WDp8VoL9r{1&g$V<2esLpD999;{Hwo^q zIi}F;7$AoXu6Vu#eCet=AkQH*fB$y3HhowlEQ&b}picmUpy0NsxxH*_y9T`aJ2xW2 z$H^k{R%r_ohNUcH7FTCIhhjvS$#gq8$VQpTCaMPdk1rRSU-EqaVdTH5T?+XiwTkd= znxRE1hgf`x{{y!2a})P74zw;Z6E#wTzy7?sGl=mQz+G|^QFGQ;X7|Fy7`MEf9BB%$ zELM*V3{!%?>4Kj>zk}Jodq|zJ$(6H2GLDctJQFm^#>9-fF0t|`vd1JtE5Cnt;X%LiF=~$M=pi_{PZdfEl zmiO$v|NDRMy)*B;Gt9sYoY`l0&f$5^`F($%4~l`@itdW7xR%PzU2LzoR+OIbhrWej z3^<>elo{IdgcIkHZBG_p&L03{bc2$Mpd7>&x%zDUL&h9F&4GQM z{ao(7Z`$N#nn2^m%;Uq_Q)S`XrsKGrGca%)t<${DZ_Of?(!s>r$sA zUtkT|;B&|(BUUofnvJ~>IY2A!3jU*J2v{`J8`Uj<^?^Bz!=&urW7baXosrQ?GILFh z{a*53|99QZcw=-}-4MG<(v$Rm<_~#12D3kaEfVD0Ru}Fr>b%Bpq6}TH6o;m{<5lc* z^X&r}lr@_vNnZj3s%3h5S~yTD@Pm}7H?`XaK&(98N3>0Ua3uhpbhE&c^rR<_n|l$m zF^VhP)0OxbN{4lP3;JjSD?0jRpPl~*p43gA_-EjZMF$Q=tbt9eDB|jJzvo+)xB*=u zFtx`4F*bO9s^*ZD4RiNDbv)sl5>m? zk;&TmS2Kl-WX}OkOyCf15OnQWQ~VFwVSIA0*EUH;COMh>N=Hj;JMjoR4R`+`w4)Il zM#g4U>T@|OJ>>9tMQl-vj!x?MB4q(`d;=;tj-N~mldg-@iCtgwu6pb~vyVPZG0`p!#< z8%WKRy}XWopPZZ!9N1{^6dugMNsEU;vHOd;^<}Z;$Fc{9Jw#=(8wrF_nb{#-B5&Na zH8A=I5}`Zoy3|j;Aa}q#DhKgO*W=D=F6K7@az2dYN4Br`@S||E^re;b4ArjV<%dw3 zn09hoGwQvhxQbp`2Q=)jEiFldFB21=wPE%k?68FZ+rDQY#y^tEb00vgS*qPU!ksGz zKcy*kUh$Crkt6#}MyQh7sUX=m#&ow3Kl-K~a?@7l$ZilC!ClJ^nQEx3(@n=Z2d&UR zWu?D+QF;XDj|OIaNovq9^-ZiIM3Fevz`(e^q24k(E8VYf zajRSnN8TSA&*ITKA~(FSkhkG8SzllK2WnQS_REcS_Go3=t==E;=k4vJ%cawBr)j|F z0?Ff^>M3;dmELwiRsJbcyF&FR8MCD>s{~KBI2#$QK9jJ(6z3xSiA>Q+lhErd8Mkva z&P=5{WvSPN`zgivey#nqdz-)>Z$~DG0F#QA`Ak1Ti=UjqXhK{m8%Rn*JnS*(kfg$ zn^~##lw(MeUT&2r3{h03V^k1VqL40gOyGj`lP=fJ?p{>2ul+7g>G?gf4V$lQDr)q11@q=8zpmx2A|Sgbq+4 z4->EXk1C!`MnyG}C{(WFF!w%%j)e&{c)nZ?TP1wWXh^F1LY9SEm^eK1H5lG4nJWKc-qCx7X z6Uiwkc^V=#m|PaiuGH@~tBOA{HodhQB<&c;*MH^kY(BdI0Op>RU-&nG zh6&SlAZKvXzWqpuB;e z`R$dK8Rx^$ha5HaeL)(}iJx4vichSGcW_){cu21YMM^D`T)994QWm1xT8=sh6$|(3 z!l+;(uqVUm?>80|o$tny-rV#H0bLm`QH$oaLA0BK25~_HV+tkMKt3$%JKH&{%H${S z?yMTbm;~EhAPJ6S0aqL;Q}KjHM$lp;5|AAi<<_s(=}X5;;zFUpscfK79IOZFxiwo@Q5u z=p!euM-o$Lo>A+S&pe?i^TI)2Jd~wP$E&G@7`9C|m6+p%3b->M5h+PSo`_R*y>bo; z|95Dik)vC2D47*P)F#LO>ky9G{YXPdl=EfjAyt!E9tl;p^D{oA{*j8X6KCuLl9aCx z;<3id9yDQ{V%=Z;X;>ZSUfiT31=pi9vrF_5WQYIaTL&HfC^}xOE}PhNwue;IMxJHh zWG2^;(E34_qnVQy$LXUnL4#LyY>WlA{p9MBBR+TderUW~&!X>Q#}gdH&2*NpA}O&P zuJ#rBn&z$XavAkG^HmfExCQcT#0Vn@Oseqa+ZcVak3!!RwY9YcgsR+%$1S|e+zl8z zGbN}M|1fA4#Cldnhq-2SBtJT(NyjQTUaAU}9gIUpR!tETB+khV_3ufRWtURIE$G}hSk!xYF6tY(TY@WHpe!?mifQ6jL*nydxYnIX@Ijo`_g>aPDg zv++#uQThz7cvrQFt&he}94X4L0<0Ux1Pt<9ABC&~edp%pY_c9$>R1X?_@6V2Jiw_5 z9uU4!^Lag8Hdy#rm8rSTCh@Ww z%$^|eJShTB#3a?c8z*@i9t*wjWSJA8;lr9XK)qy-DuC`fzN6Voc_vsTEHJn&suGLc z@9=4}(;^56m7Cr&bUayG&%7I4RftE#N{;-auFpjmsii39$g{N7G112_502_*ePSLuQ zA3^xv0`CrOVT--b&bSPFO&?2^vH=;mUe`JItsC8{5AZ`2LIpeCzihadKa(HW#pK&7khQTgh#?8XV3FPe=v%xC+DgsO-pT_P&^C7W@J} zalEuP8Wu}i)<=J%fOFE*2QaYkO7sjJoLaTr$GnnmW%Xw@zJi2 zzbHJmEpJC_vmhblC@26C+KRZ*KW#oqIu?K75JY>A*GD;D< zg=4^-m7;L$d37&tvIwIQKh@hu0{q#0=(qM%adDFSg7=?qu3jR`&mt6iKai<@mbAyd#=!qo&}?Q2O+U&}7EmeqFSarREjmmUFfJ=e;BeyW2TH3iXar zvJ^C?+3=J)td5by@8qA0g3@CVy<}jnmc7m;Gq{;kfCoK5& zuJ>yNt-z|Ber|TQ@Og03!+RoT`={>J9=4{#9&TNoZ)mrEN8;kOMDaGY*s)EsJeor7 z6DHztMetA6@Vs zQuagdnXy%P-C=(fXHZXoS$rZRRB$Qnjf<*cd@^?f)^S5J?BbR$)r+Yg*J7(tp<#23 zlvt&z(#ArA74#Kbm2KBJsJUGD*6DB>iIH%pfQ%#aR%|tx1&)Zm^x+xB` za&U!MHU!BO`h>APTa-r`dCQ%3GSIeIQGN7}$h3E8>uQCBVJ+eTX>UJ1dd#Y*(VYl3 zpocS{CMLzrn!)rSc9JXrpg;qp7G9Zg4&ivp$mgvNEqrZHs>gYs;-o2X;(5gp3nryj zfc&kiS4Vbk+B@Nx_zyfgbEL`54|G|@P0EUqi}E=bPTjS*uKLT9*4ik}olTJu=?BTs z+sak#2Qa^{x%yATNqCf{Ox6dOkUV*G(n#wp??MR{{EocgSQl%Ev7-8B90buRH;8j! z50QisS?l61eeO$Y?QVIQZYC`CcKv9P`WW%?@s}e;!MF zWHPe*g9Y$uE3S_T@o)^f8?)D;r4{V%ER3OG83N+yFw};}!IQa`wtC_E=T0%Y(^0W} z<#@$W5aVhFA@j`SQA2Cb0(nBtn*{PU{UxWXK)tizbF_jK$Ee{TNYG_pQo^%~27#0e z7^)@W!x+TQU@KRRL#gNs$wAu8rP0Gsgtsxo(>s?WlQts{2k8p}EKi)=cj4bbq>mN? zA{+#&iO&NoE6G@~Z%IIs?6?~R@#B)LyQ`i~J1IX2doF9UIG(I(0 zvyznKadZH}`-g8V|LQXwxdkD55m$#VXzxA6}#oT=M z!)%226zhPYwgokoD?+vgQKz zWjwATaL^Ic(@$MnTE_U0wQLV$3)|WWubCX(zq~t~-~2e4!7i}3IDu*A?u?`K&NVM! zK$(Fcrxv$a>FGCE^X?!{z{ZDes&M0-#onfG)doR-G?Jh2K^Ua5IKDE>x-=5CiE`+x;XpUM75FX4k1T#{_)Alu2jf$ z6BhkNWXa-L+MDS-V@5AqP_^oRIiS<*7*O+=;FkdHK~YhQCFY}Tx+@i^*b5L-kAF!k z(!csq^0v+wmGkF(kgp}r6_*u@o+SS|a;p1Ij{kgRySs;6$n0`YL$~S}eB>`k9IN#B zNi}G5Z32z!%$iP@)JN!WJ@`Y0n=t82$*m(1*YY>_{rNDn48^L6)!j zwP8FTh654&va)-1zqa{FV=$7}8ce^fyF1Ge>mBRx8{Y&N^udK33`**FvB(eM9AGb^ z?l7N(_VmPPs47ZQE&y`gyNVD5xE2q}WQ)Y`uHhiD;}}|bK2;@(I>l&Byn7H4$fHtU zN~#nqdRErcP7o%!;c6E&J~6=w098AFpx&_cDo2KlfgzQg5EsluvVKr+ZT4lVvH$AI z5+OfxDZjHY=-0BKspTTL07=g1)3?A~D4^}JMUWKtsTf@Nmhg)EcBC4V*oAmp{wAL5 zHqYF*h7#sgvnO~!8n>%hZLiR}GAu04iWg`_>X*61()N{^LlTspdH48DR~Q8eUd%KC z%!{s*QV4~t_kNpychSrmcOd1^grCq!iOu;NdL)wbo5FKC?E&*>4trDs~r)J#OO_Nu`5X5 zgFFBRtYvZ-3T7P*1TjdlWUF%}fvwJ?LBEQ9r?)*9E4P9UbmxvSh$6)NIwD=ks+WLc zlhZb3X+QO*u6}J424=Vc4$in+^QKL;z)_W-k598v#TkMd1E+dw@Cobt_p#2ie>q%j z&unz!azL9at7YpMZ5Nr^8YpN|LiCK+4bkc6w4IW-KA&%k4orR5xk|@YC!Y#f)7#=n zYwjazj5<4eYTWe5TN|?!J43Mk_JaNt<50Mk@~EjvR0r-HYKFK(^Q^3{KK^~|+}t;m zB~G=TExH*iI$*-(;_lu)y_56xzX4C-KNF`4i;BFzd`E0%JO6H=m2Z8-{!IY$=K+%f zs9{%3bsK>X`9`$)AH{78KT{n{TdtsEmZL%t%})UK%DruN-5kiz0)ayDHuC59f*rIO z0(In2SXb8-0v&=bM6b|+cGWCm%YP>-HFUi?2>k9d$g9#Fg7)VG%kI`K4K-1b(Vr{( z_5{V#M7w?`?SCPTd~2z^0KhX~yHYNkd-PJ5bC=!jvjOBX=D;9!5!Da|JUk9!x0~;2`KdIMB1IH6T_I_?;bCVKL9J?E{asv+KGfHbh)V4{l1zZAkw{0; znz?K2)*%_hwr|U}Jc+!B0;^a_Mn>)RoaevjJOm_d&-M5rH!g`!L*Y`To3}RZ%F%Na zr(p-+AN!qRj22)% z#3I$6RwM9w&%TPQ+0TAm%kWeWy60gtS@CG>83Q1Af~2Ttx^AyL+u4y7TK`I(ZCGYz zNqu6q3m&e$xqfPu`{g4K+t_#IU?KY!Qc zcw;5eHV=v*k2|9e_YIm+6}btZqW6-;^}^iH zux?3sKq_J$Zm2$-rne6<%8Y>RoobhmaU-ctg<5GMt!9**H~rs8=c3r+FnI_ z{I96bfyv-$J&7e`V=HL2L)cbz-xOF1*$>HTBFE9_h@|Fb%fJI|lyCi5`+eapYp3OI zyx^*)wXzq9P}nhF(L}qyErw|m9es8;>y6bPaK(Lm+&q@2i=R_(8b6scqbVX#1BfdK zFVMK1E?muCu&|ix2G73jpagnra0@L0=n(XWAkz^e03)1D#5mG0x6#6(s70AEF%Fov zx^}*PF*9o=&9AsWqW5^&cmHi^>6-G3C~82a8pu-n;A%X#?nS|KbD;+0 zE9b33_k&5))S{wrqoMh|FY7MfVU%W$Ss6n(iG=xtT*HUGpbYUk4}FXnc{O~93k4h( zaZr1h^7C(ZXeu55u1Z8Sb#HWR?9(@5N-pHL0YHttmcSqyG_=&EMl9#isg?{|7y$X@ zKPxxCs;VhQJ}k1vCmt$Hi~5wSikqxLd4|NK%LWD!wXPl1fLUM>@&}ni<&(CZ{yHGp&*DW# ztx)iiZC5vxLgJ060w=apw5n6|Abn!=k{bANBVi;S!dO7TSRkr;#roXSUYT(XaO@xu zZXXpBA3GZ#dudxQd+-e+0u>Px6ov{4i#`+kqc0la^|f{(Miy|<5zyXXHsX4+H@2aKV@jPbN{kT$aS@^I{*Lx literal 0 HcmV?d00001 diff --git a/doc/src/week42/figures/simplenn3.pdf b/doc/src/week42/figures/simplenn3.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c27014f4a3ed2827ab05bc19816901e077b5fa6d GIT binary patch literal 30861 zcma%>Ly#^2lV!`cZQHhOd}Z6V>y`1!w*AVsZQHgz|4d9rOvLn}b9Hl*%d@yA$(6(; z=$RSVVaVr}hSy*?h?s~RjICk#_+S|2%IN`c)>*kobV`x-%H9>2Fwm7zRGDY`_rD1vP zg29A;{HSBeWRfzT6*hpHfqgv0l*z>J`JE&qTSX!-Lpn}z&s3q2l9(4Pjbhc1ho-mp zLx^&063`K4F0iWKND{fGr3kx&$)qMz;10Z2)KRRy?rX~9XX_f#f>r*8d)1RGt7#=e z6#s@)d(*oCRY?~P>2bQNfZGH+3ge*00R4ptE`mTH_3lJjIq(c&Hy58Ns$^;BbiB?94Y9SJ^&V)7p$?pLw6TZkS5RX+un4;`6*u zQ7i=f`nqruDzUPy?cOcR&`wyMPcFc<9i3dbzMW&rAjosM9z1oXt4EETH+$Y6Eia~( zSrn(l(Rkx0rly?X`GJUMrTD9AP=5p#%gs+(@i0Kkk!67E6$9P!FscSyEiiKJ4iuf9kE0#@SEW_~sThluK%7?#Nt0LNPE zv{CXCF(yF06waf(Dxw#S$ZPT|sDOX7p|6_-dQ!I{;wrcLii1zxK;uHGb7iUrIv1%^ zHJ3s6Eq0ofg%DS7eJ`=LGLf4m*16!NtI*1;9Umatdg>it=py~-#JqjL(4luLU>)b) zM7Vi$P^>*y1=V0Nko9dJMG9A9vbOH}vbv~@N+ z{1d+Jn3q^p1IFmNCfOj{9M`1SA{Rn`HqK1|4rZ7X&kl8O0YiA9nE8^TB!D~|6SR}Gf^bC%3%HP3M{r6Zch`v=~^sdrbR6v^tJft7<%R#+1H}1CiFk$|P>r>c~Q7IirI9 z{lEHO7pE2_JikCWtLUmo@R%^Y`^s5qqDA zxw%zZYu2T;F4x%ef!(j`eQoEfdU(#r-@RWL?$-arR4zL<<$yg%x`Zz+WG#Iokx>h~ zq`;dzSJe@jl2f+mQa(V=_ROqS8kbx`S1_vbNNN;A|L5|t%o|6d(Bsi+@5IU$Y=PDB zejp(Rj_=geS{i#c&0ai0Qu~5hoKH+y9ivu^!-nMZZ-+l>sGaWUH_td55CTPTUY2lfLp;y9FXC zY3l8g;syM~M3CUnayAqc7%9%%m`|_V&-dTDOTRk3)ffI#Uj@I})34fHF2Hl_8@g=* zhogEiZ7LxNP%0SB`I)6qpAZv33J4AZ%+V2x^xw=~?=ujBDi=0M9qzE*8xUmzD z85IWPl5htIJ0TA!v4~JaIhmLo3LFGXa6d`_Xo975KFbom5$qB&kTr?kJ{C|o3d->x zi8WqBC$Ap?kaoi@p@`5>tX)Qr{9stT*g#>uAReV^&@J#ZWC%A19AYCVu9Ztd;;Y`xkWOPq21Gr(eM=$+-Z?2z-16|?&L0SyUx+*4UtI(w5}C=`$GKccEHt?sfbW+&EL?WJSKm+0?K*O_TifQf&Ehu zz$oNtJ*ebi1l>$Y5{P27Jd^@Y?j!ZyECogO7g`GOGpxITmf_ww=ra_Ai48pS(!e7w{O-s0kr$FxZk^F$7WHG0Eka5#+$;Q z+VyWl=(ZnQD9*dQe%A!9i-92BU-J%b02th$pYYGD@9*)$Z_)cK)i3ShZzsue_{>c2 zm2NNaZ#@1*u;a@|NT2d6F-9u{kJJ`R+Yjdw+>eik+g?jVIK!{qa-WFPW&WOZd=Z2rV|u8`yDhRGLJH5XxTOZa=qo4IT`- zjijCnlc?50FdrWU{*BubnwbddLlyi@60j!9hLiS+N$AM^n;Kxpg?9rT4lJNZWTes` zr`|s;i4Frt@HHvAAf`ht=>h&8{y-=FZ=0q_1J+3*13O8(PEF85SdAWk(-nO?uEAXJ z=7UbXamf>!@UF4EcB64^NKThP7d!ry+q8>Scqj06i%vR0s{v^{wrmzndR~FX!e-ZQ zNciScQ~k8e6H(!UM4fs1ule@MOH?789gYO_qJsfkeJqFH$RvqRQ-E23lNUx4mKi~A zb(XuNY{xSe(y6in^X}#@r~LZx5;AuA`H!bp7C)M71VwK2j9w7c?Ohn-Qm(PVKi5T8=25+H4Q9S zVZS=tFXE9VR`h}&^Oh&rq$G1dMC$61=jqtjx@e7QEaV`6zpgKj6ALD}?CzH}w|3$7 zwa9(ePo-3$uF8JN_@4f2BdquQ;7vBSUmrS)J;j?4-*CY?pv6#|q>Xxyq2%wJn}?W8 z#yGsv_3BR5`zwNP*B?gxtCkiTXN8Uw$qorsaj#iT#m~}Xe!=zNz@3#o722ZYE`D7M z1m8)1Ou8l?y}$4NB|Fm$B)g}ADgv=0G9X$F`n9d%V3Lb4Y=3sCbF{C#8)kKb2~DCk zi?dXH8!)9FYa!QTD8U>VVPbgJ?*w-H6hSXQl-PH~>Yf~=GvZp-qhET?CMStUo&MQ4!;|SJ_~V&?2xinM&PD3l?*O5!p}sqB)9MhUV~c< z5xM%~pf}Y$?uWYKu7EjJ@y=o{MD-m}_#cWy^zPzoH{}5!1!brjf%Xsf&7R&fWDgV;%BuG!x(;n?kSyzF5Dv!Y(FUA}=25Ts;`*+4)EK?!^ z=wj3nc<>hCK1sfWyUI*uW!WzKn$V)KE$+#0I~SeGE4#0rX9;1WW5e;u)CA5FJw>lM5=l`pT(u{u>l9{cISmvdn|!Nn_wNCGsKW*+P)%U}9P zKh6))4vqXGW^g@$h=z$xC&sa9CN1qW6J${~aHe~KhFWjw?AExayC_!3E` zF{#3LzP*a->EP0?=0Dhb14fpj<$b@@;?4nHB}d~9snRBrVrmWk?J zx|Gzt@09uM6L8v`yKy-GdGoaHB-s-G0@FX1Mcb+!>mh})t1ajP?rlvy!4xI>P?=Mt zh3c|0(-vlZ;<>n3A}UvE+72MX>pCpPdM!R3@KRkNwST$kc*1=4nMdaj&Am~h=lljq zReN7wye-~TYII)q=B38P+Au7s^l_->K4Et?<4K@DU%-b-i$jJbEW4RjS2W!$5GY@roHi+y=50W0tW$ zbtM_Q7CQGR`1#W;(viRi&SRiVBakNl)`7$R+ahtZ`qSUZ>?qM1!sNU*2AF& zL>9!j%vj=3g3arDFUlz|+w?_1t-#Em7_57!@8BuV3_moTpqQub z`MT0C!CZLs%Q0CuZ?LdWQI>@XYS*N1y|imDrCVGc&2)2}8XfAQ^Mg~E`*T`r>!~GyeEU*atq>nwKArt5V1tJ5vJz%aDU_3i zZ0CriI%!Je{P%M#eMMpuk&Kt@Kb39)mTfa5|>MYNY28*}5@rUMPLJ>Yf{Pigw)|Ay@P11Fb5nqo6z@<9r z$p<7U4#U|hMe|`vH36_Kd?FkB+GS=U&+W65Utx(8v#UZ)Fk=Ov z>0{E4c1-#vy%+ND)$@p~1gQWC_ZVwC5lX{0^q>11ZbpmR(dZ3LobdWO7XVa?oN+lv zZ%ty$B6{ao{KH)K2$rhskX!}Z!D-g{`lBL+ysKD(W_ewKhNXkha%9Hs-*lhToZbSU z*f?g$Dy~eFcTY~NX$1ZvGI`o2gF0v@yNLhXeia<%8_ncXsiNN?Vc1cL7D>}EBEO)_)5kAv}R zd%j&vflUci7%G=r>?YqE6n5kwjGfSmCL!jb<59ei{ueiG=-DF2VaPAuJ6uo70^7R9cAH8)Yh`fuui8awo9%`x zUHoOrPF7>J_@x!zVSUQ#n$stBNi2>mY97#jwBOhKd#$Q-qC(@hkrXaiYQF;W**f@f zjs^eewcsmA_yUJLDbAlYW z?OJH^U1vU~fLQBoqm@`{rt{+6Hn-?oyCs7q?C*|r8hq33Jq#2FhMbBe_kS(g=NwFk ze3@3e{(dz0qwz-d)wAVyTTB7tF78i3F|fdb`GTN$)%A$d=p*mCFP7M;D_*_s=Zzr|K07HdscT<-SzdU=?v zAjY%0CTN`tZRw6p{N&)%*T^+`6}CFVa5|ohlwd6cRXcmDMZJ&CFA(^HhXGSNi@_u7 zN=Q8}OQ~YS&>&XZ7DF930$Py*QZ=<|6 z;z1>5GA5#(3qZB%iSBibvbD!kQ+2{R)lKO{M+2Ga<~A5H%t>#jVniO#3}#DPjSI)2 z)WI?y?@91!P5Bj{U39V}L7%8&5$VN4iTAa$eKaeTH?8RExvmyxDo}5E%StWFQnL8@ z(xJ5_R<-)jp7Pm6Zj8rM>}wBUp{1#RIF^S7(Fsm{JiG86w`HJkTSs}jDE`@#du~&E z-!I#VpAGreonl#e3s}0Kqk{7?sb=QzXVy~Ao8@JOe^}DEPd7*McU~#HNNO>3?&eD2 z93G#^N@zqlv0mZOM97~y@UnWo(rx7ROQjIuoZMcp9(irW@JNZcK{M!R73a9()mJ(Y zpDZ!WlS8Cpie0XYfI>4O)1mjqE?xqk6w$6c%HFiDG+a`X#q|7m5BI5dOx&t)^QSvH zr$pi}=UkECVPp3H*g8jRx?+{=;z$gXBb;c*{<_jb>_2Grbz(=Da9mP)+>Fij&LK=R ziKMJObyzc!0LFzQsjfR{$n1+mHlR2b1teX3r6o&`t__vc@_YBFy0k|Vn@+Uo*8Q#5 z-0(-%LFFR#=<2wi|EyGNN)h*E-5`9`ROIjB!wcmnz1-46SNEnaHUDj5K7OMQA$zYc zi~;-nStjhpZMr(=doHa1r1V3ueDGT#Qo-Wz7Co?gCM6JJ5 z&1YJ{Ah?05jBNBVajku2DC*@TLOl$2QRSC=gO11;-46X2V-=I1HF@$!H?^R>$y7rHPVI#@9TXj~4M0IB7lJduRXG7iDQ9iFQEV*py7Rni=w`5*St%S80Thf*X(q#2Z zqQw&zPDBX!t0hr(wUyr^ZdG&Z;5trH#O^{}=G;{{{5N+-Yq}FRH=4r&lgupOjv3~N zI|#*kc=OG9_Q)$z2T7S8vBwPo=j}l$sG4m@tgEaQ7ke;dK-Yw?r#3NQ?@G;*^wzqs zlb6a23->q^9h2dV9Zy>U^{ag*{{6XfWrLDl^^mB=*y-a>d9_$_lxdY@fcXkkEKE!k zoj_+}v8seg0DYHItFQ6ts zk~G%*r+IiTtftRbno}e*R{)Z+CL!D1~RBc7~ZOK&x z)DYDgBf3>o5_>0M)~wcczJ5WVnL0Zf(p4NTIJyYS~I*l!t zeycm~E++G5JupsHn6CcYFghSj**p57>=`tq{UxI(}mN#!7m7KQtI3_kew#oH$`1gaS(fdSNq+xmlH^* z7abfGw&_Ob-mRLM6|T;E*2f^`-gr?{`r3D17;6{~V_2X50PrP)|-+}{8*eI(xGHcUTero+b?%zd<>?{m?8)W z`$QdJZ8oy*jq-`j+WEJ1-~C|ZZx8l^h`AWaU5j$vR4mT=IhLl$Fel2AFY~~m5K&p5 z7d0GjUU2I2M*=S?b&m&Gz~E<7ssG_CS}$ANH6Bc?Jt+EORYifDfy4~`9k7UMMl4P!8sg=cs1 zB>PPw`&a{CR^3B3B;>@(PgoM+5SC%|Og77m00-N^77dipmW@De8Q#5ix3b(TJb}aR zp^x_bPYeZ~fW^L%SE&qUQ__^d61Tda-1`_VHOf+&pq{Rsj)+jf1JzGcUSrb7J>@~} zKv$9EMl6W)UMBVrVsw(kip=UAL3De6?BatZOo`pYxDu7E`uCM3_{y5xEKjnNs}4YD zHffu6=8^U|zD&{s_sm%a;%c~gsl0}xf4Mpcb&UTOq%r@xA2`&Mv0jVy7Qv`N!_8%L zJ(Rg%bTtjDmFX~1BYd>*Z2TGj+EYQg(Fa*?x}v^M{8Q7bd84I4dEe+wTXV5)G7rX& zX_svw1uY;~_A!lrU%MYKGIERVb)l^Qest`djHOb5$V{C>7?bpBEIiN1;Mo*OA8^>AKq7sR~DH(3j8U)?X5X6ktI(eiDeotbcg6q17|skUmJB zHOuNM!lV6Eqo{NgDf)P!kyIBJI5vo21Nyx+7&*GNBB)P+i}kcp-cY{JBONOI)Lrac63fO6ZJX zAo&mWzl5CtZMy$^{VD0@`S(I~5KOyaP z$7UbOY`F4ueDHLdoo_h7{q=e>tt6Ii`qJ}hm*StB4~)+^@Fp7rB`JG5Nx9}TR$&(H zm`fbAofG)e%gJg9@hvfpp&Qn?XS2yG&iBP;wbZL?t?%D|9TI3Ya2;UV=?rzStFU)` zC^_>LzBWXnpE6uqzc)-P_kZepyy@qswv8_7l+1%~(0+suyW1(g(h4e_i}U^nXym=u zp^2YiZ!yMpF*{yuKYlaroFx4@QR_bDU9gpAQ&&$6DiHVSo*{TSqSr>%NZp0U=PHEq zUH$hlm9cShEdKE-e{5%NF{?1oiV+i^q+pBr+4L!ILyoRX1u?QP3I zFR1c+jK6(Q)W2}YHn!~7nFi1v5{tK3fGf2cv)C&pB1u$t+hq;|?Vc!EE1mDNwT_C2 zFX7481`~UbQ%(uki6tUDOED?;ck_D0a5IGjUBxYGiutHvrQz{nO!};NZ#u#^{cw1! zBT!yCwjJ-{1_5i;Yi>F@tZ1)KGOk&huPbl|af~s=Tam=8T7dtjj+cZtJKsiWi>ud& z2vrHcMOp4$4k{A&EW2)=SgGZ}6CLA4xQ=*qHivp}uZCEb&>Lj65`cN9qJ5-7nf0m+ zdh-U~B46vH2tWSv&elULXB_*yRa`6Pf0rYTG?F|G13O+Pt_A*!C~niB@!zOF&i_vR zu`~Za>W`a~>%Uch%uJlzO#j1YjM!Fiwd6}24ml6m)4<||?d@$nP&lR`I0RAI5E7|0 zSJGG_C>4@zdc6fGDe2Yl8^4+L)30Cc-B$COjMI+G_QQ@{c1G0h7J~2?{3VEDs6Tz& zo?PwXz|J@wd|N>F_5up__WE=b6n;!w;7_es(c?&9E|8+cg}+RHLxKYDBs2Jsek{v? z{{X4+>jLrl0ucx+6A&xg+XFFjwvP3}6)-6W^39>zLeA}hm5UG)dC*aCmuDfuUF|^y zbe$MT0#(uD21<&FMSREe3XPzHfo_5j1Hlt*As>Bs7SYdv=R-DwhwKUbq6e#QAtQL8 z9v?hBJRE>^bhwBZAtc&f0WmUdA?L$7L-z9ty9e%qgR%wH=J(PZ4jF@-@dj`ICSD3( z@9PU;LETX~0&fNr-NX|RAn_u>-Fb2gDT?FgA43FxnN|O!0rT$N?f`jmdHvwuINa_A z_5Br`5z^5qufjz-01e{;y6R^j3QR5Jq1?iqK>!lD^2Y_y+bW3v5mF>t1Gjv%yrXm> z5>l3c1oEZ3yM5TQGcx3C<1OP|JNAgw?H%NE*8(=CgmiNQ7v8%8^gbp*4hprz+r938 z*fh)_m17RwIv65`t*!qu49pD1P6C1W?*pqO{B9m8hdvW#fI=Z4?VX%ppzMPN>H`Vw zw8Z`A5TO-K$GT2k;ePG|+pnte@5=auNH?%Q-q#iofbRG@n`-wU`a^xU-axQM11jcpddpa#FG;|(XWj)*oTc3yO6sf;_B~mWnWNyG!dwi z)iscCFKxF0XK!nf0sDf8m`G!|rqS%2@XhG3 zE|u^$?O#Nq$HVp9B+TPYR!?!UgANEQ7f!L^xlTx%*X4dexw#cAue-hLA0| z$s9?AeM^)#NZq~*#Qas$`XobhHm-lOhfGN>X2#naQ2GG7(-y4t*M*1bt8qm@e)!>m zy7i)2z+m5pWAsGdVOyKQ zSp5|{`K&fEXKMwMHRS=)vtrOz+9>l+%ClsqU6pDmD}%lmMgd|$a3&&$fl7QF3^m?6 zS~aUd{fQZsuU}Ft-ee)E25e;>4E-UGb`9BaGUIcBsv-b2o&Uy+?m$!dVk^7iBAyNL z-l+qHdhg2$U^RzFsgy6;TTeL5+@(smMTv{EL+)(f+UTyY&tN0n0L$LRQ~4D2_d5Q) zmJu!WwN##ys%%@2R$iI{rsB=`TZgiAdJ)u5OVN8Z4B$)Z7kT--|8k(i_`-tjMUh&_ z9c$mLcfZPbN@so?%|k5!CY{v7Z3A^sW4GL4D&99v28c46*wPmAw_e`ZHT3jVq?Roy z3&{9g@<1cpdj{>{g9P{}c8!-cA z%;@mC$c(8Vn49q5MT)voebNxHHy%&$GZQkRmOW|ko_9G2KOeF!W8bAax9M1uh@|m& z+%P*}y}xsYeso6_8$tJ~fC?c@H_>Sg9_L2gk@4dByW?E>Bozhk$iC>R!tp)DmCC~i zq7|#<&a+?lTqwlx(fYDKLX6eI8>);2b$PH0i1@TNQH^uUvwWmJnh$r1SV3w)s@;hn z->W##bSa#1Tb`(hBPt)5L`vuV4xKvyv-y_`fg9%**L>@jrST(?;)ye?&J$S1b+|X2 zPxTk=(ybb^|E=MJ7^{+0pn*RzDwf|K|4y1OIyJoWiCkhM#PhNC(nxP`+4jV__i*0X zeof}s9!bLQI~);P;B24>Egn_;k?uo*;_Qm8eze%5UCu zxe5;^A23_R?o{=Mc~xOix0boc(NzD^*ITV)EMhgwUh)mRyJK5ynFrmK(J*B6@1J^jBr!N?z$i`_JFk4g`xzW)D5^y&vvH z`3C@n0}y_rt!9tUUUymg&CwJA>dG_%eDk;}eVRC7{h+niI>1-kt$1@)B@s=OI1?%Qlu`{+FoiQQf-|o2>44iXTtIj6 zOdm+S_J|OYwo7<{lE9IO(ab?B@k)}`Y;ECm(^Od$1LK-IM9Uniar(wdk-NE2w9sB58+Yg z=CG;Io$q0v5P}rde#p3ZebuVlSsx{()zt%gZ#=qjLmx^FxI*LRK)(|kNyst z7=?`tAXOFwMtv}vH*D_F^aF43 zxuSRt*(cv2V4DjSqp97Sd=p! zZ{HImWLR|<=wa#AK9|zKsnJq@xGZPZJO4M`ocR`n{amD8)yql9K*rnLB{-%XN5w=}%#g5^sAS4JwEgzWxQ z*O$In2}94CPK;`oM7W}Lcj|!8OS70Q0SpIrQT#RvO9KFDR0IUf`)Ap!Lp;T@$Y$?8ZLe8HO_~_nasolW zW`@omR&-mwV$+talz-4}%&preJaIw|j@DG}@_WNX-gvDTqaamUD_ta7DVByx zZvtu#2%9K2jcfOtJphk}SvN&&UZ(N)#6UucOW3qC>-W? z|9gSOHLNFRXIEu^(-y33iJ;QF7euSpwsE4hDh~U7gqw}qD=hAj>DemDQkL}P;q9J$ zB=&<&zJ9NFZlWRu&(qAtUfSbDm(K)!gFKX3rH)&T=7t`1&^j0W?}Luc%PFef{ONW3apyDcPj~U z5(D{UQtqd98Lt6XYUf!;%LOfmi%jHP~5a=Y{cOa;f}PR>fF3-NEv*HPa`)l!yN zI6fkSl}0lKUv~M?&NddEOxYxrF|pJXp40V(>M%rYS;W)TuG<9{CrzZv?Lta(@q7w+ zhwu91k7Ww4Ng}y6ZT@`^j+xozDagGM*+)|PiOiRGWnUsLL)pOH@=ZIR_pjMyah}h^ zl*E#-pBe5V&qyBje6r?_!y#h7It;B26X<7;(@cgEq3o=P)PxZ=m`3jladbaER-2u6 z>!l;Myc&X=u9S>gC0gY&OGSzLTQI1Th@TLLm0~rZiTygfJ_Ln*eRWbYIKNiW0Q0=^ z5VxKz$z8YStZlfrkZu+++teoDnfkJ_=PZ;~8%{4i7<}~MG2;vZPrP|V`UA4w<9$>K zT*JNuxITd*5j+P|in*j)dgg;JFTMeKsA2d}UrE@UlP`Fm;li0Oazqa($Y?^X8uOl2 zkC7rOxP+r{#MkTSDYsP>)AG?3VlMb2;mI?+{(VASmF)y_AW;txPkD*WY?&qW&6A~D zYo7^XEfOK&kw2DZn*-Z8*(Y_JJfXfgo3#8zFc0x#(9zk#8<3KZj0-e!4yh`a_!>Z@ zJFiGwX(*Rrn<{srRL*?8hC!{yd9%11EDoL^YhhS z5s#yw6Eo@C4 zPXm1Pcjqh@)Cuh(@!8l1kJz?k>$cpnN=m*0=LPXZE!>`|Wo%E~uE(SezcQ_hEC}@N z{tX|f(v5IGv3!AJa7$Jg2C>~U@jvipo`m!x8pM~N(vCMWEe4^x3F8e}u>`dw)Vb^B zOGLQgR8!VlGKGe{Dq>b!j1W=yE5NS!GV1;1v#PAl0;uUa--2k`oN->-Kh97z^Fq9Q zCqB%hRqo=;#Iyl>(A9@n;J-FyC)70J_?uUE>ggYTs#?8dfEwHtoL9=n(T&rp~ugeISZ)8M_C5~2v-Nq?GRepcs}MowH3p|w*2 zwv*MXCwr+AY$j-3a^# z5CrkUf^eCg2;8FD53LSz6l3cAQgJ3}M*i03H4{B^XA9oRVn1rT4&A3wl_8#5Ol24A z_~ZK&AOx?N_powt!S}UO$w*|=QuE2Hh7FmKvZts`Krvr^jQ1{}pC0_#wQ_y^ z>2ThqL+EhAQ2tLhVQ8R~$sO1A!sPAo_%*IuUqA5ki54s{L%4gsIm&~bTfuM!z27{W zHK$>sbem7WL%+Acq&PKxRer$7xEuBM(2gBqnJaZbC(i1Sq&(NC5wEfvdN;iY{y_7* z1o@NjD%e`Wf+V3e^UBCm2ZBC+1(&i^ww~vZ)RCk{wT)S;?aZFqQ+_FSE-2~hyBTZX zsKq-z#W-4B^_(^>C9r^Vyz$6>sqh--58rf~igCN`OE z3x$Z0jZOe+$ZBO6HrbDlwu3<^P}D5dP=kNsThj{VJl-LK5!iy9Zg(0> z9fkbQh9&JEA8?vP=Bnj|uR6DKF3u0T#41Axqc;(Mm8Voo&3j;Mt;LsHWT!+|aLu~Y zvLz0ETgvWFW1(+|$rZ@ym82gThb;e+6T*gLMA|PtuX|{Pc@fBXI^IFu1*|RVm8G7V zQn$nC(mS%jWGMGanbuqXclWpl@Fjp{7^n*Fwr;*)3}T(J2oQsM;bRa^Eq#b9hYD3` zn6+5)$Br>qb|>luS9_c2S=OHx!>e@QT9HS~VP=n0HrQ#*g6VeCW$U)SAl)jSk4+QH zzqDJ0rLPnTlQ@RZ*+wbZ>v~wj?dXoo`R~zi@>ylh;$q?Kax@uo+!>>HeKd>CRzVwH zUJ2^IrN1em#;e#g1{X2i5*sv~J7DR#ZymjGi1&(C((q`R^i zWa`(4>aS>`mAUv=)~<}m%!(HSl1cDshX*Y6JKN@Gq01Tc4AHp0>=-2y&>04Q8~prd znzTc_^4GhlswjybE?Ax}C5O%$Y{O>GYp3E7ct1MnWpPtEN+h>;25ETZeWQn>^_i3URfbKk)Q-xl+ew1I>$$RlB7%EPLKrCDL_HER ztiO_H!$Ow1sLWGiqMkUKi^8f*Cz1gool>0`b>jkY@^I964>NJpHn4u=Ri*q)>r5A? zF%L!6E@D@z znnmCYs@L&Bv=UbwD1VYUh4d5J8VdjrIvsH^L%u z$`Sv>KuyvJ=qR$qrw)Dwa9N2{QwtaG;s?INGqf|As1qdfx6f?eZMVZ=VAJqW(;XJ$ zrfz!RnB6!W2h+UVY(<5;m1e3c_f)`9K*%x3$m9xSgm2-`sS;f5HYk+Jg ze~5G%W0htH3am=9+8ko5+Cw~JXd=seh!jspI#)%PTJeT?2Up8nf+^m6-1XbO+;bwC zGom;cGe6LR+DgZk*JCHYP#m_8PkZKblO4|S`?+_FCe%8Z=jp`=V=vkw<8$Or<-+Ms#}-X!;yg&GzF z$=6E~rwJpOMwspUS8D`me=CLtBm$NeKpwwG@Ts|t#y!;w)7@>wuLp7IjW|C^8Ow0XVf8F#Gb z7$L9AB#v);R?&s$EZnM-{YS_L5gxLfGi}yQZ+@#G%FrOf?8;@;D9BSX zh9W+dN;PvT6(Rmg-t^{YW08=Cpe+9*mP3oX=e#*U%5rn?My-kPqd>2cB6g$chZIK` zbVk2OA7>$zo>E-xH%JDHcp8gIaW4O^jW?d7KQ_sv_7&A!eN#BZ!{>nz`A3^}bH8;P zpuL@;t1{P)M({RBJ0DE*rG>~pqM#9@Kt&OtMCH{)n#;E#JIHb$(+4@AS0EKLYqGqy zO}dH?uLeLk7LCAlo^9HVnt*=GH7F`vA;idavyj>&wZ85` zWaJ9h=bU+Qs1sJ(s9u>xn0Tu&d@J2VT^OOAIB*X#n^X9)tGU{NQj+9<Xzf*C}*uV?|& z?264Xa&`Se$f?))CNceTX7{fQsT0I#rRCHfX_C1Q#IPhU56`|;#wrmATT?26MP2jx z8oEORSMUtWkwVkZh7ObGOhs$1u*x`Vv&by5%HQ{tT*S8irFE|1Gyeds5%bk6L9m`X!VCiY|4`Dl;#Pvgo>iUbE2PO)nGDvlQ-v@AFK zuo&V$X&g0phJyn*Lsdt=TTWLtmRp;$OkQ6nO?f!X;uAfHJ*T~-Y%ESi3?M`5Mz2JEGav9tmmG1 zofU^-=u&aAt0yy>F6~l6o2^NJcXoMd20u|hREI)|Mk?5+9$FB{DgUp|-U23*U|ZKl z26uONcZUymcXvMA-Q5QF85{-}+}+*XhQZz49WMLddnf0=XD8?6CY^Lwb#-cWCr@>y zYxVms;dQ)Hg65$RqGk4d*z@{~=5sSNZ^_@mTq|>~hV+)mK*3f+(mpI#m`tC7H@hVJ zuQgmkxBTVurldY&>wgmw;EUiHPO^=*)Y-AW_&XKeb?nvu^JVe`P|G!pyZ0>M9nnU& zsZ%NQbG+4x_a}h!qK>+gZ$G6$MSXN@tmB4?*-YZ@we_{Y&n*mEhDx%N)^f3_b95no0Qs&<+eB_c{L}h z3F}UYNh%gvDDHi-(p%_@G}MPkP;7*E-Al+M!Mloa9y`Xhyegu5H)yX`KmJJ9XlR8~ zJ<~;e6g2T%J=}%%Jv;ZMSOO~i5ou@B&fP{O+8f!Bm~&}Ak3Mr+i_C6ROwH4lV`7&U z%VQLK$Im!*{bHGxOKw|;07-`>t_u~M4Q1<>4eEoOSRQjC2>Y+=m$BSM8D?t+{E9B{ z9k@4QSmbcUqS*~l81@klxqDo(V;Pb$8s)`cmZ`>3eFkuyYrfq#@+c_>e4q-;;%qoO zq3hj}yqF1(*3o$znZ^_HW`Qo>)inJMHh;9T%^M|w5Z9-|BwrdFXPuY%uPAHJDq1(6 z4oVIGF(GbZ#*@3#Y|Czdwya8K$^Pkv0Onq-QJoAL(0GEW{6cD}ouTr5L(X_$zR=DU zZ#N8&`k6#*6RX#pvLmtu>VySwRNWW>RO%7qw%gy=7%pPcQ>P6o5pEQ5&Zje<`m zARF`MdI`=*ZvLgXcB#AFEc()NTJ6aI!|iUn;P{_XBBVN}J@#ICH;58qUdUqN7VveDoJuOfy+Uhj>v9=dhMR=lBT zEGwXJwQh|vyB-?`0IV(Y*2&oWI>KlNuBapsd+G7HJIx6}b-Vg&o-3495oZg|zul0o zwa?wIKY>{a5mqGxn*8F{0R#a_7n21slI|s;_Xhhq?UG(yQo!Z!ys;|qhg%!qdIo7P zCuii-kHdre?5DEm!*1hnIcgRj&y)0;t&~J^@{pD)>CnZ4MqZiZG5*~3u7%rO3p;iP z2vmpt30ZjBx&L{Bsog_`-kY!@`I zuA7fNx<7GEvMSngN7za4euAS+r5orwvGO_=?%J)PjCvYhfbeLpSRANPStYHrXsOa* znv0>D-&6JA?E9ZDNrO>ezhV=P+vH~ax@?a%xF!jV12#h(f+I2@l>qyVX*E*y0SUc` z9EC)Xb@=3NuHPI4x@K;+Vz|iai5OBLr(92xs5;dDEL{`!3UzbF8VbL8r!??4n03xt z6FcrFJLjg2A{QWGic$W4v*P-3?y5}&P9#)`pG^|0&rbaPa;P?e9N}S~8eYjR7VXN4 zL8xEg4+vu#MB{?v0QK@Of6AA@x3%6O9ml+6O4&1sf{996g_7d|y49STpg1kTT6-BD z480}1E`vp-6?O0#RbRA*EE*J{w3 z630P>uhy5`e&`(ZVknBaylRp82iT9R#pO~u4qfJHJx%q(I)wejZXMRB=njJ=9!k~^ z`^CaJK4{=78@{&pP&d>bsMu~Xd_r zkI)}bHVw{`GUUUlNo;2Lyd2Cvl6e$ zTBY?LzN#$ik(oSa$x_uVb*scsAz7EVqrXKhkB3c>UVea#_b$0$u!&o$Mt{5p-OW4D z;O{ZLV3_Y-v`UJay0Og%y<62?X3AbB3z{J*_$X9d>Zhy3B@2!ksUu)Q5q_d(Z5IPj zXwbJF3T^Ukg5ke@w&-3h#TZ??u^bPk+xT%yIJx*BUVu@AaZibBG+O;`W{5T=`!_L`-0!L7I@&WB%a+2;LRQ1`ywyxH zj~ulhS=oNyVayfDxJ|myEr%)4I(spH*EUX(poDQ>9+_>YzA*^V@kM=MDXl6+8| zRi6}n@D7Dj#R|S*I}Pz&jg8K(l~H*Y=u8!*h9L#{=1%?N{lAp8e)Gm)z-;c!UL|>9 zX;HQv9m=6Cs=O+4k4e{MVPOUMfmz?FxD0KF`?VEc7RRhHNT=01_wjRk5Bb?=u7n3o zl-cj58E!D$S4gifKiCWNUco>SHm;|z65S*DI>1iQK8C~B5^6VS8rfrz@mLW8`{~K6 z{o6851)P}f+u|6BWA~t|qHM$*Ux{_1?A&x*W6xil!F$d@x@(LZu=NH0gw8CYD0Ya9 zF-pgKM#KWnFb}%DOz1xUUTBOu4bWYqSX|#ZQ2xHdI#p`|UwwDZdx@1v#mL6Eb5xL@ zqM~!~s?5ZxPK2DKDxegQ6I>hWG}fBhRjr6p{>AjcGp|;~#k`23#Ejju`#JfnIhONj z-c&5Jqi+I#>X%z|AA?Z>MSCfD-V`t|Q|zRE!jDb>05co2Gwn)XahjdO3?!r2pp#>i zQZpD$)K*>}(wWmY@03rD5mf`giI7r%S`m#`C|{rE!N+L(`dO zkWZM2o}*wdqRJ1M^GV$(3_~Usf+bl1A*Gf_PN|;JPkuG2%9yt70PHqw69DYfpz|jQRsRm4GeMSuv8d5eYq!$$g z6x8M06x;_C1hiIPEdZKbW;@{eM2j0;vWGp7J2;&L7h>V|9!!Kh2pI<=vH}bnJ_RIX z|C3Y96eAELiYkL;+o)hzsROyw z0+L0I9Oe8BgucHKMtjTmR|>^iNH@CTbD~y~UHk@3vGAzr24JQK3%Nyk6!v^CL%ySf z-%jFCW<0SZM~&-y2Jzo74(jPM4%J2a_HJKJ1$!Rf_o>}Zs>E2;E#r7S7)Y-K?O@na zHDrkS4f+b{0CE_#sMn~bf*%|D5wZ{5=pxUscM_cCYLk2qOKwE)TfDGoU{xsU774T_ z_cm1EW6(=)5O)O=|F_TX?*W$}$j~zop#7lP+xF1tTe(NyurFRL6p#2ZQ9wE%D4~(S zzJ0#DSpXGh(%^9SF`nX|FAsuaIBeM~@5KGF-iT!65ePsAqNPW`2FVM1g`mYmz^G-l zLf`nJSkSLEF}DJI-IS{=^WJjJV4Ckd`uqdj46mwtk-Qj4lESZ=jl=IE)OS(burgL!R#Q zKn97w^k|$yOM|^o2@!~b2?JCe`)1x=rR34!&u?tn$P0miO7xhx(8LWtYmy;%NJ3d$ zgP`6tn32ijM0TMWg+#nagf!VPugY^(ZiJAxwO=4UmP3Mw;kQ|Yyn6MWKJyZ~X9PRn z@F>Y4`ko6uS(3qngSY+)_H@sMKAXU7v~04HE?hl+JVt{=K=l|0UMsVoz39hldTbJ^ zl{r5}Gu+W3!q960p}%hr*T!~9HlUY^FxB-H*ZP!|3fSEl9&kW-s8hBmUY9!b6vTrH80a>0F`2OC+Q61y$mum!`GXt`T08m&{lMy!6hAjU%*)G!USDxoZ0=-QYP9Q*>;B<#&&GXrfwbwTq`uN% zW4_H_*rQAgVh=0%1uiLdTwo~D&P=K8(q5SB=8sDmr6=CXzgA{3(5w`(6vf9KG>xPR zv3@A@wDYVtA9Gz5=mBFmsy#CmbH>VuI;J|R6M%Y<_omH&dVXxcE4Je_AJBRo{fG;*pLVD|yC*KWUo(MZ zAuwA**D*@H-}&nUrToY&|7@=z@pVm(;p|A=h)h|$-FA4OJ!kvnF7>~P5i!o zo+uX4A!~m?c!(P?xvxF#{d=qq{utn2K;Gk2htKMY$LwzB+{=HfSHgaW+Fd2|+4?vk zYO3Ew%OX0o7X%y!nf7043Y_9dEp6^k)|P9U_h!8?&YV7^L|<2@qAh+EOrV?Lx~2G) zE%|;@{ImCpvR(CppsmAgr>R4M(b0MOCQSe5HC!XHFO^RhUEi`GM=hJe7A!4*bLV{oGvag#q&>Zp0uyCogC}`;!v3d~#n>b;|P_?lSTmwb9^kXyjHoCeJu;7+Y`1nbJbwlA{xINxi zQJKslgA$S2rYyKv)(p~MewpqL;*m~txYFiUX&*={J>9f;npKRxE=eW>igscLoGhOs z8YNY6=k$8J4ae6=Dn&{k=7k-8Q~4(C0g(9~g%R4n>fb?NP2Exg&l;OWOMN>x4Ik^)<2TcK)2 z%=?mIa*@5lsNZQi!~nY1qT^+l<1g&Cm%U4cZENnIlSI2ZCVct&{fT;YaU6jJ!@;)^vFm zZ?t``zT(k}IQlp?7}xaW7N)nL*)o-ERhBYPClvly{H@v!b+f~)bwDE$^-)?{z_qY+n01O?8|X;ke0a|!4mHc@k>g=cAgvs+1a;BERfC&^;H*`dxW? zZ8g$Ieq1M5{t9_XF=JS7N^Y;C-9`!t$r$ zZ(RP-J1)GsUXtk(qqpzrCXv7qp?6! zpZ8{GjImge@d}xns-Hf}u#y1teMs=nTC@Qg5eI@0u5|69j{*}cE7~)gfd4K93@3jf z6Pevqx?Wb}Jb4tbybo;ZhqQsB;l$ctR@LM7AMkFKI>mye;}k^Dfu|j|64RGxo%+tS zKde0ru@lbM4G3$J`r+^zllemkI|gOguYA{4*^;tpn{*OM)T@WLYY#(r7b6^{yQP?iC6ny= zxH((He%UwRvx4xliH;YF*Fv2ip11dX$RyX;Sk%sJa-)#GMKF%n@lh5PZ9OSeOydIg zsyAh<>bluNJI**gX&T2Lq^Pwt+I&ybw z5^{sD|KuU`!FT(BN#jPl@Tblu193MYCjgsF+2jS;)0pNB z!&~Y7eyI`KUL9r-vr%+EHn_R6#;%`()1(nG?7K1Xy50e=$TIJr z=Mc4lrEKIi0Dt>_OiHSm=T65TBM{U!Z;s`j!6qg%Nzys%SC){cGPUav=<8TWs35?0 zpKw*#cqi^|Z)Ve&`l$!-5>OLtiy~Nz`Fpm-m>=o*zD6)6Js)QeqUxrL=` zs}O2o$>DK&oi!|hQEqQj(QO4(t-K`9=0zFmB1_j{FoKUkvfKLm-HR-%-=y>>rJ1$- z@UXh3>rW=bBQ)~+e%|m$jNDuNY{61c0Gg5SK0_HN;JjY+1%_TYsXoX+jv z8EpP;$RZlK9;$&9Po%kAl6C^%-$q!O7BZ|1>=UZ!r8?6EI9sf) z4TN#t*D~9pHpb#E&fbfN;MRCHYRSdv>4)aUE}gjCnOf1NHJjb;Co`g6Iukk-i;v~d zH(IqE=q4*hy2=+mo|v;qUCLe9qo4BvFI7U?aSBSU7c-K?hS9wS1~bXV3|u=lRD)fK zIJ!1tVpi_&*CXY-c&VzP^IY6zO#FD7?JUJw7fodW$(Ufg z!JIc0w_*rXaJ-9evyI-*0bpCx#g z5+Z9W>sf(~xmf{vCD-x=6+AihuELq!PV`HA@$_-u{LR>F)D1g#-?H7qy97xR)k}U( zKf)W-yt##&Xg^5+1d=+~LPOr(WTYnE@tW-nmqaXc&+!Xa#{U#f7o=Tg|7PTzr=1%g zIIHXvslMV7pc2L7T|A!h^(m<&9EUv3nvLbMLvsZZ`^zKbtLDh+E2HMu%ocAcH3Pe- zl+t$k=|i8{YXmWB8a=99lEwS)>R*t%pd;iYsY=3VB00rOk^<^sdwZPzVWrlMSqLgJAamr{Fn>P_-20>vw8V?E>< zuv9bYhKsUTq~Sc3o4&^ki^3Yd#A4@IXB3jIokQS(D+4AkE&NeB3yR&G$BbQRL%1}s zrxM4CGS5ATCL0&C5jj?E3+o$3;<&W}3)=j#c{yTrqvIaW@}+~L2$fY7ahE02vix`` zi`BVS$A-D%9}qb_a#!7uwhEB=v?;_$<{=I=@8!4_{i;=Rg4uVTX}9IyM|(SmNi^@` zE~_RGzUSxG01(@j9ZYk6+jmD|Uw{#GzWT#2Sh$ohMzNAaxR)CW?``jnRHID;7=VQ> zr)v0({seg{z@m>vk4-j-J_(C(0EKd9fVxA6sRuXDyLn?$*9ZZw0!rnmof9qSJJad& zVQwYmktLPJ?+RNcrQ(-9Zuz6zcu6@TW-JQEbipt)O+g5M^_2le$uj{3-T2t{Dh^L^ zpN6O5#(7G{FwN{D2*w|O7b4u! zH;?8|)p(a0NJk!^RiR;pTkH^9&xSvy%wAqq)6iUZuIduQG4ewI`y^)5#O5S-FsVuL z=!LVc*e|^90c4Ppzs_Zy`_IwkTfjK~y!xf9-0(r?=GPaz-+|CvmG*i-cL$ve=mp%Y zBcB$;s83Lr9t@V1=liZna6eqw1MgcA2;lP!Qq!y;<<}hj7Yz}MRuJKF<1==VjeW(R z;#~;YugpAA3n@~wS~^flpCk@&UVx))X}J-Kl9qHWdJ(Vq$!K;Oy1jpRi@;P;NyC$Q z3%NhtG`5WNPTv!wgGnL`j8_TJ9P;z6A&L<}YsjV14GO>IZu8ese#7I%>aXa zZ>cky+vTREPCmltb_$??`G`&ql2#&7Rl+T!N7BAI?K3xa1s@+tqvN;07a6rJYrVu> z{@9xN$(?A}Ifqgc_@z3c$p~@?BKp6$99pjDf?E6AnZH?{7o>^}($gwe;XTpXA`ku+ zgF#v(4^Ca5ivCp6?*+ZlzMnlMyHT{D@!!cG`o1`YrFEw*R0Tg0zgO{V>)Sq>rk5Oy z3Zt?dW#q=M?##v5JwCtir=%v?57zT{W_x$_&LUYbl@u;YC{fvtaEWw$3hyP=iZX#Q zTs?S)CN7r%vM7D#OtC&$o)ynkyy8+w^xMP%;LT`ljolLA;g*l`*!%)qDF+B1BbxKq z92WI;xV?HghGGp)IJZ)xKdd6)NuHBx6~tRXDj+Vpf!t8?*g}4M31`g;7*&pJ&gfBw z6&;}z@x6oG?mhj(Gr6&^z~!pHaBA9)E=zyyW2n71cZI-H_24S!$UVuT0F|4WS>)nCF9AIH~eK^7DC z-B|@io@}Icn+eCRImyZ*TcCPn{q3n~!L%6Zf$7hDaPm9pAsHn(@uUch+H&!uvmP#R zWB}NBBCv=h%qEt=B4G7nIJ)AB{`}zwi{u%~#x{nPLtl%<_qV=Dp5MB^>tov>(?ZnG za3Q2NmBsoN36@e;(w{0l_KmCsPpevYO5t3TcqlaumT^1#RWdXv@|>eYElAQopOm78X3ONBIqyB|GmG{!q<8a7W29R>^;%zpVdMoMZ)=UUiu0eYu4}N zyt%U*mTTb7nb*0}u>X|;zvW-gn%5iB*c!ghL!^lLC`3C{eqfVgrLyU}Kvm)n>w^7O zbx5DWwcmFCq~B%s3T+A5M9KlsY042R6m+}k%Rk{F~#r%-UcWSDX zA7gjEsu{Pv4*Hdq)av=Qh$eX-?r%|-x*!E{agpIdS-Ey4i<6$f@#v)Nq(TWboyYJC zvr!~e)jcr#Yn5A0HPgiDE$!Lka-7bm+HE#zaC`UVP>SXXDt+~12Tl3WEV;XV1Lsu& zTRXYJ@YdjU4;MqQetWg7ahAjN0+<+VAE<=*i$)(Ce!rIAD9=ONURVQ0us7RP>jM6y zFEaO(42P3;OrmHzd7e%eFB`2L4C|v;_{LtKOrqpP;{e3`khyy={{cDFiql74n|}3x+N$BV}Qea<=d%i!lTaVW(P&r7Z40 zTUr^xvYRrvMR~Ag2rJfMpO~dHKuF181}4Ch-ked?++^ZI{(^0=#kny&AwGP0BxI?N%|{yu!A7JQ=QOXg$@FoxT2suw4^Egkk;UgXvr~|5V5uFUkx4H_>nels+I$b}%FU&ecTu$`N;Mg9@LR3%96dxC zlPre+5&$M{BqB&pI}m(9?O$r2h28(b6Ik1X{uBxH)#IxBK`QXV+ZtBV|2#70nr)1T3S$C>?x+)w3ZC>{!3S$E{eJnBM$Xba7r_b2W$EE} z9FPIA%LDsxS%sJ${D$;R7wl4Slf!$ygz+theP?@}_t`9kGFJ``{MC+>uSHi62iPtH z6_2&q{ITXCy3On1(`nA~NY(l>lB>eJ_$Lau4|$8sFl13T6oHidhJ|AN;3*I7w63 z&YMdchi#-GPI!NSdD@zM$6G<*S{x%e3IeVJ}R%=$T{+o|*z(;L#P3(N2~3L#+PQtFp6U`+0+`7Do@4LvWABlRIxTR z^7I$6_7gbEYVyW@`=q8@#U@lRZx!J=w_LB0&jnOKkjb#i`=sxNNzbKVPE&lM3mDb) z(NqcOpSbY%$8%`SScIa*TBHRF4^8YpyNjC#oyR=}Be52Up^Dl%p_|}OWD4o6s9i`1 zl|#4F$EPb8++hDTJ{j9a%z|sJ$ik4H53|M`V7p$knVwc#aCj!QAC+t~9Y^uyJYtzByphz0x z5s5KJ6DO~zALm=Gg$bF}C_52vuWQ)OKNolI{JJN@@oinYc3iquRlQu zokh(5`zkCe%YR#iCFbPe{O9T_@&9bZWTz(Yu)~bpd8p}$l)3^wNJJh4OR@};$Ze~e zYTc+QCC7y-tppwM@p_MkR-qw)ET6ME(Qysu@_6dTH>=*d*;Q&rL%i`3z$Z8!b~zAwz%Wa>8-6%!v&sGGxtGOfw63AOWEA7V-YH`| zue_FTU;a@Rah}hJ`lSiQqqlja&3OSx=y>NxYFciHVANq;zMCY-iTFrn2Z+u8=&JKu z_pf=ao?DxJaONdoFk>dH)TpR~(wv?A?69HF{NUb=z5Vd2lbdV5Y8$JnJDjg-c&~b| znN7OI(8V02t+Ts7mvhdo6O36~aC*i^D%wp2hu`4>E_^Bz6lFYtuuvW(AS@5e`z8Kcf-U+X}`k>Mf`L1m9;;W9phH4MqTi zU1~vYWa&B^&73u)ClR_G1ICQhk5h<;7VdI_pAfMPZ*9g8|3ez&gfpxC{@UB(_PT96 zp#9?nUK6g+X(S=3fbIujF4;pWCgkJA*#m(**PHI4Mg z)Sb0mh_7!r-xb}+7jvBgG>`C9nJ8F%^q#OiMes=iix*_6Eu~6-sLiAue3@1HJyMTf@KeKh`y z3Fg!GEPr*5n#u3-0fvE22Tcpv&Bo#NY#}>ATu>);_h4{U7Udnv={#iMKXAbM-#Os= zcb38bh6D9~aDd+NZw}n0z|=#{4n=or?9|H^zh3g|QVDs;i0_Z5*JPf{41-eS$M%qwAMj z_MiP_>u$msb;p_6{0Q&Gs!zyrhA%63@~^!Y4k^p&_s^bdcqK;sT8!Fu?oIjkA00f{ zobOXxsW%^9>~d|j#Omsrne$Gn>HhQnKZ*B!N9XGFAb(eWX3AO)x^t&!#d~JGs2NHrSXENFGYmlC$9*y2!-2V>FjJ*lfWZ3WsvTfg(MSqZvu-9idHuFR>XN3Y*96@{i!_K{Ba-(o73;XAj(2inx4`n^gB;`1~(!3!_%~OK&(8Gd2&TqpQ8< zZftT~k8TSs0E)afUURk6$qK$XcWZ~zOujh{Ull8d(*HR1=rtEMoy_Q$!+LR>0+U-T z_redXKh176?0PNQwd>>a_PZ~0*<6L16==H1??+o|ICvo>YA4fZjrSQD?na0)7~5jH zzAaJu56FC_K6*-6sAX)-PlFwO3{?Rk0j^xSN>~t*&b4)=vg& z%tB8M@OUrDj-b;T%CAk~hx|eY)>G;)hyn5xsAK~;6h7JzV0Wqo(^ZC~mMT5c^r?sY zeitK#O;Eu&D_}=4Yo`g4KZsBa+JdAfgijhm3l-}Q=B{TFjM+PZ#(5@WLa02?X2x+u zL|Gr)U`}T*k1K~H;({2839R!w^c&obI&m(4(a7j`GSFnCHBHzyyBQ`5eN#v+6v$1I z`-L8uQVZy;gsG9h9LpGxENQVUXl`W%e_>Iz1g0Y#mz@yERAqvvs3)8|$oRvF}_$w+l8e$tqV^>QTpfR!gzb@otWMllgr!V`?e>>7K|CfP7&C3x; z%%o^yt?FtA%cMZe!ugL$r?ZPIF$?q8;lI2=xtRa=Q_@QHRfSa{WWPtv3Da`e%f%n5 zq~xe*LhKka)ac?~%u^=x5UX`u40oq|?TGSnc^KA)8ue@z=2Oes)!=!=!s0!FE-?6- z5MV6{J&tHjMquR8LDC5Ej7}Rpg@bntcuAp$F^uT1TEd)ry zR^Kv?glP69LLBv?*&-_vF<8}jIXB(POu6QfVIE3v?ZW;Olmm1EdUXFtVCs$404AY^ z74v#i?NPtzDoD*qnYFoFhLu_n=;|c)dJ6TVT|Cj`=tQAJi0rrcm_%rJNTH!M8K)8! ztP)exoo8+|pM>b&W9BP8_;6IF#6MnUdw$t;>w`#TZ!XIF!`*Ivlls+-m7bFWkJYrm zpNAeye042WbO)TqTv*Zl7EdbZ|KM!nk)s7|Xv?hY8i5@3j8au)fd5*C70dnjJmh}b zmnn-w+N2wz^b9(HqOsMdo}iUq-~M+a3ImMrb=$<^&x0e{y1Be^X4x^~tL#f);_lwd zo!OlgdwLd2RFHbc?h)U>OzgXmBli~QRE`WezEakUa^`Yr1Z-kK17K0yKYmrqbD*iV z7Ti3Ix2SvkF$-Ft?G?PVJl=(D#+8Q0HBJ+|@mG zjr{Acurze(+#;oWCY7W51f@=fP)n5r3>s;1>p5i6KYOKAjNb7r(R&9^FKbcJ220_& z^2M=EABGL$4mRa7^472W)pyy)6+=6b&&r9pZKdA4i>JJ`(=UHoD&0&%j90855zq9; zH){VLPIt=h{<4Y%hA9?H0*d)EPfe^YbYBg4_kj0}^mW#yLG_RIUQV?J3Ew}Z!kwCc z{f~dI9_x=cLjgDnhDnL85pqJ!dIN-5wbO$i?Y z4Sv+!iatS14b27rhf(9dy5;^87%df`IV_X3y&2GxSeKcYl}jI%N!7~x3&O9ASxsVH z7GhRnmajS$2M1STR`xI8RQ}Q2+<}<&A4AYT3Sz{%{GvP(BAgPOU#PK(3v-FGaEXXW zaIy>Y@UV)AvWhTs0*L?TCSSe$i&E}?`S<>B-u@UPlQ4nI*dg16IsrP3+`&HF!O_sc zP;fisgzKYW?3>%$6rfi;->AY;L_ibsB#w(rtZRixzumX{pI{-ljtBIr3d`t-av pW_zk_@$5pW{DyA%?~l&K)!5n9)A{RRu`zS7vcpnPh$~9K{vWENdNKe2 literal 0 HcmV?d00001 diff --git a/doc/src/week42/figures/structure.pdf b/doc/src/week42/figures/structure.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d21e6d3d9188c02c2d805842b6213ec1711a0660 GIT binary patch literal 60903 zcma&sLy#~^&@SkHPA@IusiL1A(29B@_=26upe8t-13*0%k@=R)YU)LD7p@SUa0K63~lT8#S!em_qUKK{+`)ni|?bxo@86NXKorBlgbKDcEXan2aBC_XAB}OaOU3nfZf^pT={H zn+2`=zcu>tt9nNym)ll|^BUU$%*jOQR#Kr|Re_6Y%J9n=eVz*Hd{95Mr5}^9t_>dI z5A`f9eZj-$k&%3wMJB@sR_)I`KZy)I!@3PXN&-9g?fZVxEkeP$|17~Pf4uLoJE(XP znsi6AW8**(!7YF;?nP83>qywGnDC-Q(a7Lu3YK*w>&c};+coFu><>y1@EP^G^YhlD zf@(7wT4_3;@lbip%T-n|F~ju<4T(9t0NH6FR%gJ3{A5iNbawFBZrtgzcxd{ax)TWR z>qHlwh`%b*ep>MN{l%wNGF`sgjaMqsteJ4fk4s4=KD|HRXYmJS0l}kPKR?$|E}$fq z`N{eHyzKRTZY{M*gf^WFP|BNx2DQ9vA4-WA-i06n`AHIqIdG8bV+=RIp8Vdcyj+PE zx!##UlL9wv2;G%N7P{$EPyn>M%$~!c>~}Ot35bR$ZG%zk>il^w6c)lYvZA-9t_}!k z*#bfE4gvKwj(){^*Xgj-mm6D=2^(Uf-B`yB2zlOybaK;>aG_heL!yNoXVm*Y3ne0RD6G<^ma83y03Qbh+K#?{D_(Bq3s;N|?QLLbMQH8ab!f7wn1Cq1F zZv+*m8l)JiPc*b9YbRBIM1!J*Z|mtXhB*x~eeEJRKmi`m6Ot%rCy6~H`cHS(+%syQ zL63sRrV~)eQBJ^u*boVv8Zzi`U*QrC`xEAY55x>P$1h5BOhRbR*CSH1Az?EvMswTp z%#USMfYtWQQBA(}uL6Ci){zZC}BL5;zFcY%6&H{M8(p{*lUFfSg9lV%Z7 z<#OgRm?H%=j73d!WExhuR5A^o#oz(AiGS={8p>l6&8|u~&;!0fH7d)`j)5Xno^KZ^ zMp%_SpQJC3KhhwzYz8aEHWd`b)>Hy9>Ksla+4~#t@ zHH6vz6?GA^H;yhs06S(wOBxNs*l2E`O5Nx$BFrmTMo%_C=;Rn|KA6Nnbo8AlqA~~{ z*bKe56I7C}ad@d-l?|^G=Ip^Z*ixLtT-02%&kCzt&(>KiPtdV4+-7NFK4*<>oTh1m zE16Dwt_pf0p=ukjPO~Us9h%V;J>ntkmQsRD)vn}gOOh&WXRM&n!gJBa%V@zlPT24~ zD?_X%3K}!8;UQy+$+3yt6k1S&R}P(=%GYDU@v!YFuF>Jk#S_sENNC2t)^^r&Sw2Uo?T?Q?gG$D%U@`~1vq252h@soo(v2^T?*fO|(@5@AW$3U@JMH2cTYr_=L%U(sUmL_)t}f?d zvQED2TiB@OMsDMz^jvAI`qD9{Ib)CblLk9DA34-t=b*&R?#P@M`x9EO8a;+dQA9ub zA>$}lU1u%@Hd#abUA;pv*<@LGWT;2fQJkcs14g3|Q9oQOxZxgF>Co0JpW{Ta$(xcS zm<8Uo#qE5A+xZYnTjf2t_cxGd_i%Pm2HE~JAi$!e30%{VKEeP7MBeD|PPAl_*#6BA z|2r9zXor=pbIsF=QrVi+{)9L(6(1{S9PpccB`uI@zPyt?%`5%D%OSnw*JjQ zWPiTaPNn6GW@}S^&yvPVwuEz!(Q>ic)D3LwqfYA9y61FuwO z%t6R+R6JHMFvrzUGnlKM*NJa0VRX0Q4~^s&o)8wS{(9|PEk{01xk0Ql@N}FiYSvD+0mLiydNe8(e%^G$ipvCQ)`umiO*D@O( zy0YL>1JdI0&GX5^PJV@w766Cg%--sea)Mn`r%L%-rMiRFv}IwiXEga>oAU3Knyg0J zkLFkM7*{@IP`6;us63Ue8GXmh&$h$Xc%}vR%UY)F{AG5jzO;M#*&%MEVsOtq-w&n? z?)LN5_U;*bK!`3dq=Z*5`0?6oo9$CnG0(g1n(%U*NS22y~dhwxyoR* z79SU&m%~QEN!8cp~pH{lV+^DFFG4gnSDO10LO&v%b`9vee>apC#mkFA6kJL zcMryS^m;IIgYt%~T8Go_7%m&fjVIahV9ZJbG2rq6sD?Wb)D}#7e>r2U%g=>bsa9>I zMC)H9JhUd`ey*q%o z$x9Wl-8ZL*v+*57222y0sl%A$8WSvm&sBk4N@>(B3WoKQj$XZ$Zw}hAQ%@g!#;6iwEBRsnjoWB7J$K&GI zyh@L|@#$I%p?NSG%!Tiuw=*M;Osz&9mRO2Dn65^C?XmPKP+u}PJY{PoOp@&m=T5v5q9p+tv+BJ?bt)Az}m>uf`)1+ngIcPcrArLn&C%ray9JY>Mm zkT%>#Cs|I={LTSq8T1eZ?>0|fBC0u?tR}^MAtP((N1mDoVh&dgv}Sqv#TIJj^g z)%r|*&GcKVyAdrE)O74igS+A;RJ4M-oiDZB;Wwl*y;*KE>|%yz$e>$-B!I(J#4n=iF+lu{X->Um%y_xDdhpny$ON?tkw z00`pKkRgr-%uJR;39bkJ-WxDo2JFotV5q+P5SZ*4ARhLUKnd~uD;X#RCPwi3XQ1~_ z(NED3z<>aG1oG$k1u%3l2%sH9G=Nk%0i_ry)-kgb!A{Qs!!))C@IJqvAPia!0PUe6 zA?yDn!6i8T4-lXcQ1YRK*8OquxU~GQ0ks&QAutXe@JZ7}Z4e;D4e99E+1cgdgP6%- zY-wi);p~G5tN}6&%o(;(ts&ogm;_*t0e-z$<*dd0TVaPk>ip5Bp$@-yuv{- zF2Casfm%R@0O5B4wIZqf)ng6L^$k+`0IvP};KBQ6CVuK1{mlR9KmvXx!8A4cac~3` z;31Z=3_u&ef%PV;?1{D!uK^I2AK4L1?qIye!8-y8UJ0JS`|}nA0X!&S0nQDF`re#c z{e^K5XR~J!zV^5sf9Rgu;pqzp{e~dK+ z^HbIO*6N)dju!)jvUU0+8~u#&bk_e~HG>fS(FF(;{Pht4T!H*|Xsz3Q+2>#FLH-~e z|6KX6_m55>oIo^qs{vmFGy?E_>wB^VYYhN^Zh>AOKKBCtn(I3{08K+eYl)Vjzz{yIw!@r0%}v(&IQzfb z1T+BtG^Jr*`*j%nKlVh}fqDV7c)T6{{FZ*?C;wtj`l`P9BYyXgC%V?R{vw$FK;QWf z3Bbd9#`eO&Pdf?lRPv#`=7FC33Ox7wim73n+P?47iGZH-K`F~ye)?h&4{6{Xz_lxd z5zyHD7Eb8uLX6{T;c2B&%ot=Ap`J-pYmu=^3fQk z_Nb0L+6@tK;2!&Z+mFGoK>MS<1Y=VT-uP+2>>Y#*!^2SehaC9;s4;N9-eRDj0NS5@ zyM8r<;L*yX|tiU4Cv9u;aCpF_?37Kps|_BIvid zP`0bMqRhZ|WoxXglb@)MwozgnL@q@zKF;JB%reSk6PFALILa9`@$oy5WwLW)_-8SC z%M?B5LT{STC$i2=>xYqDI4?N$3ulhDr3M5H1>{0JKiv(IC!{^Ef{-@c>H_5vqm1^J zK!u9Dd_i1?<*2Td=07TvhJc|thn0R*w8Am9n73(^4qiLahAaU}&sbE`^#r3qq0}5q zpL;+{E#?U9=Hm-0qbFNI+J@Q6AF0zNvR@m0RkOv3;3AO`|>Bh_W zoNwy%cv9cm|FPAPj+U;~pE6HSTJ%E{STre&UqS?)#j@9`?VA&H5y5A${Q5d%8pJ6r z64@sX4}4_6Mec@=rj-2R-83!8L4H4MHJ|w!+R2E6 zm&0>poiK(uU+uCOr*pxGdN4I9b5vgM9}Zj!Jmc0jFYorbWzjA}HSXi^Z%VK1{i5hoVB{JZQn<}1 zxr`bZO+yAO%j^TCC`+11L z65u)aAjpN(?I;FyfVik8SiIkcCcwus{WhqxKPcX}V! zm%sAIUH?SUJNqGAy}`G4;|Eoryi6z%+regHv%2jQH10X}MWRqQE5@$b(ao5)MK8<@&8i*H%$;HXTKCjmg-I!6C-5Yt~XK2OX`%}cL=@(>@xwF7(Fma^38DK6<{sFZOa-{xj`;P6>B zEaGI&BgCI+Jg+Or<=#5ihS56B()oSnvh%<>_`Yy@I>_$nacdUg+i=h2=Ti@HT4+R^ zuPLb;4HeZ;p*s~$DLuBz0K3X*Ga}b^IC899opzZcCOfdsxnx+3&jMmU(b#t%$eWVB z>6n7A{(LXYX5G<<-cUX4$B?9Z5XXYIoGWsZ9999Ljcai`Pc`s^Y1)&Xxz?6(g{d#8 zz(kIT?K;FmW%{WfsZG~9dU$Aa+kEl!!ax3E{`y;rdq0MQ;c=fdwHTE*&*4i!H~D4 z-Ri3MNyQX*?nTRgV1TMy^g?(n5YX)qfz6MfEQe$4i+R93qv>wZ6U{{y@~8|i0sYgX zo(PkH#MT7k5XI>yPbu7qk4F9#_QJ9%%J>NqQxp8ldt_Ud(X2)AJE+5u#L-9wtK%MV zA#bK^ddSypS-M%fq(p{L8}68Iy#TZr%SoIba`pO|11?IRvVWn;aY3OUhKs2iO@dVI z8Cy`^)uz(L=%HI2rdPYaCoVr~G9lAZiPG~?yAypI{#~SiF|b}!CM%E1@ec5ICvl-5 zWAg^&2R*ON=;k!mb+hL;19!J)e6 zqiV8RMENFCa1x@08-N^usHgz|meB&L;w!C)G38IP{!lp zN&bF8?KqH-xQ?$#5J|=>)Ndvamo6qmY74a$e$uQM^7hgcqmtgxB$TYwNk;4zkHc=W z@n%6o%VQUcK5pPb{OKk$tnRdSS-q(G+Tr2p)=xC?xanSc z@FB9?V*=<@{*esH4USsA7I)s~`RXzBjOcf6t~m1ir=;Uc(M`-a&8kM+zLrXG6njo7 zb*-w2#ZPJ68G=vcW7}pY3nOQw@mB@w%%j3Esd4_bf0mC)dz-93v~SQa*2NPGo!bN% z9P8~xK3=p*8_L^D){icIN3S{;$|xSC4mZ&O+(vO$L=e*~&|KE^lVPqg)m z-QmVR)1nCnF$u|ZLA0kDvDR)g*QKXU>%|=J@OU%aAQj6kJ#8dRClrdCP&vb`9=)%T zAoQ*@vX=@YLa)0K`N=S;YO;!j9J&rwZU(AYD9Oju^$c<5oQ9coF=^nrRj3@l&um=W z{>uU)m$d4(B3^1D8aG(kLAF6RiaPJM=wVyNWbtNf z(o;Wl5yTH|xXAeT9I2|RK8%=^T^cpRm z{-OH4;;vsDSfv-Ke?V}i0P7&Xz3abkgWk2*Zd@F?b~;UaXvBJ15jc7gZmR_ch`3#T zu|>%p&Tg;a_c}T1E|gO-sySGO4bI61LDc79kCCQdl`xx&Ab6!san<7JH*<()IuSmm z4h7wc=$n()T?Km-At~3e18-IYz8Et^cbHfQrfQ5847~Qlxzi#q!R-}$fo7Nkxoctv zUq3l{?)t%xa`g)@;we@`7y;U@PYNY%VJCMYj|t!FvN-v^UwzX|)LOEa%PS#vtb8Ed z=Vrz|gal_=$KQ^=Z(~XAFlTI$AC>6w*cesoH8|fg%B;4b7Sm!xWNkH}b z@hBZ2!|4v9`SfB7ad2~ih#jvNbns5FCROJYysrAw_$fmc7{@>)7< z4Q0$IYhP7PdX#z_Hvj#HJVEp@^}LfSv?Y&Bm?Gowx{lvbX+nQ))(!bv(}z3ZfZ9-f8?kc@i?Z8&{mc{?Za>G z9C%KRQ~nzX^~JoBnE{pbv2~hOLD7d(tiG*!8DNi!3yp`G;mQF9;ksn4U^YOBsB#I! z)lE*`V>3z6TXeOah2#4>WLl>dYUqRn5IwcRhx3eI*O6;&RqTj4PK%6%_DZI0^ub8V zzWkEel@X>nUIrGR>q#?A;Dj^ebirtWVd`{DFhVcrHuX2Bt>>}pdfFj8tcyu;B+Dn7 z#pT&jZ-L9Uf3}Uxd|jIrnMaOyTXD~~%FzQ&t4bZ;4?qGCmw-DNL)?}`()1xC8R6l` z@VxT$%&pPrHJYMgTM7^(cf>p4@&>o|1d_|0xvRPS?)}O5-c66wtKmxD4_nRm1RMXy z`ryKQj9p{?zp7(P8Eqsxn^>D6E_tnc?NJaDLco>+VleqVjHK6A1{4SI6gw)*DxovH zf-Y4Xl)G1k1c3ZI8jK+pE_Ry6Y@{G%31uN!ixV~POmiO5FvX%zSXf~9blc};kkZWD z8rwx&*L4&<7^9U7GnA~ z?0a{8KFA9(C@PgmJ{xgW^!9Ms{JXN)Rg~#Z^5qGdc5uik<{zwuWreYTJ@8 zmon7Zifzs0fn(ZzlxwyMduhfT&y?BvVr;I;)D<2d^E-yoyJ6$v_uCmIqv15p;(^p2 zolXQv1p5Dakz{aqc{WOXI7VEYBY0iI(F}a zp;24M$2>6!RGNJ)nN+VCe=>#H?ps(96oG*9UGfF? zZw{&a3NxA~BNs4>^RHg0nZ{`hu@ol1lvc`X)F{>|uo}5K9zN==bwtuFOtf{sbA)Ym zaSrh+EVFC)UO8=~grWTp*~#W;x?(u#A4-@?k~VV*?y8WU5j|(r>SqHTA+hABv&F+? zfs@;kYk1YGIuBvEp!^_GuFU}Lg$PK(TeZ^y^Tu`q z&$F+E{E|;zHRlIZ`)dM2BxKoDwe2x9E1!y6+2AXefsSz87Bbw?$04iIJf0^oKup49 z+jpeL*yLzA?HMTNeIg>twAlY37J60bYMmu#c=pV^$S5uj3JO}T7R8-0_^aP@?Vuzu=6#CHFBtj1gdzw^EuE)qZPDAg zJp+nfN*2Vd^g0Y_6H9VYcH%bgzEl^M=7X`mJfBR@h$rXV^v1geBlT9yWB_Q#x+WmE zrH-$Q-%RV!tDh7{LZIfv!2pAu?ggng0S>b7ifyG8yfg8MKwu&LZ6qtGO{-BLugYCn zTHfUDT#**uNoXydi;=g>(kELmZw-(aEcHXlxKlBO*TeeW|64^V#? zx|Y<5JCh*h7a0XHq2fY0a}m_`v=@5flcB~Yci$~;MH+*wzptw`?IKm*R9RXqDG$d` zx>gffC`&6GWXTvPs;$F>3*`MC2b(OTWCys>n5QOL&}?bS?tNVmk6FGLC%MS>0Ie3= z#ZWVy^1?4BzcpJEt3HpzB0U%vvTpA0QmMn3GfRk?pem4%2ukcaH$kS{@W`~AO5R*n zIpKPxDa}&Aco4OxhzWNO&WZJ`l%&uiurkp*+T|3aPeB7W;_xQ=E@urIgUmKXrI;4} zIgQZ=@v;g!k?$O-T(YOMaqb;K_T2@)cDVT@Cs=n!g)6(@AhAZrg1||mYzKCUD!??m z$rWw{Lb1X&-OyEYhdmMBNlE~VO&kiI3(`7^%Dt1Jo+^vyWucpU)@!KS?J8{F49vZm z`Ff^7La!lS7FLZ&^jR)F4BzE=S@?n|QKb1q==aX`&-I5XSX2_ys56Pz4%jO3C@7Pp zWqh8i03WKzkr~}ljiA0IMZEO4%J-wV=iVBih7GB}ULD=)(4oL>`fta8q9WCFDX1QD zHqQjDg#N@cCn}P3hypAwu6Y%M^MOrkBAX;%4tLj#AF35trJIS8!`ZIkl`oN|NYjow z@KFiKkgqo`zWvw=JWfm>Ac5{}cGpHXH~SS#p0<1uv^7~6LV`9S%I{SUXf*+zz2me7 z4u5<@mHffITWE>2NZZuxv*7{WB_4%g3leFPzqZ$!l}3303G1nDCz+Z@!wsXLKiDg3 z_psHDp9)^P&;~?9Sadg3TZcf~yjyXt2on} zc0$;d&vkR|>qZdp%Q*`8eEN|2#0ow`I1y&9t~9)z)L*cq9lY4g1LpVr_rTM=xt9%O7g+RNG37Vt>%`TGG6zW&i-j@x$?Y`Qo zsBAg*+e72-R@WGHe5vx)1m0R~e&VDo_gAc2om5fGs+(fKIn6rsDXAX0GxCV&7KhI- z8;olvJSt4gc5z>+^mM?m#+m&}+>!rqhvh-V8+eBmzQ0Y4YM8|Avt4!PT*proI;1M$ z8_kfK{KeWDkIgkW17@C=xg~Qd2jRH2*!8(!lVJL*kGC21RqQ3NW(!_%GDN_;l=~H= zm0oAZ@_8iVBUE1=&(o?ct9$%h+`>wBaCN>PaGn$AN z1V_p2kx3RQOtF_0eqaLLy-XrMBv1cD`;u)f8!XrMUg@tI*Cmc6dy;n*&Q3Pfh#kU+ zL?R2fCYvUbcX9+)dbHeC1=+b(`L_BcGo&TH+EpUXw7h`JKDNaX@SjxY29+k@8iz$P zPq)7K#S2G5#W;A|I8He}i56~?EZaMk(vCpRa0LJ&cDWVwR~yE@3O#tCXm8H-(DgY6 z4cQX&D3#`f5435}p;N(FYZaEtu-Qc@!b)Uz=c;?H+QCb_V%(7OYI#6u&$8HW$x|rM zr;E9>;4>c8YZrmN>r@#%VPDvv5SxF*|~@Z2^I z6L3-+xmO$LG-+Z^a>+DS%@z*9cr|aa){BcTr&I0<2n@E!@AOulPow_|RmX8U5sD1K zM1Hj$mZM$5<* zu0&Wnm>q|Njuz%mm`N#S?dCdHaYvx5r;tsWMbb(U!z`beIav$c0*=HMJ{gS^>#y6Jf zYi&qdRFP(exUT{DSnL{6&OgDx=v%CN;eH|;OWygBDY2r_$6VMf9HZ%`D-W~If0{I} z2z`baomZ`*Nzr^~$*bR|oz^u*K4T;wZpkxruQm!yF|&`BW6m~C)~!asW7nMj32~?U z?-mAP?X1M!O;#XU1J>#;>%RLumkY@7V2VCP7MRvHZna0M^%JE@<=xixRt+k+_9JF3 zkdsHvSI%Bn{JoZ1l#I)OZ3a@l0WoMa@lD!HcgIvem-mx!eGm}f@vMU08y1UwrSJGE z-8SOVXTw|ZFf_qzLXYyF+MLZyZWhsEQU@n{`(sFU)l;#y?&wpHCP+rDP_}8&qzW8` zy%8g7DF0dliP2P~s9=O|I0J#qp$}TLWAWaw9ilRp)>?0Nw7l_NJL z*h;6B#43wMrvMt7Ksk~NP#mdk zWzp|ClQy!h#8s>IEP~-PV0#LduTgd11f@$Ux6hRa|4~~8&Y*ND@v~FMM8m>N^=0)1 zPEC|*L4~(EukYV6qkr}V8s0+UCfUUu1-H2(=6R!9TIE|H+NpU{<5T&zy-313Z69y4}xa=b&R@V=vuxKL*%IYq<;|-63(&y%ra5~z@57O{+HTT zX~V&XTw`;*pdYd@K6zu95Lsk0vt9tZN82}vlZ?}8@c?bm=l2Z{!@bJhGX%AIq92Bj zzru{2ibEQi`)Oef{f>L1D|Mf>cNSfoEg%n|k~gW48}-ST#ZoHKY76OeRcWjt4~uw= z=tUO6LIR1mDjW-{nP+@^tJ1^wOY>FhJNsgZ@H{-5H>^5KPf=BlPwKCligFSjo$)E8 z{3RM)Lz^vAWUMLS7 z{pyJfsfiGB@rRzcP7) z4SLv`bK(5L69(`ZbdeOszncnscBX)~?#>sZ7pT&^0j`sgo#5%_u9F{m$iC%H2j$eD z7_30|^QpXcDCC3;l})>$igu5EbDoqxV7%QBk^dj4@gKeTe;M)rpa%Q@0yQ|8IsQjy z{4Y?0lZp9%8)~$Gs-WB=vqlFyhvg3g5fQXVrxDPSN=P-)K+`lpPc4$}?CeA)H!s2h zEyno!`#0woFGu}K{q4EUe&zCp?U>(A_b|^*^VBm`S~*H)5KrenLU2Sv55uB>j+amJ zf&&5y5a6>Tj7N?I2>mr2HJl41#4>>BJhQ`NlAuI)HcSL6vd30Ja0pCHpaC2L z`-6xI3Ka?v_@hIhe2gIm{P$8|j^QbQs;xf~5S+`faf0A_7olOQS~+%(pD)ll!8Jg9 zcsRmYoxOiDE(&Cj{{et1!z7fYTM-1{77z;}0tU*-D>!IYJ1UeZ{p8Tm;o%@`tJAUA ze`T(70@5)gqY40DLWpt+&j9jEgYkfF3I5T=V8?;xn}RxhL(hw7cHQ0$KyKnDpN z#Ih60**j4n*%!ghtEd2$ZwMdwaaMhw^h3UXIRX&a&i6U{iT{xV4)}ov5h4WV@M7G8 z1!)7(7_=(*XLduePoj=M0u0UgaRu7oBCYIWF6=U7`pOQid3*xi9+>SoFI@s)kp|_9nl0ac> zZ6BIooXEZKY&fO|AS)@p+U8;)@A$JwQ9uy@BJZIfp@8z?0LrO{BhSkGQ?q!V(IGx9 zdMgH|hVTsG>)R=TFF~99cfAihI0ADD08kboFHau0@6RCNL4eYS2q^q9myy86zT$pH zVOrj=yqEk)rcn(5x-t0?0N$UT?&yoMAesLgg6{k}{jC_`#h6=0epcf6rG3*=(8kz< zy4ycI0=K)1f&>5{^a1b-NQ9&BzmO+zVZ6E#^?_0k(jovP^gZ@;l>0fl{(3>I|L!8V z_UGxyBeEAkgVz7BUxW$(5*(kxpZ|I6`h|S|#eB%8{#_OS*@->MwYK)zxAx)u@^h@i znO(iMjm?~M71*u{pgJ_c-2b_(g#5O2RVp~8C+++dMpE3*a7hf%+WhrMq!;l_qgvL& z2@6*LW~TRDZ}Pj%pn-)|65|;1>!SmpLqL4vUz-gOu(8vJAI)g|2IjMWJ(U&X-~=-K zb^PlF3It$4f^;3~vUr-21lR}c+>mf8zp4`iySE=gEYCvdm-WyCfCfRjGnWT-2jnr6 zU>FATe1g)-pH|>M=tKDBkH^mRL)^!QnEd4rzSBQeKl3X?enkD$2!f&2=CeE13bN(f6Qv{znj0i z^sE1&qkrVS&+5IG)r*yLb?y6oNgu+wu^F)K>*xQ3cM#egF7&-QpRxaT`hB?+DyUU} z*u6#pB0@9JMV)unB6OJxv9dW|;F!v^h2l;dqiNi@07rBs`pSaolOH#(8198j{FrmJ z7LPl_pyO@5vZ@clKFnjy_0mfpwv4W2J)DVFc!5E;mR5CJI#_hH;MIHUcD`SJXc;K) zTd5pQJlZFbDb5x5y<9BW*V@tjeh{8t-fVhd51f2QA2HTR*hCENts~QLoM9rb$gnMH zKexwAyy-%c$H9$DGKJS?rWkSD_qY*xMD;wUe-O)&?W9k(eBf$b*e+WrD;?q>`eOL( z7xpBUyqIkkS3EHK1u;$z`DqC(9%x9VpB$wOQuUXL5x@obWc$~zOSeOw(@q=%Kn8>4 zJ1@9@O&$Jm>gP1E+v}!6Rb*KkEgOaS+^--%6|~WjJYNex@2o8+P%b#|rA!AHhMy6_u^6?(Y8Qns^$oZ+$ZU=`3;9?X=26C1Ms1g|*dyI%Fk;hD z%$ZjEEC-^m7--p6*V&4?Fb{E^oV&F>(lQWTn7)0D2B0`;CK^hk^o9X*PIPU1_qO?0^cjzNMedH^?&Z zh1GFf%edJ%yE$h-ODbqW;pcT}FR<`)EV+LL)%G>v@s^+g!un>`;X$Qa10M3RC|MuJ zcS|e%-X*=QYe$C0>A{tGFwOcl!|50CBP30&|4OJ^R2Zr5ngeVzBDEmSyk-%RWXwfQ zRCk*G2%c3aycK61Lw4b&B#u~Ux3>ynhdMqImNvlGyU7}!$|Q%?%R>fp!76b|sF$>a zf#T(t-+)P(DloiEJihwVsr8Xt40DtpoL++JHANBt)c#9d3-P^S6&k|k&^P9``WD2r zF^)NWW*Z_>y7zPpVf2=%a)${~y6rpeU4MGNuR+ZB;ueHp`&82!wC(f1<#iiYO> z8Q+}=;WDCUpi59TVa|MN(94zZL_$a|TZSG@4 zhEHjC^L}mSV=&`-NUnY+9f}Y}{3n*=Wk3~2+zI*4w=C}cah#&K1AbQhNXc#elhiYq z%@_F04s#Zl>EcrT9FG4_$0M2X@zXooZqs1LHxi}zsw$c-YPx}v;h`+oi}w@Lyy-BK z>Pv3xsw-$}tJ)?`_@4l8@|1TAp2%0_Bs3$vOzfPjSrAL4ISamzra@-Sj1*L&0qAQ^6>= zdL}b2$TEB35?5Sy@KjNvZJ2H{Q zU}0Dx>+5&Hvw+&<$-oLobU56cIl8wA8+4Jo2)fFfzx(T8^T?B@L>z@!^ZMHT0_mJN~`;c8=*>bg3z)9qG&6l!S}c`zLLmZ3_73s z5!4y7D|nFJ9s13-`hDW$ai zrt$G_%a*`XbS~`^!hrf`$P$7S(#r>Y8ixoLyd+y_%Jav6Ea)PU5!0@*+!ouCVIVF8Bok%ceyZm(BE=e+DS1gpsy!9}8-(@aOi=?_?c8{~ z_+JNqS9=a8{MD=lAFk-jP9+3A;l0? zn9fznjI5HPv}IMUJ$bFtTwyRfnz9&TGv`o3L?!4=aT~TS%N0WO;F|e?wEgcI@m9#5#X)%{hSb2ode> znF|r4ceQcWxEZP-nPv__0OPvrsHk3otMo_k_7$aoo&R&>>O;)F(>u5Y-P(Eko)%)H zX5?yZoT@z`Uiu`p7!0N4PmK#WWG=jX5Qe?6m@TIqh*y2^n*PT9<|sw1ZRGvZZ9^z@ zR4$Ku#;XJjoGu)g5InIEqti!gid{4sCfb~xv`V}Ac$jCKl`h2#0E=ZogDRZt+&$pvgb$iR|I|+l5k z$oDZF#vOONLtYc-77}qNn)WQA#k#%auhM>;y}TQ*GQgx>fp}mciT|%w@5#oy9$9gf zLqn$5ervWVAWFHeNRhWLKX~f`MC84#mm*6z`fFEPwJHbz_`D3lOWhwr$6+o^$Fsw-8<&Wad8%4njy>UI*t762N*)~qDDK8V6cqd~2l ze?HPNuRq>sGRKAJ2* z%|1?hpT%JNd`q{U!~x2_rJUx~MV~qepj{`sM+W@Ppsc2EMIznkRaZ7I?U^29rfC%GP89pN#MgGYrR-J-18Xv$QcDKg3NZ1Tuv6!*X*A_ zPK0@dnuvv<73}(IOj%_-ta(+q|B`4>AE)Z#UuAmNr9Qb9&dwG|PL7}T28FA%S0&01 zJiF}nygkNK@Qqb@;0rzn?4mis}5xRw(3!JGZTAB6#6IWku}jpWQf!9kOeGkWp;0rwh=@5 zpsReWz{in?_zp?)%@VJ72bq~zD~dg9;PXe()XMry+c}-j6B7q+rLX7*i4ui9&%8Q4 z{7duaKxUpKBDmE4NST_HW68Es+lLRWVT` z)YV25(|KRt=hX>D?4z`qgX`ynh=3Mcs}uKpaehr`>*qSzJ&4GPBLNAf3E>*|{G zM8+QMI!v)CwMBM5Ia?N+poshq9EhH+xfJ7doam{BSgOQ`-9gE7Aa1B}?-oov^=1^T zjHcQYBUdexdy!Lc{vV8;Ly#t1v}My@+Nw&Mm9}l$wr$(CZQEw0ZQHh9^`Ij0I{V-K94J4B(=M73x?W+wi*&X4fXk;{C0_qEZzlVO<_-n5Wi2<8^cRFn;&Q0C`eX zhn|Zx=uAtJs9~)tj8a6SdC9bD?<<*7a5PpCQfJ1F(V{cP>wz@HLXCIche?V2qE4s$ z^ImZ?keV!h%5joE^pQZ_eooeJoprBhg3@Ga(@Pq#!UG()L`9v#u3DIICPzPc-|$QW z8}P+BWx2c6feP1}XEhM}?pTl*pWRX~|7Uch$Mn|R6`61Z822#PWk~XIJk|5_a!bX+ z_CTp#d8!ja{CwrWtEL39h5u8OnQLZ7aEBv+s?1-?GfjbHqqw3p}o!du2Qx}&SkE|tCX&>v5!!G-f zZBawBdXpan!8}%$*=aq7T#N6P(eOUC!%#%QfX-}5Y6*E2h1YWSYu``kXS4T<(B3GN zjQ-Yb^VMO;f%V(F{+rcK(kn}iDY!$2G*N?t@^v9djQZrJMyoMD@$Wvr;JSHlbs(Xv}p{KYDxA{`XK(b63@q#pTnQdZT$~r}T zKoK>2vJb;*x-IMfRbT+Rld1L(XSKiCK;|mO!PHJE4+A(NHgQ*#G>hatnEsg= zjZ-_lO#(FbT?IE(GKN|;EmH}hSty?^U^hGBbu64#yHhXQfX;FKJ1NS)m=3H#A zI56GL-w57l-Ie3$Xdv6OJH*1XkEb5V(-!>0X)L5A#=e}zBB;wrp2SkMmN57dKGu}G z*?16L(OWmheXt8h68^jA!N&!rLww(cuL+KpXjN*jT+1tWS+RX)00?o^S z(=YZJ+hMAiYz;oYo`6V01w+dE9*M=c4fi;dN6>KG%b;-%3sJ>NGG(q!WZMxqF zN05$PW6nb$n2T6?lfg{V+@V-1^tbzOZVnS8XNJ~KsHbuH>>Dus_wxibI^THOck~qv zd&WI;X3#qK5?(j?uwL;ai6_f@WVqA(AwgaBU=4o^S+Sg0?o)d&49Tne@4bF=2OF)? zzKYfEYGu)u1~C)ovo&--Gy%9_z>GvF&kB$C=IZz)_n4P7|yL@(WC=p~L@ zDUQ}#j5?1c-^u|VJ-lCvs)0?0pS47_d!al<=L?Sb946YANGAhM6nn!e{aTJMudeBo zodEJ+ZbsmzNGS}N>&as?lZlZH5`QFBqWgkIYfsFN&(s2$!+8#}*#0I9OA3*ZfB9XO zqaoV1(-A2d+-$5c)Toxlc(8|K_=}{vaCRu#0Kz4}r7^mR!Le{t~(E-yQ8tM5MQgJ~&I<+dKVddHvl^v146xF1}E zvX&WW0+(csmAKdtx+0GMGCksYU{hT~kJKU2C|Y!l*%>TSNwH=4`)N0Zp=PO(0*d`; zNS((x$1KI+eR>A6S=8-=Xf>;u8e+$jZ8%`Nhl{yQQrEvOM!1|h8cZYjoGa&U>y$4j zvE->&mas;kBN^^!puW}*@d0>t5vQ6c$mipk4^D1RoYy{BH2|-;M%h6-7<_l-Kr)xW zeh>$mI-yZJSGq&;R_VV4o$Fm2CP0OlGu2&<(1DVbKI34lT9E~5h9NKg$*`7X;P5;t) zYK}5{@e~!|jdUpd$)gyv|Trtj_3()B#i{+LS89t?_$vz?;;0+9_iH#^8-gMDto?6V)D(p^43*V(Q)NbI%l2 zB$mIw-3#UJ;;?_}+0t#18W8_M0J8**;0|Cfgles0Z6Yxk%>Kh0>3 zE}K?}a!H0ztIDr)MX6e-Bh(Z_YX~PxJUN~=+JL~H>8dD*L_@tp8~|CPF_LF$JCu5H zT&QY(n`*IIDwVV4Rh!pA=dNm^cQ8b=CA9z{f3*C{G|<4?;LB$~gWvG#5ZqPM_lNzR z#6M1nLWtvKO%l=&_D#?b(_itO*X9#jUDlv2?+&SsPoyE{R`odP<=q`lW5sntooeBI zuDPb|1WBI;eSJS37yoO_XJi8%prBDqZ7-PrUdg}ya{F{oJ;W>C4h)p?KoFSf3{logH8fperi#9lZ`CB|iKh9^qg zow7dYJ2#~@=RALSXsVdHY$ncCq>hBs5G^;8>%PNkl@5&EV&e@Y6*5})8=fszLEqaD z>>~sNUSE;xy9n8I7mEUtEiKOv8@)hYu+jj|!@2BOrZWlJ;V}laTs)!U{j@8`RWjri z0)#BkCYM<2H522ES#y!FkX zXLP58tvwqMd-)$~N--jl?vJGXG`hxZ8g$1HEu_kE%D*t6A|L5aP2GnHTZkC8KHb43 zK@8OgXuZhJ1k~c+O;i(ze@jT2s&NM&tmTF>DV3ri3=|3SZ_i z$j>h$aTfX;;{25BS*Jcim`)Wy3%202b8HPjNKnil8n`SeMH~+aj9uee5*mh_P`o_c zJ#?8IvFt?B)ufmO&qRA>;m0-E!?~_rSgw;ag+dP%0pLGzn zyDBA5-HqWj3N?fwj!S1FH@gDDi*b0u^V~krhX?Uu8!Z!Wog6J6;<^nLk00f{&Qq8A z4bjE^>V_ROd?p zGS7$JSv36LCbTmMUtG}KIXC7mD-TPgaDQpIwOW14EYnzW0LN z7#Yk<9$#AEO*vd3F$IOsZCh8r0xVeYP=QV04#X&wNw`iT)O~0=5xzaTs2g4Zy9qXZ zIxz}*wj$3l9K72Ku7f)ZOj{ z>EH6@6wH$g!b0A^hH-TTW9!q_2l8gd0gggA#s%89`GJA77bdFvMx=$@L^B8l|CGT2 zkyB&>u1s5kN$52F$eiq#7O2JD?Y{V2ng`y zo*Zl+{4t~}*_*3V)$ik5(UE67@E2e%AcHdi2#~M$cXPu4!zAwB_4$YG>kT?gMp;?F zQuNz5<(sYu2}uw_U#_kiWFIXtG?1axANb)bIHl)nqKaXH@Z$7gTS*ELfCCVA6{RkI z<0k~+i|6%V(}24MZDjQ`%=WM<%w#IZBU^$h3CIy#pBI-W4NE1F|f31;@XWHeY7gN$pCg*wB!=@EbaD*RS;dn-~kAq$>JNe4o9~aOlqpUz% zE?M4GQBE!A>nAaNJr6@JSg``>pmFDQXTA(ynV&jD85?EjkY=_>eL#s9xRMMT{OtBo z8zU&HI)$aI6fSAU*ufT8JOUL)j6F##tOaDAxRdB;rYn8BoVTL3TzXERyD9iCpI)(7 zUs|izJCM0XaITSZ4*B)GWz)$MnwBTe%fAI%%0Dx6We>sScnk`(zwvA-I-P20Z zXdvmY_bi}n6-J)qtea;bY`<_H_mpOmd%js5Y3gyCYF4ERve`^*RVe*MQ|8IA%9-K| zoekiDU&&e{x~_v_%m*N!$yDpLpEM@KY6eVE1P70I?Vh2LvVvdqKI_u9u-(6)v{SFz z+N$nnJeNhe#1==~X;x*!jZu5JbbumzX#%FoV&tD6Tr@eL%O2Gm=FeOOQ2{9*%^l%B zx}QS7FaqO1>JEEWFT*^rC8=hS$5nV5Hy;k;-q38ixMj+9eAdqRA*VWdgs5WZ!d1=1 zCeKoKhS64s8wB5Au~5RC8A1%VM@JYLuyzr)9dTK zN(Z&g$HqMxA=fteHdO^L$Zol0X!bYjG1e`gW9B@yd1?q;%aGmLB$*NuXD#C$8lOD1 znaAFIgv$)iqzB;Rv^^bLmLwZITS%i~f%4Kv{rmx0FCir~a)11rOF&qlsGI_JI(0EyDnQuz%j7%Q~+%ml-* zG_&OFlPD{>@x`b%*sXr(K0lA;lHc4Hn4LqD+P(QEgpg-)X<^aGd-unGKq}T6d*T&m z;df4%j$WDXZB3q-1p>u{TK=lLs1xgS#%|^fl0~mC0H{^;mI`T3qze7Z@cDZf=abPg zr1?xZhO;V%B68oX2pf*^X{kg5mmb|%hI;&)3Dnkgit-S(q64XwjA;Bgg+mTtw!0k=V|h61mF23A%^$*8z*W<@E(b>sz}Cnz;iupj zI;Yo9><_SMq-#VUvRD8%*ea;M5+**m^WQ36weUFv5zb6y=lgHq_`dN3Y~~a@Peez& zCzUgwTR~glS~(CyeBW|(ukp$mTZW&C8WW9XxkMe@Z{POM?p(eCvb@Hs&yqRpzE6OM zTH;-gUlUNp!5q=6+(U)zV%Zvuv%~8nQ$}7cJr}0>M&FRqn#~^qX3<&hcDWHI4}?P( z`eZ7a=ZtbH?4Z013+*3JcQ}<8)=oJHPEfrsmWyZ6p{J*@WThQ62GGBbNDaUrbd`JL zTm-4BKt$Pe!FdFyH|u}SZ-j^lF@({ax7Ia}-fBQ5Z~MMY5VlFdY;5T*y31zeX=Q`D zfH6``s-Wc0tyPQyvj43R@UXZ0u#BlzLTswm;^MH!vU|A!L?XtKZoh1qyKy+4{y^s# zh;9=ej?w+n58^_oftp5QM(v=b7S3E4qr*4M=Q&4scL!%)DykenIcVWOYxs&93R(jI zLZ`bIH=r>Z0tN^qMF~Gt8#twI{IG1247S%t~xyOP^&i8e=@8oNllU>pn}G05f@ zjPK*dC>|p7POaH+*baX<&ScPKqBf{}zxD1aTeks+s4cs5G^#gVlW%lb`Dic0{Jo}M zWcguh!-m29bl4VGb2^6;o)n7`k!6HUy*RDIPIYgx#t|co*;7?RaP0G~>_J}iH*PZM zsGeco<|{kkamL0NTYMEFbshBDEFs{bF7Y+-Nr5)&69G+^X+6up&7;8hr9_sx96Mck74l3|)rNhiARS znSk{+x0+n+AulfUhx9OSaf>d{N<4Ej$mM&7oaxo#He%%*O{EV)FCHc~RjblB)Md=p z;sf>f<%hPNc-wJiZQr6LzELlMu~CM;T@a`VU%mrY&dWF*9I52zasEWtYk$ye>KYof z=vy`f(~DRPG*oe1TB_fV?NA)lZ#Ru2v-Br6=FW*`w{!)tvHKV4u)?v>cmd{DP9We% zB0}0_J#)AdzdbNcbt4wSa(MMk<0RgkYWs#lL`L~|@_T%5V{#mZGwQGBL|>;x8_e62 zGLyV-Vsopu)xHQOKgQFysyIcB6%DvE3~jmeXj7Zz|5)hTu~nQU_!w%CiLW0Mgm>## zq>zdVN?NoI1HG7P{AMMJebO8a)<7^r3B)F)xFxL=H#njU*OumlKe5(Fx`ColD6_j?Z!!Zr?j!N9L{<6cN2BQ&3t;le=*^X zv0|!i*yKItbD=y@oBmMwH`W;QdL|;AfC&qsWN$$vD{6q@!DOs*wlvhIBVYVS-kzMD zb42;{a9UYyx{IIg?drF~x=o)G=&?t9W5Sn6TDev5PSULtnVSMtne!FDwCC56sQv~_ z?WbfBC*|T{NKN_LL=};LZ7cOF{l%{{Eaf9+fqz4^FEJO*NH~5xJ^G;ExwhxAmF|Xd zjqu)K^RAiATol9F?cwIkYHppseAzbBNE})wpKx!tqK-nCPtYk^@#)K)y!%Ri8t&5V zZGq|y`vQQ7&`;*^#VV91mzeSv0Y|I9?6p=wT7oImZ~RSG ztN$Ff-H-@hmOc+{oh69)4xa?SktDn`Lxk8iKgiQiwfs0BcfR<^+0jO~Lh^3Df6dbR z7`pY8J=i^yaJbVlm*b{2%2fer>wqo*sZ+X)tB)%7DoPjv?n>oSBmp^NhLl1_tc~Jb z8pmg9q}6sM(L{3`vVa6zn%Y{FrU|!r%cek4#n*?CP5kEH6{psWfFBdz$9oH=aOwWQ zoltM}l|*M_5PS!`WY+x;obdKYFP6YkGYNoB6hNeV@Ox&+cE#|O2zJ2yofa|lh zo2%i#>u&MGZ3JO;DoT6w35p*};{JBN_G5>zWQ?Pl9&q>(R#eFS@l$p`$}ucIa&ob9 z-vz&cFI$!An+U`{aFsEyyZ}y65~;+=hR^8g5Sw%F z8S*&r?mIs)1wrkBoOgOz3DQN#V56OBk679+nn}MITF%j5;KN6I{cgGGIOvqkR}030 zAC6fPF;k_zix*Yfv5-Y$&#H+m9^3miB@^bsaFWXr6d%gG>YL9;`Ceo(NHW5mf+b$~ znJ&EUE3Nh_5x5-dFFu!8&;}FZYPf2bEJU`FJ=*S**!HwcuY--t$tW z>%-U8)H;Ig2Z@ir**#C!_XnW{~gRxNL_8*uh@wC2f%xN&y0iAs(be=S=<0G`^fnuySse^#T1XB&O>83>nNg`sI0vV8Es7zx2g=<3F60wu?`$I9TH0 zPgHS=b>Sohx*#^Z(=QOU%dIZiY*8d^QmH;lt_}z`wYy%IAruFca+;BT2=o>&QBF|w z_UbM(ECK|KB^&x;;+_aATGS|GGsOk3(X71J_1tP)1Ul|t9e+wqV?nnj8(3d`7cXci zVqWvD=Un{n+UXTBwoXaGEi#FnD)DVJt{c`9V@40(!z*n6&TF+6o@7%mSzp!Q@TZYP zFkrE_oxDI)Ne9^~dl)TCV_#F>P@&W%Yr8uOvEp#(D(nyaEj`JehNobNW~~W_PAVu~ zuYUhk#2iF9ux@dv&3SH-x2Qyw$_6n!3aFwh7)Y1)fhE6g#P8Pg4ak;qlbF8@o=D%V z_yO4)F=HUKB>q`GD7|^VP3c9GEOS&4Ar#fvv3X)X|Bm}D%kJFkHVDiNziohM?46#r z0U{Ot&QOgwlCh51R3N}LO+(;)u-Dk#QR&UZqnPt#l3Z1IZ^JL3J(l%3{=Q7sdof?n zq9R^hhJQIxAfjMggnyjX*@!Ta87#@^tDHH-`| z3&@f?46Vu^qOMb&zYt^%9G&z4m$(XyS)%o?IFlqAViSIV&Pxv2SHp8tPCAo}#ey1? zPBtDtLQNd2jS==(>GG~dC;iiCIwyc@vypn=4w4($j3pCW^o4-XKSdZzKEU~~&Gl!F zYBfDijo8X{P2;j&_wj7rNg??P;g7q5RkB$yh#v|k=(t7;GhVj$401+cr5+hoOy5k~ z=A}2ERV88QM(mfU0x#yc+%ruP(WgCMWqZF+kGUr8OUTaWX-Q_U zs-{*eZV0RGt0-v_h$u;+M;cH3qR8(92Kt%Y;CtDqe-=7qH8&`H zD?ECjHD6*=`Yf_I$8u~e6=rFUS=0~-B8AE41O5UPK0uH>%?1gFSV% zwY-t{MH&jv-xH&R_k8OJanT}s2i^rqHNTE66%WbuZ^haAs;!d~jBvBkD~!E(yOB&C zl!x#guUxES-yWI3=ZOZ>vCPy<@(0Qq}QJ)%&9z(A7+1P3w zKjsqQ;d}_kx^v^^87e3QS&%8d6#7H?+bPpsu@6|w%v(;CFjN>VF)UqD$*uwc#j*Ri-4(rK39_9D3xf6j-B(OBL`tqJbu&MN#L5wCA0nH~%|7w}C1gsFWUMvc9I=v*{2kI+22( zh_Gw|Rc>juKZnecN@PdWPcqJrpp%(!k6{d!R*fvFo>P2$+OkKaAxrclC{|q~g&IcJ z`*=@~7Yf?m5H*3SD}-qN$nGe6fI4+zyB$(q?gt^0B`{W+tRkliuzJR{#Z6XndKW`LxIxOpcEgZyaK2 zn&oRHNm;ozY(8?O#H!7Gw73wA+hh6Fa}i{=$tha}k1xTZ5eSkyyeYcQhHOKB5)+-; zbf}Unq<7`hhCdjrhHe%VABJidodJI;%i%zOjsl$Zy#97s_vQghNNiqGL1Zhw+ueol zJ2>}zWDB}W&@aKr<775S3w3D?66oDAgy)4K-Cdp;MtbAaP}mZy*|63vKDIEkn`blg z-264A-gSxTZqPg`j)&_TqT39NAptJ;@UMyHRCd0ZWhnWx%~t!)Z#X_WqvocbQ!y8O zU3{17GP9T?$MYx{76Oktld`nlJ5VU+o#wPH{7$YFI||4gu!LFNQ}6o#NMY08kA+B70wiU0}W~RW$#w}@lE25=(^M$Mf%BJ zg@>aoS-9liNy3U!W+LFa%h!f7olzYcQU(y!4-fS8f%IF^32Q@cig43lCRdz0)*M>I z^s$+TDk;o0`ye{~)+JtKQijIcDI?Z(NELgwrt z=cg@`n=2Dp{*eBgvt5oDK%5bfE@j5#Zy`))w$GV;3cYOF4YuPI_v`0vK?MqAvouy* zH9(lag}6Jd;NYNR$?)!0zun?Z%vN{m`n`R=)QH8CSc;cBh@HOgh%B~(C5BK)3FV!2 zG~ZlBEINvfBDeZe@j`J@4_M-};TK@quH91Et_r;I9X?crr=WJ4JzL6fPvc%$+!0qO zoLTIgs5Xl|8OU4uEn1qz7?6D{JUa7b8{I`OPr4bU?Jz`zE|0=&mz9QG=9#0Fh`Zdc z{-T~ISqfkLWp2Ecs`iwrc!LY}nS%g)@S!4AMkrCpFpTR>hpq}0gH5A*S6 zM$4?1a~)uLQf@1jhl%gT#3i>OytJYt5t|453sKP)0`;VlE7Ew3$sE(wEK7{iVYdx_ zH1dCV{w3=X0cTGG`eE8XdR=&uH12@raQIv__3p0;MCJ4@OyPGD@oHu7>Od-|9R9Ts zvj-!T+CxB=`F4RQLTc|MjfVi|1OuRi#t&vVwvz?i*c+wp_X6CFE!REU9h@&m5A=Q3QHkgTx>4r>Wx z6uc}f(>c`fQc$-U+-9vI<(=p(Hp5#G7CMMYc8-+f;$5T9R}4Iz zmR3K{#cV!Ydh3uc59LnTsB6~s^F!@HKkmxZ9WiamFw(xs#C7jNURI!{w*)y&6-si* zKXvf*k$B*tfT((UcGYbrdqlNJZgIzUhP|bCopZHIx0Bz4U4NZi!av(DOjByfjO#$M z*qs0VDjeoaGg~nvDV|Z$-enJ+XaHm#NuRmBCkLiJEzQMLAs(Ir_Fb*-T}uqMB7_G; zcuYgcX{NvX;xSse>;4cii+#jN?yNthgT_%39J=p1JjgisT|V1bwA#Rfrn!$l%`G@6 zvddMV;+Y&AeIFD(R#05Iknh%;*W=}1W42hj->-Hl4erRQfBY` zr{1##IhOFga0vOQmk7<&KDOhdOeQt+ZaNiO+rF0NL5ho<0aov#v)&I@>JzARkGwzY zrRr0?Dr*(Oi0Zy8t=_=n(lm7@bLB{4Cw6Q(9AeJTx@~mEL^LC&Z@+SnB=MCknBN+y zKCNy$BIk%NHG4G9zpU0Kc5It+w>Q6%uiCed8;iwhNcFil08d{5#B%<-#iN zX%VY7;F;GZl3b!!TgGf!NK6*}i1<`M7@>DuVCT_rtiZShb9&LLxtwZghxHaakzIkN z5Hi>i|0$10p~tQ-L;G=U8<)C|Snca2nYgAlo;fSuouS#o#{sRU|` z*ed#d82f%j>bk+Dh=M4&xfp5U$z4Tr{f;7F9!|n~5y-gORc-7Z?psG?>&m3#N|4Q< zhR`d>h2sC$m$J8Q>RLC*QIIfqSEqhhi(WgcLL*}jrk2F!53~lf7al=OkO7R#S17_x z_t)*|1$xf6U`YMzA@Oe84)OE-Xf08OD2G@48w+mtWs^2|dA^hsTP-n9uW;a6F{HVo zOF7wKdz7N&Bbv&O$B5!}!C=p*^ww-HB*OazG*8fbS8UEiin*G1|4y^3-EdOi1gA=&Y z;?^<`MVJ#2zMw0t_B{SPPQoBvAEF=}9P&Iq{uarW_B;U&A)G{DQjyq0e$PwKOZU%@ z#!jnA^-fmzOXJJhN7f1ZNu-1T8?n_9BH4vt&emTg^gMT6pq3T^bfWC~(KM@CfTsC89cVaOnBUT?AbSsD% zUlxH(4WeUTtv@g&KId5OI2+w5u!f4<39pZ|CrlgG7!-(`-Ks!8V%mdX2+Sn*KA77o zcHWx;RNgs|{}(XT7jPZ$HwP9_9O&zfgRk)qC0Ov641|CGvkN?a8UWPYs>1?*^#O z4HIM^wAqjAQ{Rm}P>TSlEfHpD^nw@kr-{yw&yO7y7LdkIolh6lTglr-fbM%4I^i17 z0SeoO+Xeu$`+EQKG=Aixb@WWD9^JX!d6_7$uB@#s9ep?5_Z=-Ny4(WRx#R=lWW~b( zf4zCU)d?A)kl8@!e7a?!HYT5#p(PO1}fW6(brAPY!EDHch z`6h10MaIH>-U5H;mVa{|{fyq~Du1&k{J07_(ELL@3`+GL{K#S&0h*s)VQ%>}FkS@1<5B}7Ut;mjsR7QRX_fqYdX>CM zZwD*iB+Q!RumvVJuwPFtTt@cxU;lI*x_X(RFN1IXslEdECxH5XU)=**tb6=Y-)90C z(2dNWc^p!g@NxFg_MD69mG*tC;B*Bao9|Q_UKcdxhXk4Pm=`+s{GKZS&+X&P1~R1D z2Ymv##E$wmEhawN;}>xcOl+nBT*~yyDz23Tve&W0Kl+TNadE#bLPUs`P&N!BUzRRS zk41o2V<&7&9GhL^VchY7R4IhuN zdeBxvTRg8&Bb_;9r_hd!p_Y8U*mH`eF=cnMO|jpMy4rZ6>al%M*60-@Pw$fE5z1X3 z`IXJ`XAQMA3kF$`T4LQL3c4*wf%eN!*!E^914U^W+k^55w!MdUIv&Bw7MbpYe(}B_ zU!sK5xP}h%$X@oPFmjIQ{5hBskX1Z;@o-2NNA1^;K8t~T2-Re6 z0zFwnq0Gbg<M!YBW19K$oqF5(~eh!V8S!vRv^qNlG7R3 zK3(JfVZa^d(3mj5kEH-B7j2lXvZeg*{ldlm0p^@(jE*A~9_^P5bm1og8;T&sZMd8P z2owrxt?dh#CwGX;i5WVN#mOeOiS&i68X}D?%;OX@KB{P2zvK&U6AF^H%CS~jJy`b! z?@&Xl#Hbk3vNKLOmR3PKZDqfHi_CPolI@nery|#pN^XWAoH=|XV!3c;q9V~@YOTD8 zr{&9j@^*Jz2USA*!B&f36F>b8pGQdzl5~waHXRpEl>^H6eM(}^mnZ2k!$32pi?!qL z!CQAhOUaIYvrXm(g*|YG?C(=OGG{;xvM6WECFKyYDu-hql-GNLiGRvWKT8^Xr*k1y zkjhI*D4WsEv(-QFtZHcRu6YOj2bDY3KbOFU+!5}BeF!Bh%m*vhQ>}&crs8iU#|vKF zuDX?Q&En8$XpamN)kHod^q$T+ z9HgzLK4dzAMLe*zFX$@8FR&gR>Q60wC?;j}xrVC$Te)<4E6AGy$e4Y+zs{E6=i7Qw zd3Gos{HL!;8U=dVbnU#gqY}nbEcj0!{mU2QDhc%wrh4s9w=-sr8=9I$y%jU(o9F}q z!x>U-geI!_MFmh+>K3<-@?7SDd24_d@|PJjZ|KtuzB{?Z_?3K2TJ@1@NT+*iMZn-} z9>t9wkGZzP@#a{OpVv=i;_RIIH>zqb;{b?`O}QMsRZ&mDc)a2$!~4O?MZwY0bzxVD zV8S!EA$vF^tHZ!MchGMLiU3x8c8ks>4p9lAYwCGbo1XDOK7lS(DZ9 zrcOQ-;}fOY{NRN$HVV^A_-g~C}Nqgg*CPx@w`<>uMOH6F~u5)n!C$q z0Tyj<=0xSQMD z`G%>SkRK9kCKLRY3L34DbyEVEoC_-t?`dD}7 zyv%e@2!q7euK7Wz?^yV&RmZ%W;UJ++mHBfQ6h#V~>z^MiT=(x#PAnFael4$xnJ>Y_ zv<+t)()s8zBg(mfM9ufw!P6;V$S8#qwv)Dgn9MVPNby)G=-RolJy!619`V^7-9W#Hg>T9=-3>F;*fY6gGVYK( z@Nn_7WZjorPRC=hr&o}A(bS`OG#KW{JtB+(lbI|18SRL@ZV9fj#QNXe4QJIxvY2wV z$Q&M;kldLc%OUfeGb?kjv0CYf)&px}ELSl_cZbO_}OZ>+d(!sXeJ{ zF1k*c(+CX=V=tDzv9AD)Nnuo>7Gw;PIeZtUl;Sg(Ur@rFLdeYw=*u1lV>?C-zVlRm zJMYtD@}dyUMAUj=ytvcMyCv_ZgMVwDgU;Var+elL;x?BXW5+DfIdltrj+-BRu(|fY zPv2aIl;~Gvup0~18wz1(65(f58-v!W-|2=e3`e;v@K$cTORc%s$wT_r(`#>~rnzFA zvU<8%x{@4>G+*EPSS8hu!r+1|Qazv4064w|00>}Ef>B{E=a1AKTibJIWch;R_GrHD zxl}(IQm~5qOyyq98;eNyY36pb(gevzH*7eZl}y(W(^7I#v{lCZF6e?Nq#nuT}2A>96 zq;Y;Zx)ZJiWJhul8-8mcde%z6RPHzR-!H?~$uQ;<(@h-qi$$SMPL{zn4_tQ~DzZzz z1=XT|B>y;YwafDc2fB>iQcSnD!d@~Xa%5lHV|gTGjCM;70m5rqnogoVN6lpkEx>8_ z3F{71H%LH_!REKIl=)2s@7(a;PhqYHiLf&Bvv7^jMMN<>4KTuDMlwt4$-Y&Yw;?Zq>`?4>Vr`-} z)|(y~lRsxh6a2>dr%2wIbQ=_5{-Z63H-gS{8Qa^R&k_VHv7)*GGOfZVhYEouQqwYc_+I2ac{)g``X%RyMG%tD zp7WqF0L^pF-tEI*nkm17L-cKpA4v_RKzRRw$eqbI=^FWU94ob!?YwfaoqnB%DI-GX z)3x!7$E;8%u`=^J$w2-O`4W>MJ>Fuc(%4XxoMyT};kQfh_GKcNc`lJ3bIAjXJCkVkIXBV&HbUn#%g zS|Z^*p65HExuVl+8GA?%+AVUt&xu$vC&*)BOOn!^cLV%f4y^aLa@CkaS|aAuZR7uf z?t(OlV0pHayk1j6C2%lUEZ-tNd5l+>aaFtbQ_RYnonvySgrk;UN;%G!K(&D=6G5{^ zZzzNR6jpDpFj-TMl)xd}R1>HNy_|qOa2>KHExkKDua#_xz)xfDsp&&KR+|Q9=0q~6 zU44cTxaUqlB)+Kb>k>Hs_E>7V9IxXngO3pLwt#|u-%n)77qjQho6d_)Z>6W>`wT;A zbk`UnqTTzlz9Ab^lw~hALU?~|yb>!Ol$?N04vs>2BVM6XQXotUU=f~oT;F=0pJ@D* z2(KR?cZo5wG+Abu;r1E;S}j;>7I{HjomS~8oU4L49VZlL{j0|`Oj~Tx$(p^lkW5ud zVFymv5X`FkrCQ{F7(1sXL81nUwlQtnwr$(CZQGuGYYPdP=FW`JjpC?wG<)?FmC+TAXVcs}B%2RB&`!F3qIqvVmh=Pd=PXw#dHh zi2wxMrN5fb^(i%1!>YKFs*(rW6uUE%cW2(#o?g?LGjd$OKA>ok&#gYhoKL0-x`*L} zoA{{;F zIt+DF%-K{sZ#s|es0oi}Lz|r~x6PA-mT>E`bi&(iyej}e8UXdBIUQ?^nR65`VdrY8 zeb-KDJFY9>(xa%yVxz~psuIagx;Ml-D6ktix>dFSb>=yeO^T!5TXVy^)UxDe4YS3) z78A2@RUlra#k+t*g4~?86I#&{gB^Z+Ng+ys z>1>gp5L?$j+OCVu^synU-;Bnvi9982)3Kf-`WsUO@Hxm}iHkm#fm$TgzQqx+4vcA+HsY4Xj-una$j=5e7KG_ z(Kb!*OO{AQSA>TokQkc58$eGeNMw2?6X+%cg&HO+91ty_V%1d~iV(}V`{R2M06$KK zY^z7CHC&B(Lvxj!ihy->#YFeS)aJB9+c=h|+xKMBZ6WF9IYe0TLRgwlBE^SS0V~JI z%-WcQkcG~t-7}R7Z-1s;((})btK}{<&OIh0_Lv>6ZM{U3W3EIrx3a?lIfWm&$avfR z@e>5>IEpHB&cj=KX{MRz(es`<5EQE5Hgp-!ooaDV;>Q&o+kIux)0x^hkOr>38&#^V zxs}`m_X&Yoa{*D(J)2om_7M+RnTL?p(HbsxAcom`;tJFD0mL@GmARtAz!n;pj& zqfG&U*$po|EJ9xuubC!3?FTVXg4vGxKlXeCq{N9!ez1+Bk z00kC|T{B;VmeW;^ngV>M7i3M%S!^(}Qv-RTcbe@z6WO*HU|tl}@}b&(jnRHh;{*pUFI}5vGY7+^Ds^B%8bZoSu{h*zrgLrV+u0Rfn!< zDf?Scn-p$#B@bLaM&;E+bIhwjw%xWRrPcg zcr%d4ji5(2BL&t5OvZ_*R%PD}sto6~eLT}5t?4Ht*S<;cCm-Xe_%&hYQINCwib{dV zCUhoz&+3?bF)f_XVXV@8l4N!`1UokD^@0*skr{kR;- zrLGz~^EuwUxE8m(Bc$`m(<(6ME^*QMW46T?G6)YlEc4`+_j=IL{*ZpmZ8MXTpRC{n zr+eh^GsfB6n4LyO)fmi2VSF42a_3A9Vn{~e>7#x3QdkKMYQ+1Qv`Ik(k>H9#jF18?cgmhR@(}EK7@NtVpX5L-f^{LT8OK{_9Tbr5SfSrQRKIo?DOkc6$@<})y{B$b~8tE?RmL_AG*S` zZqUC6+cP;OGapfGeRKD(_0ln!3WT%f>vAn8p@624X<&Hfgv{|DUf3kqp6@oi7j+Ga z2Z=nc3@n_C6*WXbCW=r!YDhUy)I?;#TG(MzamC@W_&_}a9k?K4!vwg~ip$ahV(wG< z1?s^yQS}>2zgIB234FRw%}51!&Cxy4Urtx39?#QeUIiVma4TVlb23316)Kb zFTj~6DbNmGQ%};a})AZ_xv z)FmSA;1`EyZ}Jm0xArIGRXLA2JDYHRb8fjAKB+*_QY6?Bm*i@j%<2YMLI$+^k1!&7|MC2msYR~hr$4^gSEhPf%6 zmLL=E9&3br8*^S&7?k8t7VpLtT-7iM}n^t5nAr>jua`vEqp-gZ+}#af>dIWY7Qy28-##UEblbvifv4L4 zwq{QHAj{Xc)FC0Yt5s9TB|U+}l+q00xmS=`S)r-BAi*Y-8spWD$tA|t&nud$$TG>| zoAzH0O~0yK{&aqRF2i2<16AoZ>7x62VC>0-EBvIgJ7HKZwk=~ul>+6iYay#lY&2+| zRKlyfbJs`=&7Nm}NAsR|xZ9;}=QTP~6i)fAZY|}h$P`*1k~i8Uqv#k*<03aidnt$= zX?|Ult*A^8lj9CuPeQ9j#FI8f=5y?V^~$p5>m1qK*lW*{&;Hp^vCNRrX)r4Cww(EG z`B|dfOqq#}SmFttEM_u{$Ay$Te`?v6r8ssN0xP$i0ZoRz_8p~+GRrW3O`noW?(*+_ zdAF&(6rg`rq^K|E*BqU}R+fe@E4IV zE4OL`gC&-|_(r$p5c1A}cy4}OUr7JJ6y7+zzFE4l$yqx8(E&pUkpXoD1U$(0Q4nTpx3s?yWmYBc>m5xKrqV;030@mFFK>? zUg$PJzgIATBhVkNt=*koV1a$VtbSM58d_WZHag@R2z&nld;lC18Qmhd1T28i&0kiK zL2m8auYO&9a%gKWvpX^;dVURAIDj6iyW0ouD|mUfBJf&pga*%KkAZtR`aR6H3M949(a!*|72Y0F7Y~CPC;QBLuqdfTjzptme zWRqN;>zxAzJ-=zc>2$dz4H+#7u&a6~zglE;w->;7rw9k&cMuSd!2Z1wq@HdNU$gWb z`e(ZqUvPRXZ9!E0U$xe^8&9=S-9MxNpSzR&fN!(v8#QiKApmQ?1nr>zRugkU`%Hjlc81dX{xi$S0qz{Wump@orFg$DXZVcfYJG0bk8c zn($s_rIUVE%W-?1B5^@n13%lMWnqj<;D)tOA{!jv!zsP?g}Yi{K?4LGHMn!Tvw!?g zZJB?IJGE#!OWowcMsY}f%kbSYA9K|x`>(aH{c6+v>mUH4`2BtEZ$iDvAc4OE+&azS z?%#zB0r2RyP&`h7+NkK@_xAqweex(E;QPZJ+<2_Ma=pPhiXrp1cCrD&2C?J+3IzBS z`b|IX){WtKvx%sE`r-F;L7bf5gTKXUUvne40mk{QIjfEJ9`V@$gB$?Y4`!x2Hz!8f z*F>1}R3;QYpJ?FuGLAIZKBNSV`hm@#cnCQE!ibrOt`Qyx-^lo#2ED70ycn%(gj)Sx zmvns2PfK{xqynm~)IN!=YTcEHk@Nactff?5=MVY}8IgA)*k9x7WB*ewL4Fr?gz?9{Nr_1*Q(@i!HxW9f;bYQ{q*6)t z9Sv>9FXP1`fzo)4^hh@S)T6JHBu*D{IW~yRS;Y%x#v7pKxC>CYZr+zrN2-F>0>4p| z{ya{tMSV1$uo5&^@T`TGeT=Rt2i=XGy_7oxqI8O!7VK zJKr$dXKe zBr}TqWy`rZ#;ko8aqWrN3jg_VpS z&&=t;dspYku_z5)$m28ms_TJ!Xv>Ar=g}Zh=dj#WtE#M|)N%D#ZzpE%ZK~Jk*lVvk zD{}Oh&x-&bap$%WnTiN4Y~h)+O99mC%e|bC2c2kYD2O4_AQ@Vv(6xx#&K5a~2x`j! zMm28qt>Qc&Q_AAs5S=9zf(GQ-7%V{$%MIt5wRs3eTBa?I+-g)XC;AaC;f&DzH5;C&K>@Ag41J=@N(u3wVz~;cvZP7$ej2U0LP6raY6=}EVzz_NiPQ;LrjbpmM--<<;w+?0Z;;Z7HFGEEaAT|pI*m~mt@efIb zY>5qbmms%#U<^}$$*uV{o{EE+gA{LXg{%U%#f;|xUnGF+<*kY|yE-oym@}nJJG`Dw8=N$~c8H+PT^lj)Qj)C~DW z;+9HA_=1B-Z#s93)8c?IP#K~MrGKewF>s)`?~4m^yC541&Fa;iccJRdZFk@o5FQMf zDVXOBo;) z`8bBXvn}3Giu$=~1)d9Y3xhHhP51}#unYA`YZI13#pF)8MrDyJ`%#xk@&@@#vlIym zN+zeEJHLDtR>l&A+l_Z*n|WX{E@HVZi~I9e(6bCt^#6$iwa(FeDtN`SC$7&ii&RMG zWR72_nHX5eJhg+FszVyX>q-;roS#&`4(;y?q}_T(tp0GpR?OcF6hor1vfr7~?62ciR!k}0b5WuG z3OpCZ(l|u_vt6iMPe^|Vgn{rsp@x^Ao3QN*#(-ASS(;rNEo{8+FVJl2Ga{(S zs>t6k+D#7dJ)%ZF4v^Y>`nuUJ7$W8lI;fFWMT@9O;Hw4hsK*TQ>n7R>{Sm3BlK!hW zmN2QuJsm8e`sQalYUa1aZSxmYZA8|1AVSD15|!YjmhU>_t*&r0F2QGd!x(Q+^@dcP zs80oIH^bSHdiZGnV7gGvCb-8+Q>&GbpqOJ>)uvS67!^3C3f7)0l@3Yf9c}RPHL&L? zR!QwTyt%1FRM~T?25pK#XTmn~yJv_B+~_c&PyV!6I{RV)CC+Pc2hxeB;A z96$P^fOgGlDR#p=e5-@fEBq=^gCZ^msd?oytB6q`vT}8WhFEgJ50K>_y{Cpuyxzo` znj?FY6PZa=SX?oGurje3q@wh zd^-Ir`q95JQyQDK$FpeYSgrC%<4CzQDYdixf}Z>n75(`90v zJ#b&>7WRdGYdP(2Qcvwr#ie-{rXC7gyzlMtLi%E9ri4PgEU0|L7Znju%v2YHcj`#T zWw;h-2Mq{&&gL)LASXDV!4)yvE;l=bKso7-j>^Eg3K9Z)eL{}wZC7#wR!T3{Ohf0T zO~qCQH8vmLgjfLBmX0jE<+abI)a0}`sWW6jx$Adq`8bvbBCG|Rx%GB{AF*>wl0k{( zXBx4SCH6>PLsqrPcl9;ndUL^Bp~l*wq{TcL$Ru#POO`DaT1JOcnO7 zg=By<75W=B;Z-ZpGj=5j$vl#5cy^Pj6#Y87etCkkbz=Zl_&GbEn$po8k9eMrHgvKL zI@ps^woYMRE^Dz&`kidbwWsd;uMYh#5nk*YVl#$7%B~(w+`Oqw5cb>;A1*w)W9x4n zzOM%7rAa48eFc;$ZnwH%IU$(N3v`Qfq+gLA36F+z84?Gv1pXZV>5H@_p3r@{x2Q+o z7*o1FyJwH4e9B@N|9ZhYl&F(GhH|z3I(NSwTqATIFFqFh%xsvM%sq=ZhO0s4&@w9K zMLF)#xF(3k7cuZ`LW_`Et|W%1BwrWC^qY0kw7k4EC0LchRHb!Zwz<^B@Il9&it=0^ zVWuQs^Q@raCFwDG*%IQ$%$_!F&dRX|XRval&HwJOFP$~IYI}774x(-mViyQS$sQLZ zuJn&%lYpFp^9%CO?lX>U8u21{Sd58jySDk^Oq5k%J@1Q?Kr3a8I0n*-kEWy4t;J+3 z`E3r_$&}SJkzL|miaS_+q@l9ui|BUgNs)0ENCJ#BsCI1(H5Vj-P4&Gvis2}r|E_>@ z-V*xw_eBk9U7h1AERXKS3G;WQn|%OFRg_I}Vw0mQVi#`_!Y&tIHQKDQJXMr!F%gGU zHB&>d;iz_-)T}8J%_-cq(YI~F0W;JLQCZ0kygqt1fNK}a_1ln11SFQOVKAfLnwfbaw{ z`9COulY6s6OO<6mv#Mo`512Hr?6(D3l@Do*VGJ#QayZwFjp8tx{sOPhv+YL7ZMa<&WN@{DtVk-C` zukGiO=Bac5BZ*~j^Nz}n*4A1!(Pm_~+($uiR$C5h+grkHJHBJN8EGoRUFP)VwvC8t zq8NcsS+fMsBB`nM@~~UxP>^_Y;DhScUJv9B#$2IZ!=n9zqVNOoWg*U+w>cT?K6rMe zN>LUoYAh)MHwUc<*yIORNV3aO7LkUbk38ydDaiOK>yru*UnI*>VefS$f5)!d6q9m~ zav&oxU}{=w+tEIX0n>=aqNu zt)>tb;dz-Ff_gw~9jC?JzWR#sV<}S^qh!IH*=QiX7iT+H92BEO3_wu~KC+;lt1o^o zGi))nK@9L}v{(}*t2*m-6iOA=n>oD_zW7`W<;EfyhCZ+yAY8l{7CvV}Oy2Z-BYSim zr5CAIi!+ z65nWtc$0O8lg}$7lCX8(`Hpn&mZ?*C^0u_Bp6~WDw`7T8;&+30(#)Wg(X(#{FeRkI z%f8CQk5Ey0w{It^uef^()YE)j(PcO<2hNIC_HQ9N^s3q2Q3+d-RcJhrUQGqNrwNh@ z$mT4+s8MbwJQ42El09N1$}CD01AZ5o)rUtXl~KC&BF75pEoOTS9SAs$3hg=_>m|3e zN=VOa)K38*R-o$r?L&^tas>9VU5^=2dVtV<5jBIwsS>csc39mCQ|;Q$2_D?VYwWdq zzwEGT<#*TH@j5K`$v=F~8x~rp6eTjIgSH!Y82D8lSN72yPr7;Ks0{V9J8e_@_a##;@33|4s_o6IXV_eHhj&g7u$s z#@Xs(TU?Zgv}3((-+hhv^9oZ`YDiJO?vl;)6;AC?|5nRACcg6gcuF+$!?&VKOlz;Jl^3$i;GgDuIlp(R+7yHlj!ef83PZvqP z*>azwd(?ArfpZ=f>uMx5BjwxwFce}<< zDFHdKyF&pUYIm>&+097XLl4qO0VMlc>#zWCM_$uP+UvySDu`=Zy_DOtVi>ds#xi0?D)}jfh=?txUiK_VW>$`dnyR?u^1JpZC->FHF`2fXoiV z6xqz9uI-T6Z(QYMV+D&v>vGFvIc}Gs2SBL-qA!Y0t_X<$(dBa;R*{Ok8>q( z3soGVn+*8XjHp&oKDZzVStukVG2vo2c3u?hl8OO6Qv+tS)X4}&pI5c0h13HieqF2Q zC){kWDzp%7UKBMZ9!cjhx^eVj?e=d+pK%dRCF6>wN7{D*yxUllk(YQ{QrTw?V1o>LA%&XrmwN39DG?P4qjyi84daZ`yLY%;6U8m371W%N1pKwMrgZZVbo zbVn-YFIzVyQk-k3&G8wFRgXHx;_mvtC&X;4q#@uQ=@T;j2`G9-5nej0-9JdqrEjedU| zxth^r1`G502T~;mE#O~s;N+UlVF(qTF_Bvr&)TtFCX-7@SB=R}9$RmL0(Wu!`tm@> z)}ME$*iuA_-^)I{=KQw!S}6mvX*$06{J_H7BR`)S23QG@aiWVC3Tl!;3t{w zL=y>cZOs~y>*Dz&^{C4mo;a4^6^BA%M9|lap@@7YrOB`4Y|VD}w^(Ve)OTu9$qr5I zH0jcYVJ(mx%oJ3t>?Bk7va6Ve_Uah=(GPQC1Kq&2?e|7gO<2ieV#sr`lydh1Z?n(! z_|LxChh(3D)p|5{J90#RJ6rxKAvTEQ$w5_%KAbsWMwlj`L-qeKy04;QWn+j~GQ98# zgRT1F(Q8?J@jVG=Y%M)QXQ?^lh^NYja7E~qt@;nSq@(Ld%o)$0PnEk)G8}_|PWC1E z59)69?*2mOCQ;}PYsm@*BOz>J!#zY6S7wf{4iO|?r=uY<$&W`2vaj1-_fSllJlc>R z8OBKW%=9Y4NUG@*z)zwD46O2KmjI2xe6jvxb$ws9(-EA&3u}TA@Zd|qQkR{w9R1zF z<37ON0XJkzwWQNl*H|TqHA_s?Csq_^lXy7tYu8hXA==qU;QD@|zV=`-zLKe%!dpWL zYE7}KTk>!fUM`MS^dNbMxJx3w9!6i+$CJ=qt^eXu1ecS?-J&Q?X%gIRsLbrd`a@0w zjf#=)@@+;sJA7(f7t|ev$RF(F;U$*!Y=z4`&H6iF1(dPPD-;^V7jWOJK0LqXts_bk9OLSNKe1Ms1s)p`s} zLFO!>d`#{oB`DGmu*(hyArwb)l$JZ&zc=qt6C3fhL6TlTi$oP21vp8`PkaJaq~F{u z6FB9llYcPLEP4t~Bh4ehWHSgv-*0sBZlhC~S}6U-Ao0uyMs;MoPK^rPbZ0G^Al`?6GTIH`=Z}OYK5km$(a6_M5S4l~ z9th$Nn;a0VU&jWv^SIIRA&`09>nUc@;^e6EOM4#Hf#yJclr-@}$0xp`WXX-44VJ#G z1bF+PixOwGqe!x@^hlya4A+9Vbg)$7^ND8EwBH&)lNSA?IqP)j-LX1Ua{ET>vvAIA zS@+$ozZ7mX`-_qD3&>Pf?lGik%{xj84bw80F8JnWO31NFeonp%vET96RH)M@y9sN8&&T5rmmkO^#kHy`zd-!pz2SViVWwsi5Bpbq~!G+l`8L}o$cy_7$ z6s3)>W4WofJe#o0qIP9T? zcv9#a&iq`A^j?6XeX+xi>)Uml2kGK-tbh1QzB4Wc-$B5LNw6kk$CpLb{}`VfuCC88 zOWy%uDUu_J7cU&(A6AqYs1$abNo;Nw11-k9o6|L@MOVo6lHt^p=I{ znl>Ey5Dxyk5_ZVxK|IOo(R1kqTZbx@cpEZnTNaoh&)s^^)|6)ZR-q!(WM9qaX?HNC z(@fFsq>TH@!9uTEIt6grt<-M)P405VB0>Ep0UWF{E*uI)+SV-zn2yEl9%xl0S0a+w z0w${BRsyMzr^diXj?cyY`;{rwDXn|P1MO+YFD8-4`}$qsG^Fil8icj2R*;kPluZ2v zD06a^jjpD+(S%1jvGuoK`bFK~8zOw!><<~)*+>=7`dlhccdM#`23#RqSZd_WaFgEj z?RL1ioepIZxOL%~(I*4ldF*>agRaLU-;ZuUcWXar2TR4^1}Ua8lv&XC9cUORb9OQr zV7kWj3`To-Agzq>ak;%uij#Y%H6I7&L{GK62A|%adv|Mcqt=s#nQhs}w9Iw)*JNc2 zC7Bzur$0MgAmE9*YPYYNtRr@P_}iRJVuZ#yyxvTKQnd-4+pFk0^VwB^+>`qyM z&-vvB?-`cwMQjmp1OXXwOV#ygc4Jljw;jmVQ*Ei&cIqKF>$kbV!oJ#>ySJOnbUy?T zuqClLRqd}Mo*fnl5e?aYC#2j5i6z!_gO%OC_8N^j`NX(TsQ-Ey9hDw^2qO=sXR~zq zo)igDM6J{Xvc`3?KSfU|nU<^%1&ul?@g=Hzs!l5E#$Zrf%5rLv{{qF5s|_34nst&j)om{xG6Zuh#2oPa2UU*5P7MFs zsz-X`1&Sodpu|NzXrL_7KA||C91cDEaYD8lmz)s3V*OM4_jY5X>Kdq@o=qAWCd&oY z_6S|XTmHB|_0Zkks}Jp|`|8n5&oh?)V_@M9GB-O?1<7H29q7Iw5H<#c>8H=bA}+Zn zLbg$AxSOf#$|IVA;qMR`vji!B*=LgojklL{r)n~#e2Tcmq>&PLaPfv(tfVU3VE5&= z@X9rlMk8GLPWZkoiISS;Ykcf{I&pE&^d;uT3PNE%B|afFOzW~xpebs|{5816sQ%dgo))pI1C<>0oCTti3Fy{7yO^pC0wu^?1~f)oeu}AQnc? zfwIKi##>;;@g6r{eH>)MT1*9uO3%;}Bm_RCZn7J#AN2~f^POlFF}7@)SdIt2!aPEB08urFci5uFb8vPR(%-5c^?e;1cVS%( zC1VP)v}|9=e7WXbIelfW$A12*d1fTXty{e;5Z~{8p7N_4>8mxz$(mjZG_^T0&lK$H zqoQ*8PF7L0^Zkx29*$~mPp&}(qX%pAbuu!{RDjDqQpTnhI$z9blPx0(Sy@4mX_>p? zOHUA%F=-E0gM6^C4}N%xxY-V-s3p`G`~f!KW|a1~ggECw^%C?V;1u2pl1O?;juBUV zWpXIf&1lLbRAR1#!?w8l6lZ#^Ei#s%q-9_Vg7R2dIYC_hbQv**W_UFQZo3P2xK5^6 zcFqlRZ18emY#t{HtexFo|3B<-$5@|Wl&DP3*xNWuCx5%a9R#7Y%U;rW6|A8 z!e0qaIpWUxbnT(t;-x*Gk!L8m^T9o>IUb07gJ;K88a4#Y@!7-3F9xm}%f8Wa58XJm zcKVXX?n0~9vXT!hBV5r5#oKsYC{533j-zr^evU;*a~dvwf0Gov{m&Nyk59~w*uNQ+ zG^7s{G)o@vIoQhZUXeB;hTJqV8!AS3Q_A^z?y;9uD)pLDomyQdaA-(*dXRP3bTaV1 zCfaY@Cp8Vv(~7vlJ=q&kJuh-JFVXk`?Z2qI+ODa^Y<_}N%9bt%jMq&-3m?PmHWpii zvdCxBTOY-1mt;j&ezyMU4)>4|SSzK(vFRz7CX`N^ZE^!6P+%6{PewJf%q|;_Dwn%s z$tGGQfQ|#>&sR~2z@l3jiEuVkZBJ5nm6S_%QfQBdCxf-`lw_05+CXEK?nTiF(W_M7 zbYUf{vX&>=#`yNr@u=QTT5lb`Tyba*&DeedJ?ygN#N$UiId-0?`d?2Z=RkPG^4&)` zK8sCOxL!V;%wT_92+2{EM`%5W?^c6oLv+>^p@oi06GW;A7&(8OLa zr0&%tTtE1s;;Jhs3rpHZ?m_TS)>fo~;k#gBS0BerJr3tH?4g2V$S;+r2ZWixhmRK# zi(qM@pXFMX#nn!(VC;+Oa>aPE?gQ+e6rl-Cdr zSHG;Rk}}I~4A_Y-kqdBaQ8*QFm!gy}z2^Q%rTcaVzI4wqS;r?-bDfUA-nyKJ<|OHj z0>y;)!4D{ocv+ zuwzWjYINoB%Iz!f(B9i1CsxHK1XKmEL{A?F?AND&q*q78$JY<&^Bnxm z0|2CA#|+{pK>1140fq@PyiX%Vj&x%ZBnbHU+2IdIh9|59h*ts#3P{ke`|`t9FbB;e zWd_*)!<+zcQ5a_r&Cd_~4{<8m=B0mM`eg=*90&}h z2KOV3zJz5S6G*VPw&zz2ulUqw=!3$a*Vo4nSq3Kx6ymCeiV6%yScl8c#10k%>)#3F zHxG&_(1a4uHH5OdMT`AsNtFpy@c!$^Ff1S+p+`Lm1Z@91lHg1Y^NYlaK!-)1 zm)D*c75GghmlOylY~PLt{i(BBS6mq%@zv})l)t0nyI}9)7OFv*@at>eGsDkgkD!k) zjSCwp2{5SN@j)Md3l0J<(8J%Cq!r>N@xBu;9(7uOUFApmRh#PtzZ1fkv9^D)O zJ6z;*aM1pD^X?W000^*yP+vU{Y6~X-ONbHDA1p+-+f=ee@`#BR}I$P zroA9;r~kL@*Oqo{U_nz)OzMekzgOnsq6{BjR$CF4oR$I@Ob|aIfuG}x@M>2>IxoDp z0^k>^GR#>B@c0)wvbXpb8T(G{HRJ~q+F-BO38V1tZwPS64?-J{IvDhzpTJMA$q(1j zZ^@0G#t$CQ@2zLkcXzj+R)}xa_^F5F*O@$n#mTk=0=y?_=T_z=I2-;LnF0zGf9 zK<6-SgFiunKZ3tapSRZT=$8SZzeq7=Km%X}KagEuAR&G8e*S(ALGOk4?{&S+VPrqK z-?wHUuyBEc07^*YYtNlIwKqLH6^cf1$a)ycm?-M?Gw`URGWnTba%B=qA$rEU>0)>l z#V;+3zM3nDobuTxyrVW<8qr33F6lhw(CL zke_|T8M4zR&YMarPYqF8DSaSKR99#Ez}tCR&08IivW}lzYTdPFawaUWm$^+f6QZN% zP7e)~6glCg_jg2gb7A*Xb52&)&15Q%B7hVeG8HY9u$F_}+5^1X)X4j z+5v(fFu+VEY6h4bSB)8bM*nVjT~!Skb_&6|VM75BSWQh?;|Z^g_pbgOara1yM)cf) z(6;+HYLyW0xpYyX_E?r6IdJ%$-9bl3vtv{|u5-8F;w05zC*!dB;$;(}+QcVQV0Z_v zjcHnLQ$DQ*cb(NMb@qQjnmR8hUl0aURjdy zbaqkYa%$E0vzU0!TF83|IS3toM>sSZ&TYSslX!uXdS{K)Eq9zM^wh9R0$rP385Fex zBn357rN-5RNDX1Szl4O=eWT%KoIk8YWh;=w)olIqfLUYtVX*G{Bjcf2%D=Di|uZO(ApejvnzS<;eaVsmVF~2*C!~WzUAOwn~ z&EbhCt`?&L{j9S3p>Yoz;nW9t>8L!#o$J+Y&yP}U900bkA9EPpD7pt(m7F6S2oBi` zVTnuYjYh{H`49c6nYV6Y|GnV=fj05IYp~?p)R$Uro!t|u%JE3T6gsoc9=I|B)lTXx z3q)xd0yLzT@%oj~TpceiuLi9LVp_gq4HfyeSZl6`g5<&gHiTFGa23J3*oo3EO_5b| zXY<$xwN75Qo*Qe`*n8^&T`#OvgPi>=^IP#j*LV*~&7-(Mxa$T4=J`A6^c4b?ph%!2 zn|BQz*~v>@SlCk2EBVDE->tnl9hcuTSJ6d+3T0WeE++@9&V|+7>p1G?W3pkR$vT52 ziylkv_Z=C_Y>lwI0bGoCwR9H)h+V`p-gAo`1@_xd)IH`PD>RPmM4=f7ZWJ~B_zPsF zL)NkW6Xv!|WoobHWzj4f@NVo+NBU>+cPCJo)+ZIlY+4lOc%L$BRf^|BTp%EiK8wwwAm#apIdJF=z88b>`8mHhLF@;Vy08ebtx7(-&69 z?vBG*rCBITkQPu1JyVB11w8H}TvD-(|*E(i$68+^%_ylr*FT|sN?f^O@OUTbd) z(IH+>x1&;kma-U;puul)3wbRAjlO%*^@?bdPAHsJ2d3(>H@WkpJ^8AcsB(G5Rz;#f z>JaCwvAe*9*BOODPVKA_y`7OlV{zq{=Ju%RN=~LKD+gCO;_zAt+Qx0>;M}20zn%5q z;~y*atrDxu&{5ET=OSZ>3!1iyldbM(!P$6j4s8sdsbIeU}wY; zqiG90etjhj8)teE-?O0yX(|&*s^xwnQLf3WymR#D9)#8d|5(-jLeryS&>m?zy3nvC zouB57NRP2{v3pT|FLI8F2b^ow%xZmaLsR*&qB5Dc*V5!{#;2x){KWlFd~LDyv_^kG zr`n9gIXR4@MW}lZT9i@{)hb(2O!ya2_##OYRW`fL(!xXYAnfg=dL=j);zH=K7FBGt zWayUovla<0MfFSexf_!>D-`m4Rwpy7%bjJF$xGjrBvFfu?^7TNgO_wQyvx+~fTd8jSUB@Dx zFQR5XKXHX0`L^H-r{FN@OtBuPqbncuCveWRQa+a}<+7IB@eOZN=#{RgEQMfkxp@Qh`J*#? zj7}cJyqzh#Mr&)3lWQK%yfUT{vMODho4W)msYM)N;wfSj^l+LE(PX{AAG0?=F6yJ zOhW>hK%a=S2s(HY1eNzCLFe1_N6h22N52-h#em=hy){T z51Sg7Vug|l`e)IZqq~^n|7z|lgW_t|ZjsRZ}ywd-ty0GqqPgYpw2n+8^oV3|M^F(a_2N zBjhZrv(jzvu@8@khbMcKpnZUulGE4Nu*&6(QlFOq!-x7UL~bNvLv7yiT`p!J{&J{? z{G7Y+!S-rZv~z!<6tF}N2kghwc&KVKG$xd+P)D}SyvExe1eW5Z`Do@UBTIGfPl2l_ z65&4Xxp$tXflnwz*8DvuB%Vd#Y=U3d;frXtrk=58h1)vkS4k}RT#0O-CZA6O$q_j> zPkF)CX&F(c{4B>TSnm`4l8J(^SY)d?Um*tN)aZY4upxJw#ZpJJ+Ag%)PahuEhFh{U!_l!bWw7;r>l4d?_CkYa#*pK_x`T(- zxlv(UIX3CtF4iENuAIJqP=6~UTS0;xrD-`wgxFw0#Pf?UpH!Wpho)fehPGOs9W^C` zo>;JojTlwZ<1}dCaz>NxrIl;t0%^>JLm-p{cTvFIG+31`OwLRH0s}UWKxt8JV0Et(f%J%x!ieMXtesfU%tOG^r*MibR8`CuBQ=f+FcuxUQI1s~*;hOh&X936K`GNU zavHsbp2ws$%$nq@D$Z=KPe2YuiIvs9X?%7D_+`gPYx~=(he>s9OzU>{$sak89v*Oj z=e#qRw4fH`B@IZW+qO6)u>|w1c~TyZ5Cprlw-ZN6kp>P%?icfoku9F|N}XtV0$}F$ zLuM9Fo27P30Z&ma8L%WX|B@cDp>N(cw3OEq&{_ zy3w#+iQtw8T#W1g)CsDtLPzZ-xBb~|=rLoNcuxB&m&1y}GiM;^bDIA`;lormG}IH! zOEbwz5EGNLvJUDm%Mz5>PSfgEPn0EeypQ?XNO*cGyJTsQtEsmvvr+KfO|<0ms9==SPVa-X=n_8 z?8yg_>(&aLznQD+)6q_6=7ozK+DvnqVuVLG-nm(yV2q%csA)Cw>Ro0nq@%8P{1`kt z)_c!?;5Bt9+)2tuVc?Y}d|Cyz*G&{C_##^NOh)^0xQr1E__^m%ia>Sg6uDdAZ!&E=yrHVf?JL1?^+I_WBWNTff z$J$sVAmViPTAy%~j3NK6EJ3VA{6woh?eZ--# zROt1{Og|2y?Z6L8cl&XtcJaghu#4jc@wL>;_IQD6-TPOJ*D<(al6H@-u{enXOg5== zm!J4%l8EozZ+qL?77tggmW;AH3A?m^`9{LjsFJ;Q7$xG*db^7{*jDE- zh9&SbohI+6Y-`4&@Cd%>tx;2Kra?$r!Vs~jw&yyj)RaNTZq8sXU1D9!mbJOt*>-J zh^UBlBD@N~HfmcZCcP~!E74F{NZyR zHIaR}=PWMozkf_{>K49%qJsGOpnUZ34jz43o9CKmlW54W#axwyM~35|M@FQ!>Wr{9C#^v<8?dZ+V1Zy5|_Dk!BiJ7Ty zPbA8TpEy38xOk+``We=N6zzoWMA{dWFjTRl3=?wF!y8Z;-u%p2AMtWYrb|zp2Axmo zBeCI1(BIe?*XtT=EDBR_Sg9|(hMsDdbVP5M(xQH~>ISY0L#$TB8Yn9q*J&lL6L`_9 zHXF)Uy1QJ#ZdfJH9i|gl@&>7~qkKDlHg>AJWfjCqe&H!Zz_X)M~``P zD1LK+D+5(?Y*SS}pLdObz3jJBaHeAK@Sfze@W-VStpo1sZ8DzFPz=-Ve%@K$FEvaD zINJ@+^+?C_1wu-V-1G;R&;mp#=IA0P^Sms?M!rQV?AWR=^T5k$+JBbZKu&n=%QKd3 zaE4wa+%Jhy!!e~ruw$h70vT+_LH^lQgH@^J7`3dv{4#^AWzOZ&j9{sYc3r^N6^o^* zV?ifTg@UW|arHxj&qj)8zvSy+VE)W7e|{@87ep~cW((e+(52vwC{vP>&2m1o+o8h* z-5nkcbL%i?;|dRtmsZ~cQ6`TLV;s@Hqbx)it^4F#%F@r$$y%!v1iD^(H~l)Nn-c9( zHC0;cH*6T^@$pu9l#` z@lkTl^{3&1W&?;0b4A1a?V3;noU~B(vYIiACCJffAs@8md6iJbS1TM?THdzIJmK4% zMU#_Tb-lU!BmbG?(+gZKi<&cT022U5QlmQOWe@3fUQnHXU+~xQ`PzDe8!kxjM$M<| zI*iH%8>*xlkM+HiySNm$c+iutAs$x+sNWakbmjp+XRNB-q%rB5wRn7jP?^Ou#hKXi zT?3|5dy7vbofQXLz1AE0&7KJGH-4&{UrNfaL5Tp#Nsp?TI#k4ObUk8mehw2{+*+(Z zcZ^@UUtTt<3u>hGOnT6MJ*vACJ+&C#3Q7Iq!~5=SUnct1$!fVmBuaSdc1Ek?ZB>7t z#O)g9zG5bsSkt$p7r`yP$I_un<+e}ljbnGv-Z%r(;mQI{Gn?6^@B6FCvJmG8 z!YFe3{ym#}bztA+jZ3rSgUttX*2)Oti@2m8RPMUA9~Zpkr!b}`$G$SRgi<;dd_=39 z%|q^`qC!HVi_`q#V$NHl-Mk{g+_&6CP5s5Nf=}d(fBk&+HUtS(?(QRe_hO^qJH`Ga zlmZGdc1#}Y)7bI~9F{h^mx#I1hE)E}mvOWJsxvYsVkTH7Q}6SbYx+q3FhkqX$^@W) zRqCy*^_-h@)u_AJ6R{veaw3-x2PaQyNNMdfPu+8%e#eBc!KdS822d|Mq9=cZn+tlI zrkrsCrseO5lT4~?;d1~qvZ`uiB{9+6K6aLbQ-S}@uU3eeBLq?nuk8dcJ{8PMm)9ffM9#=pm$2m@ryCo60yVz-^27=0U#o}< zOW3QVnix#GN;diusMuC2av_fI6E}8HE2>pD4aXL9M=lKwJ7*Ys1J6n(IH)ySW#-EE zdU||zhRYcU$tLj`N@>{jXbw{N)XmzFi_F2Sb@O_|9}&6%##_s{rP7PAn?ffah+$MK8pF6ey>rfwVKCBa-uNFe$*;dcx|B!OnM*4Cl*c z2kBGnr^hdT^^~vH<~8Qm4PM~W7I88b%S7Zx`mbIURfnYeR#`HCBN=UxbcyZwWU@bg zAQ-eC-|~YPC4KU!`Vf9WsR)C{CYlEL?IW;JIgV>rB%$RzwpMf;N#~3~v3B;lPd7cs zVl!Zu?&NJV@z0M&9(k%tqsIPKG>J{`M8&mNqQAVxjfrj|IL7w6uNyjl06m-XDWjO# zfwtMHUUS!^tVGh0t+h-C82r*wrm3yTeG9!INf?9R=-nn|4ocx9?v3C?);VO*0hhYCV<(Lmr;zkSjS;^Q@ND+beRSe}k1$eTXj7B>kl_#MlFDkJ zCi9WX^LtfUXuA;xaz?v}_!Z&t_dN0-l@+~b>tA0tY>yJ4tG`r!=hYN)Bavm9)-Y-9 zIq!u@ucFdJ_N-O@8gIv*}~ORb#J#iqs*U3;=5SC5i;$I_^TV& zk7z!;8`4T|ykyXt1wex&=!e9j2N|9mv*bEnh3%*8-qq~IVeGOC#MU=>uRC2Gh`bYq zOutxOU%{{F*@=8d+~;e27FH)HrWkVV@4EATr!=GF_L%HlpBwF&=gzRs$1D)Z0#}Ez zwF4PEj+SP$gBn2fat_X(HXZ;DIcEz`8yOobS8E$|AP^nw>0x8xg6^~AqND1%#Yxz{r{{x{wuIQ{ zt$^@Ss_u%~eZfViV%C(JYgHjJAGhe_o7o4munZTR{EMr5-+s!`)#i*JGCC(?Dn0UJ zraxLbJd3l6)|*Q=PT8GCMkaZoI9AMB@%4KQw^NNf{ny7p6T735Qz};9)znC38@ai8 zT9>t>eytjuwqTxb^gHf%^5;XRnpBUiS#F8zhtKAUW@9`?*3GiJL{Q$<7Em?=J*CiI zQ=)**5~am9xe5xCvvADvyy`5L-vag_0=0oo29(odeP=Qcg~$~+)*)1t9Z1HS;r!^x zFdlff$qRfjy;sF;vf5t%WaQxO2o}o4M&>5Q(Ph!i>Nv57{$8NO$__fX81{V{&t708%~Sq&sX?`L$2b%9SQ zDCt=t>gm~vLMK8Q(bvym)T0j&+#I|gF&RH8Of=f)ejH_v&zK!ELrs}*+@d{0yQ zwK9|;2L*47^@Zp{ib@Ebdx5$jj`4U%E=rbg!WDqW{-q}tdL%1;16m;zrzAvCaC9W4h%)pgbqb6>!G*0>}MobaUrUq_P$jRdnrhLgV~LC5XXBh zBmKgS9?$)?+ozmgQqtOq9M^F|7X{QqW!OPdzc2&wrw&dv-KV0jw0rjC6d`0UA6Lq< z#^YlGK-$OPLVO5WaR7+WMpp6zX}d_!?p4v{vS|4hzxv1ZUB%dO->FM*p6C8mksVN7 z_|kW(c05HbAn$r*Pmv=auK`xDw5M1KJ9Xtd)iR#K9+0PbdX;peL~V_!_cApxg4ne|X?Gc3fWU28YCGh5p}whWD?a z;T7Wj??IEI>RQN&`~K_~`WdyRUw-C?03Fom?~=n7%N5bO9rU-fvgpVpiTuBV#3&VJ zb+9Er*sYO#LrfW}zUzQ|etZ%1YjhE%!ONBQK6`zjKU+nH?FrrU4n1EYe(^Swir?6rskn5+FEZ3%KfzsQn2(X1n7%}`-bDjkTAsSWkdqr3^m~!e{blaaAJK7+NABUf|b6QrUuDW@s8!X<1 z_B|ObOEymLEh%de1uR%EXJudA|3q??3uU^APz^S$FArSj)srBL^Ec=x%yE$y`F&K2 zKqQ)TQ6;oOYMvw#`BQlgeQB4moBRf;J@qQCST@Nwe>S+KV9S%T$kL+N(t%2J%&y9E zwC`=@FJV%Sc$4`Z?_2|HidW#DQCmhoGq+fNhG)`@N12@DTq9kEZ3$f_3(A<{xh8_Q zh<#O$PnU4z2gCFMuF(WP!%n3jLmO0lLmR=-qW1%B$=$zuF43I|8D(6B<~%4jS$o{u zrNm!*k|sc9W!#UMEG={NXut0pjP9$p0l}8@cCyEg?rV&3yS?*nu2kP>PgesKtzF|* zshV>@Y;M#_Sa(pDL*&mZ<806O>b=uOn^43TD&&!W^Yrlj_37ak`qzF$|4o;x{in;t z?f7?>OGO)sPQ8ehB-;X!`{Q`0mI{WKyTO znc!w{FgtY(uY;9&H2V)c9W+o5DJt+7%7`utoGc~ZcRpRD60WW2s_z<&x%Y=?^CG%YqnvKz85 zUjLxTi+oPu1VBz1(VP77yWNR3`&n>1{_5Gc zPGzd?ymh#$Zg0A(0a^vEnM}TVX++e=T<3CgqT&Im6OUV&aepAkDc(*)#N6V8mlKr8 z{}?5f2qzJ9&P0z=1DB6BC1ZvA znV|@G$e+`8bKz%qbXLcSOsaQeQ6fQ7{#ZCL@% z{orvx6jAgMX%lIe==CX=Sy?rt*4mvJ><9lKsDRSzN(ivq& zdxi20?TPf=ymxI%<=Eq)^D9T)VorF!rnv8mbEg-=DIm@ zQNG1t24%^fvk(Y;+aLMFf%e<^xL|pTkt6Wz7LAR?kU$^T%gOcWcqTVdR@^Xrd$(^{ z84rr$e)8`7-{64%?{L8P-}j>bf8d}r1zHTm_7d6L=$!AJbPp1?#kMlzlw%5Z4z6mu zjgFQdYeAfIN%O+86_lU;jLlDxMse75SZ%pLIraJv@#wvY&$HQPM;uxnzcSBO_22)r zldQVFvgx}ouDB`98+gF7Jb!}H_WRRlt}{pJ^rRHh_M~GUv%eeC>hXA6f8BvP;u_-_ zgkglX1Ead)nAsnAqO^RrghR$AWL9gV@#Xch?)n(*9n(|qpejYi_{AGxQ=u+^DS>%7 z;){yt1w=HsB2L7>U}(S!a-jG% z<$FFdxRGJ-;v2X7Rct5gWAgcUKT+#R+~w3AD0iab6JIEjM)Pgc54BY^gX=;*L3We& z4#LUcw#~51zk$JDx3L2V@Cg0)F!(D2*9o`YlisTN-wYfZdUtd|>L1;cBuWFD!5Y+V&t;j@_}5a?-MhqxuI10=;V zt?5Q}6;Y2dmje5vFCtyAOjV|yH3wzR^9^3Xc)d-DM5sfDC8X>$cP13a7Uma^SG<|M z6{}v7AL|aXb49VBrCRs6DrS#M*Q!t5r>?1sSvLWUtX2gT7Zs@UiP%RR#wn-P$pm}U}&n&7`deKceb*9pSOS}f$|PGAcENtxg3)% zoeO=%(x}V2temqsqNYLcQ&^uQDMl>84vdF8gbmS3)38rS7epEwzpZVRLmCRu)`l)# zzd8{$HYbs{V)DggEyqR@rTr3W7ml$J<-O6xWih{3S8s%otEn)bqVk;!NNg9>-X(aq zjk&!=^*qSr>a>BjCmAk?0grP_OJJ^OyiYKOIL~9gCuy6oEjmbUh-3oG{T3U3i0-52 zL^0sI-LgGFL3sW;M*y$;8^El1JEe5g9eqo|{BrX)Eeu5VSX|^Z8cNV}PNAueq^2My z?ItBmgC#aehai{|C5R`eAsJLa%{=tVuRrI0BJrm-ia2#p0ub%M^KH4{!FONgO>kk+ zcszO_Xl8UzP8KM@eA}WCJsz(K6o{$&lrL`rkxl0hZBpD>OVwl^UfUhF>uv7yM&UG z+;=*4K-q?2?vB=K9VS}Msl0#N{vI9Hsh=ZlSa$$n_ioA;=~ZoLS@?;h9)R}qp$D=2y-D7RV%OHmfNxEy zD%k!Vt;Lm|fG_1E&deR|DCBAQ*6V8QR2&0Lu&g{!!4?IBOtpaxP8H+|>h!kG`8mFg zW&5PCUS&%^v81l@xMex5X`SzkUqR5UzKC%y)TQG0E@9;Y6@7(=c4v6?8~GzZQb8~G zdxW#iRNoPLQ+9ux0IB|hgnO}Gt8X_z3IP8t5p z7s#WIJO!X|Aaq2d{=&G_0YF5H2Vs&GJ_Tpxk7S+Tpfj;T*2hC$r@oo#LMyjJ%}E>^ z49T;PgUeBr$w3v=Pz#d$qgnS!z+3N-(=;gVW9~_=HHrw%G{IA7d*u=1@##PT;p>)3 z3|a?E0oCa* zeYs(pRRgGQdvIl~3sjeT5+&Kbvetw!q&$dbt?UgE0@a!Be3c@`wU`6*ruuHi&NDuI z^YqEyc-aB;+bKR<^^a0#1A3l+pM?!C_nRztGymG)0Ya_1_FGV&;fth8gZ_#c7(D*R z=y~JeYVBnO^K}^AtZlVz^x3%Bc{$m+`Iyi-q+!O6rz`9XtoYxD4z3^>nEeA_lo96U zn7{es&&q*l&XQRW=~&KPDH~|8~2yecfyT9O{;iTAnWG9BKe= zzCU)C2iOz9CHQ~WlK{AQ`TymH{bw_*Qh(WFnIAWBPj90(H$irVgC}qF`>PhUZQ7e+ z@52j3HXiQMx-w1A?dv0dq9R(Qx~;My#0-7&T0y(F@KM}xJU_zh!W3gsrnvix2vmqm ze9RGA_f!OTBP@IB0T)u@chpJ5iEoO+L%CHPF&!>@iaaL7Qs}!7w42F2MjOf$`SQg;ayrjA?;hur3I^6lNVDG1DJVHFIO``XG!g5Hc9mnm3s_Y; z8aC>XoVPoDSxfp;i!+vj;lU*7Bu)MJW1+OcvKA>^@fi~}2>)w2CuENLO0U`;3Awqj zPB`289XWhA9Gl*}FP~iMC%=1bs@z$FTpFL5q@qB8MOz!|i86}XWW@)!2ZQN4PAA7t ziw$^2)JyzfjHJK2l7h;HyA4e1omhh=><)aNwd=Eyx1>1J)!swsXV243#wANo9>n_q z;$ll})Ozz$KI<5x%M;U;p2oMLT475~({jv9x}USVzVsYt|6F=8pYkP1Z86WjI0^g8 z=s-tZj3#$e`NM|c-SzQK*nW$yw_RTvBq=XDFHjEs09EGcQVPz5r8|}~7Tt|QVO{2k z%-U)FF|Qg=&$&qNyL42UJp)q+y_8F9xPz(Lv`-E$F4J_YeLE;wDuoHZLNl;#yD({zMYmo}{s$tUu{%j*7$lJf&{cyI$Ffp4itcOEjLKZNEhb|q zlbAuQAn;-ow9U>hm)F9j(4nNQ`2I+l`-&9I2pps-4YEP<-IdaCcVAACC~u^L(;#(5 zQX)wJl$ZVNDDmm@=D8KCyID-qk8Z+4fPcXZqW@98@D3iH1xr$Z*RiUmc@39fOVRY9*wJ z(12csWK|Z>ft|`ZUY<`;iq-S-!9kF^mVF|VXA(P0I!^Ru(v3$@d$^DN8yVcA{a87+ zqd9+?R~nOHO%>hbBMq3^%*=kNqH-UbGH=7R3n}BI8#jVtjIi5kYJ2z ztO{y5%fnL76I9B{=(s{)u8hekG->&REz&}#Z2$zj!csFs2OQr%Ldki#d--)e7&`Ji zw3fC(Ol%{);uzm&+CJ$WXCsn~t!2SShjh1ibP#gNLuTFrQ4RVLyD6sanv$AcP)i~C zZorG58M%6(S7U7B`O6s9a|lhGabt>GOxHl5?j+l_mS2|b!O8fKM<_yl(f&#n$CMBh z3cY|Z| zyH{HG)y{oYrkdPhMsaw#3%<9zn@^DE`|wumc3RF`U_*BhU zkY?!i+sL!-Jx{GFUdT>N!S+;)Edd*KH>>s4iI@y&V~wta z>KrJ=x#vLdiv=g|^nYC*yqayG&-_}IeC;yF+|6)c@%s7melDFuCvA(7^R}Xq$?Q!Ez)EyZ~jJN>Y04`XWrmL$b04Ciy0h)iR+qwe&5q19@$N-E) zIc52zxdfzPkyA)oMoLDS6Bb4Jgd~Odxp)NxWchf2fd5Jke#PJfQ#o}iyQw7 z%kbq;2UeIlvFxTl-tTj1Lc-jz9yDk2u3?gW=asJh@82y}dNlD1sS5y;!|?kXoqZ`< zixsz*Oc)offuqFGv{p^h88&|2drZyLK z*Dq}44riu!4uuG>%vRHZm literal 0 HcmV?d00001 diff --git a/doc/src/week42/figures/structure.png b/doc/src/week42/figures/structure.png new file mode 100644 index 0000000000000000000000000000000000000000..bf82679e395250ecb502b1cb192a30bac48256ad GIT binary patch literal 28827 zcmZU41yGz(v*q9fCpZKr5Zv7f5FiAC2MF%&?(V_eJ-EBOySqcM!QJ-be{Xl|)hlGC zn3>|<+j6_lIUVv-<{L5sJ^}~?LY9;el?Q~{@UXz|#@F!x;DJO>LjDH`h?Z2xO%uDJrb!ynLGO=8mEC_$F9rVf+CZ%{V&eH-`|qoEq-r983`|no9AP z`q1Oh#%WiazBT2or%>Fa0co7PHPvhNt5DyCJ@ai5;z9eiH_K^c;hy!>=G z;^Xg0@`H>|nTR3rWS>yrfUWjQp(ha26Lmmz?nDZpd`s8+y&>sw2L6}63mo6o2h{fs z|5Q-0+sHh4UaF9uyv&86_E7lQUht{tt9TSV2-Jw&C|Hi`Cg%PY z_W0JG>l6oAXB>wQsbS2J;7j<&X9kiyL~v4l6lIpK9SpEVyp9nNiNnv^({E8IS1N3=_XwxuxD3W4FXwln$7F3wdFw$MInZ;N1tqbfuu;yddkbZYf*@A93?1flt=B>|_#Rdzo3 z?nZJYbLF014^K9>t7^zg!L&V8rhtP+Jy|nIAxK&B)H8D1-GOO#4l>`E22IPl#yBG% z_k0nX&yrXSdOIwesZaSqe1#u?9YYJ3P$;yqxR%Mbu#Mecw-$OZV1{o2zB%Z^G+^Mg zsAc26=PZ2@>$%o^cJ{U#J4|qQ%FE-r)9-;Vj2nzj{`kUVFdTftY8bA8)VVwb7YHnp zhqep}2#Ux1`}^W&s!I%(exq8oPH<(e`w#9D2glJQ0jPGqtL^;0d-E})d4*INJ8whWVc(z7t67fwINVcGWVsld{I z>ftu}px7+^Z>sH^iLLFW_Y=5{aeTx2{&HgYVxr;g5TDs<%pyP4ia7PnEtdOYF zm18821J~+vcPJKbqI7D)EpLqrBdF7~OjtR=`}sLiPSKvKx+d)XVRXL@&EJO!H%7aL zJAPvfMCouXd;skC7EQ5gv$y18sqe0Jj ztJR*9Wh3{eaciLkIh17sV`+7X` zPfcvf1VhXS#Tyc&n)B~B1^lu*2o2hz{0ol-Xg(~5RUNH%pQC;Wp_6b07F2QrEqZ}g zT@4NT3lRkk`WBf1wta4F3v1QZ%ca%G4IzZy)TThCAyqMQI4gJiT5`Jlayc@?-V=n{ zcfR!%0_1iF9$0UCqDmS|#XhuO$h77<0%u*1)Ds~BmmDp{-=C6|Ou6ag@XO3O|JX!A zts>=I7tJ%sKux|`^p-t_yNyk1oiF}l`_c>y>(HrXC@wAvAkgp`0VlX;IG&e02ajsk z^rh1Wr|p-5g4+TA-6|2MU4fU)2yF#jU&~nn88`P~=0K#EyVv4tFxn7>D5{W_IJFn# zraf1(Kkj|gzf5ZaOMY*tDBUGPIeP|3ZA|D>+=_>461+1=;YsXT9R*jAs&O>*@R=(7 zPm6+8TUaO+rW$*tt`&%S)9h6|bwYg*(stEjPsjZoB?JfaL`W>3`p| za9FKZA1_u#&U8NpL6gKriGdJ?zeGq3L&QiTfrLi>#_NBCpdV0_9WgG9GAyo(_TaED zaLhnO>;A)@uN*(&DwVI7E-8TxQpd~Z3;!!3{iHG#sw;6};kU~Cfg}>ol#|Y#<%2g# z=s20-?n>BTJ*VWwI zgn_|PG?Ivu55wn8G+z2pw@Pf-n6)|yebnz-kwU0B+Oa@XuWVU@B?Q>p>iQKCu+2;4 zaQ%Egd_=i$(ZZu z?Zo>3CZyMkhC1vBg+K94Y*X?{kRzaoT!HVTBnbLLAw)2xx{L=y=6OYyxA#k7Nc9{A zaao`7_iKh^x9bd-Rj~g6vP93go&Hz4Ajx02eNg-HvcA_;lz378#7L&wm^>>Bri&X2kQ($Kv%-M3;K++LN?qn740f0= zE!k((OiQoV8*m?U+TiCm>-L{z=SaQE>3FeFdL4?b_j#Qq7kv9YM^k@_nrp)c)G>_` zXPC#JQ2TFxgA>B$Mv?p>N+W#|9{y;#x@Z#4iQk(UQLNzhy}&KoTwN~6>?KukZyq+6szot!AuCryImfCG%&bh*Uw0-}wt$-L=CT4wRE@p{_x3=@0Z z_2u341ni-FJm!m=1>iLdk8!VuJ-te&GXapi(Jqk3&QBZ2#?z`=3aR-ow|aue7#KJh z5Vk(f-IQnqha%FF3_E|hTD?B02}zKPPMFfYy(fgvgx&gWtcQEv0=6cCs@>Vg>Sdm+ zK?^Y-yty=zGJ$Y=a68(m7Q!MkY)lqcuVCwZ6`|C;+WpG1-eu5mL{e|n@aN&wD!|(> z$)0Cu2fVHkl+60(9Yj3t;I_3MNB}Btl@Iod{3a@58{xmpAn#Jgqnzwdr(0oh!XL%V z1j+PKB-|g(2Q6lRp_%WtMVJG3cmlkq>K>;bL)MpDH}u4u>cLsJmjpK@PA=Cg*0zFY zPO!2vF7fdn?%&a~JKWl<{k@8GSf(O|=-KbnRmvTQ_dN3l4oSsj{%p3@$b*^> z)Sq;3*v`) za0l!RlY5b(Jn}T-t)QlV%ac*8|1kzih%R@wm%?kG1eQHG$`Y4Sf-uogs%VhGT@^h~Scb@-J_W{hv}{s=u+$r60_0 zX2kQQr>uDW%=J>?b=CwTeTI3~-qgMIv~|T-^)lxyHA!n{;7Ky&O%+z9l`;{&5L;t> zoMKvqOJbWukRZj7v&R-<8P;<*0Q$T#Nz8OG$_SHb8~Jc|0OMwMknIJyG)Gveg*xqTF|yG&pA7w%za`s+WQG{1cE6W?D- zA?-^*^yP-F4Ohb=jLBG$Z*mE+MnXCfFAlQrxoZr8cKOvPtU{U-m}&qQ*9QLGckPZX zGUSJeeNor?C3IXeD%>4?IbnfUS5bv{niD68hP6z}7H}?BBFGNjAq%ET*47EE4VLGjPCDD-f(5mM&ef%gk^>ZG zjUAA732^oOw6N>tYRSxkys;;EXRmyMq!fc|2as#+Ia~?Wyh$-Sdz>qbj!mpI&#bew znrsgUoz}Vard!TlTRxm{y*}Hqhxa3aoaeO+CxyVYGn`gL#E2cBZ(dn7)v&9I} zit!-p5=qfY2yV@ptFG-ui1{hJ+V#Mgm@>wRw1@F^WCzLTH08R zJ~dvA!)X-C@@m>}DGOo`c|dtfA2KUziT8l%?YK@Uf%1fJDvn2vc@+dy6hQqoOIdLU zD-rV7cYJDW*PlC^4uj&+;kdoyEmoGsYPI3)-@gcgI}>Of67leV2dv1YY1*dyqowPn zP&Pn5-iehJ=u2ir8BKEu+UVk{XsyPiRVlApHsA9@|9s!qbr?SQ!fFsFPWQQ|NtzA$ zjtt~WVoX)^oB%h37Wsk2)54xB<$#F$as)Gi(r`VBRHI(#C@nhwd$zqd{MA8)due?= zc)8B(?z&FREGjv9|NX{Tsc78h%;#K-yCs73$AoWM{yXRRQu_CwOGHKXna^Ui0UVL@ zrX&dW>tUjt8{aMtd?9N$A;O#JjP)_ZEF^BJvJ;DcRLJk(E*uV!-_-KrFn3Fpo73Fd zNP>sM%InL(fW>;Npj^emb|@_p6|m{M4d8{V^EgSsPT({qT(AVI|Ba_P^NYvW!L9rfOg0D+~=63_X1Il{OxH=i-Q9pK0P zNv2YC&Kk<+E7^$?w|W*QENI1?qfkKbO>cMahtC%x-?4pBnQ`Z&F*nlCpy*guBA~I( zamICVS)fL4!h5Z&5w0>{QkiIfzC3VwdAck1I$_q^XJ5!uHeYi!UUpcZf88x_DPLuG zcOJe2n)6Y#zOgr6+q|RUc8@1E6kO|USkUds(!wl|4UT3C@BWZ~D_+mO{!XO6Z_P!D zJ`4I3BYq94e$<+M>))aJqqQ^?$w&~WJ$HzqV z{jrWb1t1rSLIBz~}*i9X> z7SCMR;h-EU5d$LlG$4fm_a+%6s=9J*g^_E-BaA&eb-JCVop9$o|e z5l?LD^?WyZt-p=fXQg&?y3S*Ly`elPNx6Uf^c)(y2TB1D;PK7lbYNg$IImR)t0jPD zdehQss)ff)f3!cIx2W2o4U%<$4l^K_%kv3!k%A~9pHWwT8RvD<+aIM>N@PtI1)L&# z!Hg6hN{0>Qo6QB~qPfH1NGfET=1PI}kb-R4{dUuVvewIP?1ER5^;YeNOM7ln7ANp7 z*44g-*VSn9hzwL`%h~u5diEO~nf^z4Xp+pk-6yKpC_V{Q9Nw3)B)iuf(?&@Es|41w z(N}~+HQ}MpATKZPtC?-k0rU3jIkF&@N)}xFlB4VZ&m0R&w}nc>Gpt4_j681jF0KpZ%p3M+L|62AG<;pI@sGSDMJWWd-_(RyJyFco5OT;bDdw(@S+(F_GB=ThcQpT&;{6Bl~gSm107$VqHS`VGU zLYzD(RMWMgHnD6m`E#bM`vNQr2O_%p!JZ!Z(oO)rJo))3iicwEB{hm+GlW~VSKwk7 zLpGL8z5x3Qydn77Mz4;dJepF+z0-2$DWhas0>Ei&Wo6i6Mu!>v<8aWz>gc_%wNft2 zZEi@nXBA!_82|ZN#Gw4n~b&=bI2e6Wgyt+!FNb6yuq6?u<+ z5H*#6`ooArz&2D)O}wl$xa>Le5K5`!gDb&_q_4cG(nIN8bhDyA=G}oNC)JTNE*M}M ztUdbgkvb3+3CnB=%$B^LL=yk@^oEDn1llql8v3&vSRQwKo)3$( zND|DDp5+w?-w`9JcfLjwGks~8zy~#=9*$N8?0)WtTCbT6AFaJqU9qpMOf-dwDXoQR zuFeBxD+7OaSV|&H4M}wpWA`gEMBDO+PE8!~m(PlBZCtx)O{szdK^(So;=FD@@4$fL zZgb=(nb`}b)k2xfW)a*y!Nk@mhAhAZ;141F*&wh|X|ul1lI}R+f$~s>b-t<{(V`Hr zAibe)aH+rQ+*o;(R)b`N>ct&{{FlbZ;t=AuSz`Ja1xZaY! zY>g-V6%knZ=9P@QCsGwX%|@G_&QUSJQJYOfN-8>0+&MA=8x#4sf>IGQc!>6#z(DI> zh6b=ja`p8{x&~oi*>zT-e|gq|qQV=^2TLNA^|~8B5FNwkK^VcR-hv^=%zX7JLLtWp zkN}kOSM#;PLd@s=5lvSU@qAnD2+kOvSJ&OurdX9PJL`LUEg6mbDLN6cR9e+cpDd0* z?wFqoI3!%}=*W$Sx9x|FgEW<^K-N{krWL~Wsa9mHs^6B`03 z#(}n}NuVtfcbdeePUFtb&N2PdXW8bXU-94W0b&FcrMn1t>d&gCZgThSu+Se^5;9Wj zdsamA!p6QT!aBrP^}PE3spI+yv;u!{T?~0L5sT8)SXh2eyC`x? zBXy9VAqsAgc`4M#YL&58%gCMMksW1R10e*bLBS==VjOaB_j&Z9>t^+rppHvn@idw6 zpe=#RnP}ODQ1yzDCMqfhz2AXu%`DPn1g0aA*mKq)A6t6<&~GPD?wltRLrDbbW8OqD zn4)`2a!ilnu8?S4cNyy;k`#&fDknov8@rjcnVH#?d?FhfBRsPSQmfYQxe7u_4^L$~ zj48~$QjtW9Suy=3*dt{Uec6bI$^LYGDjC`v{+pS1ziYI_=nOVsDxraqK%(J9#*z{9 zt)&!KCmVy7b&Q9Sf1)3y!qkwD?|i=CK%(qBKl%k= zN|W|_7+5KLE+|NijS3B?n4G3dK=p66b{$#U4io?YfyY^fVHpQ0W3( zaC7Xk1b$PSv&n_4P8`U}gao8$Uhdtf5XbWcu2AGK*=@9;2vW=Hp&6jdyLx!_mq22j z8bGVkGY5ro$3bw!$u|}Jy<-9Y=~Nzxt7GWy(p^%8 z@19W+z7Z^-ALu)cC|MOiCiWCAX=uNtsDFrtQCTWG&$>#8Yl29cf7ib5N-pGTwDf>$^S7K0(;~BbJWZ*RMXgUMT;n*0G!X& z?9g(o;Do2OV{9qQhXf$KBr)3sn(a%mVUOwnj^df4?&~Cr4KcA5x{sPnymw%#9IYAh{fX8mO`>Mrqh*N(%oR^R=4>QPU7fu-Er64X;+EfKE?_?a*xnD-}fl8ou<&?+qlT6-)Ivv-;@-8PpMS@cS z>XrNMzM;c0i16LV-yKDu{kwbDdkL*0?uJegAUt+&iW?a1@qYe>mPf2Eik^D^ifd z@VC<%D}m9(_}}q~%-M?Ex7OS3k$dN%BgjiL)@ARrRH7gZWXS$XRdoMXiU@s=n!VJJ zx0lC`g%$I9pUX23#(ymf;K~XIp7au$?;Y%6Id%k?u37=T<%*c;VqLga^M}S_W+qwi zLbJzpz7;fQhF6*ZaRQ!DRfVLbbMNL~!>KU>h+0r9P<~V>J$dv5TG?7~P+7KK9O38B zuWayO5mr`{Vv_cq@UE&e=FS#y=&Mtg?A-ST!&A&}TlWUSf-Y@hQ$-8Ly4a9O2%9F%HGH>oSWXRg z8SMrxNyN57pJ|y-R}?hBtBYPRU0#;^eZt_x59%?rE2~!{c%3MbsM zXwdHp#X{w?e<#tZJg9X~Z4Pd*#R{1@5|XxeBqxaYb+hOCNjT>1G#GF;tsAW$?d@Vm z0&*iF#%A0xfBfMx$930K4f9788i5}GQEPa`L#enEvZ7F__11b}6=Q;!TnkOHpv!)s z)y9(W>k?^SSb1UQI=p_Q&Eu;3;QjE<%XZv%`=;}H6+>w!BHfY(Z|eX?qyxAHz?_Tb zGQ9(_Q&D_-Q166y=y<6R=|>P#a_^GU+RX~Z7TJ80zU zVLlPYFfCMQW>E`4!h!v7iX0mX)_B{V#}?h&B0m0xQS4vGIU)gU1vX0+4Ab&uXS|l4 z>CC6vAL`><{y@GF)m*8cqhGEsvsCxqFl(?V!FV3c-Ka17W4WSi^z-Ruh)VDQaxhy~ z;9&^ITn!~ZZhn4#YI&LG$WP3CP4)e=j}u;w8L+m-s!ON|7Ar@#v-NcbBV8u0FQ1y7 zs~&!6Jymd8)Bm zt;4O^+EFRy=UsadXr|aaglqyWA0LOc_F2I<;<>7(Llo|O+Zv$`m8R=1P_#ulU6H5tbHQDE_z z-_0@q?fHOk2S!-Evzx1Jk>dt55AeEPNALZc$JJQwUCoU%NvI4z>9349%;Op3ux%o) zPcnPDA-O?QHF7I2=!p_@d5TXX_vphwjVM+WuHrJFnZk z10EkMftu4P&y$=8WYg!*Vmmd+pFfu{>GdO*^2cOj`=f)rFE0_K(^IxG?#{RRDH9Q` z93bleMbu%QD2pOlY$QkwH7Q5;V;;(nx@vvO#FDM80b)iCuU!d??ss?AfI=Os=6QZ6fQb$>6U7@H_ODZIO)!s|iQdA3rF5GzK8rIoh7UYbp=w#(+dCbGgwblalJ-Xy6w?8C_ls za+Zp;^r)bouEcK<$cv^w#l<_5hqc0-iX{;~BVukrxg;zA2rU>A2#~>n>e>$i?%g7I zu_P{o$3Q+!yfW5uHR-7FdRi&9T8?>3S;M@9z-+sj_Y!n}dnxnCbfL6Ef!xJ+0ifuE zRfh$uiEu(*>i{+N55gV2y|^8INzgjbbP0Z-ym5)JJrdCiH>ZKTHLz%hCq0piIbGIV zKbwfWIiLZRq0IrPa$yGVIBS~C*6Mk6Oef-W-UT+R_cPRTJp`izfq-YBqyg_+pH)=U ze%o{)km|Xhp#qM%S!`cyzL~6LH*=&i(noUZTj0Y-ek?w$6|+GtE|G1FsF7J1dl3Na zJ~%XFD{IGf6crP5QspiJIkmY^6bWF;!f5o71f{J>5f0d}ySb6`8fmFdj)^W;mZq&| z?S~DwM*)66iK8&sMiMIs?^8D@W}eP6u`5>>r}N*&WM$7bxXk920E6)k@3Q$B; zRbhaQf;#;_=&fls!~n68#H3?>j{=T3i2^)84WT0;s|Lw)OrAjzZ+bRIcF(aZ8+h5Cz66~X2 z#Z8Y$6qVAgN1jA&P=`loN#GNO3TJX!TgkNb(#H*dN@)|h`-|A~dApJti2JC_i9L>Ool zISXZ5&~Uy9gx$V%1CmxkG67q8WNfW<*evb$h-K_}cF}x$Z95`%8%@HK08=3|Q?-1_ zUxPrpOvHCQp8q;i-eP${<+SFoSKb#^Gov_PMf`F5h zad5cVZUMX29lx`c&S!0c$X}xfZ8tm)@UKT0k+DSv-@fE`+~~C40AilUErr4~Wv#JS~f4=L1UHF3jxS3RKc!)MGePN7|R*yelOho;q`QToKvwv zTsDS6V`!nP$DB09CpgRsgmp#ohK*4%_r~lV)Tq?HnI22MqXKB8a6yqZ<73j#64fB* zOnKM+vW9t`?cOjAHH(Q|K-3}6n98rhT28n^8;-G|K(uaiqY8c33^Uz1*W$!CTiCf*UD#RX7L`W zB)nhwsh@&!yVxvOH%)jQpEo9))?FUtBzjJPTDqJ-bwHQeTTOzykQQVM(DGfFRWh#Z?#({gr%P z4qW+~`!dR^03$(UD;G6F(1lbxw(mbg(QRm#lckLe%4&4R0Ff{=zs_5=KM=#BoH%5U z?fv37Oub=-v z)zG%<`4Gi&ikCdTLM)BNg7)x0OwMS9@UA9>F8b#Vuz?`?D(w=bVVAN6vo)o4udCEN zgfpI`>Brjp5iOb_LZ|0)KrX@|44;M>YQQ||tH?V;2xVnzi%jBASGkgERa9ny%YcKB z5c3TJ8F>t#H!Y2gvo7#|jF#^ky0V@H1M2o{w3sbQUh`;Q-E z-|VSwOzb&9z?#XEVd zPM{?+nO8sk`8x;SdkShi$jp4un*U>d?;YTPEquAkA6`#KkQZkd`Kc(KF;vFG!^19N zM68_GquB)w)Y})WfsB033fhk*$7l>1+p-p+%mNlCy789t%1HEZIJky$-IQgc+b{Y9 zP_F}G^1tQf{q4ke5wDIT4K*JbWV38+Y~JkOmDNq|QTV5jKz0hwv5W5TMMRuYTG=aU(B#K~cCahRD`p}eB4mG>|UOEpHQ z=>m2QC+h&;LgOt;(5YpjlkAMIW?yEF<+>e_p*l+BR;BPG6u1H{Z< z{+|prYoR4*5(=bTzWT6PYf{nC?0A}cMJd~^PkNOSMxr!Y#dk8!kKQNZ@5<=GmO^&2#(`q6<-e4|%Km=clsMNEj_dC+)_gMLZRr{l1sUYJD&Nd3V zEZ+mPp@!yl&orl9xd>pX;bE<5jq(CQkTOJ6$;=lszG9TxWUh<8oi^9c{cZQ#nc~Q} zXl{gRp=h{A$0msdl&-=6fw!WkrxO&X^OK`IP_vDQ_k7i^p*1s=ciqEDS0@t6ToXN} zP~(I)e8=Nm;h!ckJ!~DzrW+g?xe@y&>L~*7GMR|W{~1aUD8yo<7k@aqjCE}B__B7` zLc?*T9N#4L2l6i*6EE(W!+6?s`beS||7Fd@zKSuMMu!q>IbC=(RK9~3i-39TcJw2*Jbib;IP8DTo(PTe zb2ny9!0^Y!#9-H}Y1-q9O*Khiro)lA)LUAxdw&P9kq09%y%Rvz50@3+#sQ`;Kzzml z!hQG1u%C~Z-lYvfsKfOJTw>W`agzAci;U@~6uWT`mw(IlveS5bq`5 z6B!csR(CoE!}q=+%0Cul0Ye1xL3w2PRtNy|7ad3Wtz_-$#brEm=N&L=d2L^rNjA*` zR#O+n^L8B-Zo&v?%4^74l?{?>K;$$x-xJK84{uinhzwj>#-eyoloZ-&u3Q1Bci~^- zndAQp-73#nCyDbiQc~{o08kl$F;x7+<)SC1{mI6n0Q?XZ{Loy@#AS={>*$ufxCAA- z1m#rC1Z>@eoD~KsBxxxj{Tb|oUE?{^>I>u1+fxW02KzcG+Gk*3C3;#V)J+}b7eH1C zC;s(o8%F_29HwsHHY)@bh~R$SMA`BCP7*+49@UNI2P2$H3$HfU$j(|#_KeA!#9COp zJ((Sw0-$We88f2o<^ zB&$i5un$-1bLkH(Hy!-vDMf(Sj8ebUF@GH^|19R_PDgyieWueWn=TNa%ro2ZCHH-f zE)2 z;iX}O_$|ricSiZ7u^UpUPDbl;auL}BgV6RbYGfyR_6>%nRu zT8;`nlc4zIfr}7J$Wtp!l-S5$WN%fSEOH0%qa_Aus98w;jT%C6I#J+?Wy9Sq4xrs9 z)vz9@4fqdHNTM7%c%NRI1~Bp+?JuiW?;}h_Glk{wH~+pM6*$@GI0cIqLCc|4W?4%P zsc^13>#1opShnc_PF_}hnP@K|zJgHU!yXO^g5}olwxo2465&rLB*+&tlnQfqxPEl= z$AA#3a<9#xGY4qEkuQ7G!&eyO@vkd%x8M@N5%H0~$q7Xna6Tfa(@iVY?v3Lb1o^Yl z)aOBq)CBWWHmDrfQ8-+e&$ z)HIubKuv@alH#molaHElAFODQ{jrUV;$x*>BRQ!*ZfiT-dNl6UNP*bQF}FzOwL}JC^#8DSTeA%( zh}zx&o}d2;n%)cw)g5$32hpkD&eZ^k%A?9DxYrDi_St#uc){y!;k~`x+yCPQlINqr z%4(s`=VZ=3DWSi_5Teer9_C&nrS8#Pg1uc5H<2sv<~^&G!V?Kr(}G2ID!Y2}{(K7}6lvC<0Y>eBDX!u_%5^NmzyO_^NlPCgS&S>y%j`VtaI<+>e)i^G zqToc!+v|sR5eg~kpNX(Xg7@0Ta|>EthbrdAPp%T_VYwuDdpED)oD_O%nP}BDtWFY- zeo?ERL~Lu|hm`tHeyk&W6tP8)!*&?H=MVdQv=orjTi}ejCyJ{T`C^{3`CYQB;DDs} z90nk`C}Q=arO7f|DA{k|hbRDXDY++jSQh)F=xe)Na_e0xnr;?b>nT8w4@nNS^b*Dx zX*iS)lRpF32}mLvV@aE~%~}iK=a?QYPY5B9bV+PCoO9j5$*e{@&;AmtU-Ep2G@az8 z3Kjq?7u^2zXw`XXJc4gQtY&3(`Kzfm+z~7rBqU*pl9%LE59?zFXCChif zEP$msN-$O!2m3-CBTbgxq6wSqAk}TARy(di4#H@WYqAKu!|)IFo3aE3{;o=|?3Szx zQDnsCrf3>$5;j=-4J0<4LctTS@0fYNWUiHi@TDOt6TOo}WG(hc1_2JECeG9K%*-7u z4P|MTQltJ{@mayT%R#)+iQ|H~M0ykvl~2RG`U62&^1)9jVt;)$U;5X{WHeCHhaq-Q zC3;MWiWF!9=GW*X{^$<-%q)X0&obRDf27OFPgn8NsinZibQzl-)ci3EA3RzQv&_VsL(1Y-d?y39x!&umwv}qAd`$B z6slJUOd$53vHZN$3E7BG1ZtPrLxu8|nu;b>wAwZjJTU@_9tnUE9hM4tc(OwV5Tn~y zK!)ap&%C|8)#ch;O+~*!F<${-lm-U87*FHQ6G~AxI99h#ajo&@fgr{kHD1qzd~%i*V;1E>YK% z?(CeS^Gju_KV`)K1Mpvt*z~aP`x?W?cjJa zAds8C$v);ZH{-u~zHR(KMn-lTnn{{sg&I`Z{lYB55<~Vhmd957UxWZA){740l`oEP z^co|`wshrr-A(-p++>vYz3@1m=!*b7yqX+ldMqf8WyeC-g0RQ4AtZ;DK>k;sefi(- zJZ7UjqDWl=uJTf4{%8*+KbY5GL6=sgHKk+72#&lU(fTr!4prC{!+mBW;bxAnp>u%P zSAZw_t5t>6Xo`OQiy-3nRJA{1vlVa3PV@sRe+I@g_}%{IDXyhv@EFhlRfG0XT#@J3 z&jE}$fUe~!4y^c=yL5gFnZQ$rlZ1mvohMHlI-V0xduQ%^TsG$SdYO24*~_y>u_vjfw?-iT1>%9(Dy`6$MZ zhVIv+yrD@`NVSB9bTyN(4+MxNm@rTL;RF4!ht2l1lp?+OA?IkvakD{YqWZsAo^JPG=jh z_1h`jc9i7o4!zbK_kGYsP|T4o%{~kwP~L%9wCz$d06uj;`4!Jjgqv%$=9SYMCXT}> zI7gvkwSdI<3b@ed2Z#C7KVs8xt4vP~4-JYe3evnW5$V`a4mN{Pxa&u88_myti;6_@kysd9LW;I*aSE7Tg| zv^D)Ds}H6IKp|a2oV4b=+j+5;$?vrpWbyrYdG?F&Wa!)d0GvWfYSHl!RiS|p%OYf+ zw66prI&^7bC^3@4D<>dd}AeA9_I)vT{Dz2QQvNS6&JT%4xRuBu1SiutO;BLacQ+D zkQN4!e4NOL58UT^7BxGZ|6P8HX!S>HU%&9uL`^F6n&) zE%>xf%BJ=_YeN7Loxsk<_T1+Az0F<6;46!*Shix|5}-P+a=tjI0k=7=cD+xXZNKow zo+tk*%!Y`8OMn`(4)sUl^`44iTP_fMb2%$n&~zJ;YiToKnf&|@DtGW3i+I(kyPUlm zse{x$4f|%!ztuoHDz6YY+Z<}{uOx1qLeuU}2oq8K?RL2yf4a6G(L)}FLkfNc3LU!N zJbpTTiO2Xcga;X)E86oAhZUC>2+rHjJ&u$SY&)!7cbRv|Z~U4@0Y;-z0+&AXdv_Ue zWJ>b19-Y%Q$I|egnMchMi!-XGYf1k%`2av017=X8YM|SHqX^fxWd5^qQhPB*VF3Cw z*Z|Bo&WvX8gFQP>L*Txn=RtlTip2A*UAG^sp+!e5<)Z4;LZEtIo%u>7F8EzPlW}@g_T>~HKj+LjlN1fVon-k zV_DF+>_jj=(kN1#*I05>g?!5S)jJ%n@fm)Z2xPnA>DH5fx5`@1O-cEH0qQ^#$$;t69PL~DjL|*%C?BMFEEg0j;SY}Ny_Hv-~Ubqm|l9rh@j#F<~AkdeURNO$%jR%J< zCJV<#q{;5Iy#t#M2Meq3;P-15HrYiL*j<7!zGXB1TT9b}sIKag%xR;IU#6ep$5SY& zLlOzX@ReVOY+8h&wzq<*nHD2Y`R+MP1N|`&RaK2nn?ezkpT+#%8Of>Ie?Krj!n@vt zib;jSLc!Ymi2DTrCrK2nePEb{NU2RTj5d!|#$h*3Z*IKeb!3&CHQfx`RL)UXL~o+u zXDs{kG<^6Ux#8|5smnky?6Cf_@^qWqLV?pp+f%Xj!bs}@5Q~`%8rxP%_{%CBqb%KJEgl@q(MO%r0(4Q z&pG#e_l|E2#$YoTYpk``nrp7P-sgFKZ;|!Oftw?zUpK@2A6;=WEXJ(AHXfo>mc1Tp z7%Vq5uLG9zTZ+_KfC;fyx~kdfCZq`I_v65^;FzZv zL9;~UY7H@vA@|x%yVOzqeluFDF zpH_*BbM?F$Llo>omm$U!hTHPDZ%2OptpC#J`qU2-f{x+jKW=xUt=Kdv-YXEFq)9G= z$_iJBA&IR>n}ooTkJo|k(ONdw$FS?RAO4cy5@0sE94ll%%~tbO*6%A!2<(D`3I$qK zRlY#NL#If_>`Y*X(fht5qe-bF(-k`3dbr0cpTa^bGnIRHuoy#xuoIuWc`8zeaK{Y~ zA1(ynHPuvyb1uut!Q}wYzsfXKmjGMBbAezZZqE;I2%@?;V){sADhaHD1%UuFiTD_T zmfNvk&b4Rz7k@2K1CHbQ1=mXl`9lIHUjijX4S&@;Q zmOiacmY;iFa9CLf#`l_}fDTZi%;sJ^<$lWZgV+EE1wtB+{_#rb!1i$;xXT;X%EI02 zUtf8P)w%8(kenROY`?T#gI@lo&_Ay!4^v5j#2j?-jUN!g3++LU&BY z9t*5DO(<_VhY8R!MmL^Od70=g_+P#dCzXvr1l~L{fubP@;5h(VKzxcY60ixNc1c1c zVTD~uKR^5W!mLl|ka|diT*aJp#ax%|6S!bnC8NxYQhk-XG=zcl{)<}wSzK0l785E@ zB3g#-t=v}sP(?^~$^K(EQHSw9xx)8SFJ4Ub^HAB+>!~1%2o@|)6r#}ZVS1|hV?@Lr z$6k5N=ao+SzE_MzOZwGF(=F!IF?hc=DosAp$^?vy{xAO7F0!1u} z1Q>!(rU{jGWEjgzIhWUkjolQBEdDJ%y*-<-qqeint?%lCP+HZ@1>4kwKbp{lR4P-| z5G>W{sdiZ4fdB;xge0aL#3{hX@R4oJr51JdkjEHNehfK>3ewWUf@o-EmTz+#P-bSB zmrHOn%hPZCzkzq{4(YCvJQv%Dwjct}JYmiUcJzrVI7j|4mQ5Sy%fCNE2I6ke~D%kpuSeIzB`xPdQ&@tAe zR8)wRl_g|=m}g-T-3xS4(<%yC@K>C7X{oA7S4h>wJ%+f6kyrB*#Myz*v)-0kKD-|% z2RJ1~8+-YbzXmV&MLvW&n!trr+!fTdUIdP;nT|%tL-U+m?)0Kqz!mhQj(`we$$q@fNDmP&yjrDuE z3A8|aL>8pqVOc*Ag}jh|pC7fH&g`zXHveir9c9OY!})cweUO&E*8!f$bMT59!|d=l zpRa@_;N8^pFALk+yv>1_|Lf$kqy)deA1ZDgo`;1M{=whFY^^OAC*+wXW`nTGYghWN z6fcR4b%d+~^vR7;4HCPzH+?cx6g22NT>PU}5Foi|5Tr!7r6Y$uUI{KDb^KQZ2PSW| z`;RHPa|{`e6H3Dty|=(*j@^gvzsW~^lP^It@3VN^jUq8e7~O{n z0+gg)aQD=#-u=M{vU|(FEDq3Y?U6GKKkQ3ZDsO#-~?jvQaJ`{ zw)!DR-RD0pRaoYw>wsD0R^y8S_EIumd!0VnE{4!^<3m534yw;)<^!8kg@&!ZXA$Q- zgF!r9cME1BsOrnXjjl&nUfDk{tgHoin{&_FWs|Ctjjrk*DWYyy2^J~nU${YuOAbXe zb89nI#t$X?RzgNoB&1?ob7CO5`E2nr3=fxYg=^5`Aqn#22@{_fuzgHVk-lVqb=ew4 zN#TAd@YPQKU2y3%Y)67_XMy`ZU&REpueXJTp1(9(w=!Z2IeP;|cDUm&8qQsJ((d!XdUV#hR5J~& zASiY{FRl)|X+3Oc&|k2llqny=%JHKOB0FKj5ONl1QHh<+crU60gdiPwwsy);F5(fF z$7u&({A;(r$V>z77z>y;N4U*U|3F#>;mOSS_^ns!LvUawWymI3OM*(R_updhE2;0U z!XHQolnNJX%bjajLzweC+*RbKo!8zz_?CJe-RmAu(wS!Zg4IN#-?u6Rt#R-DV#1*_ zA5It7zNa|ZtVqy&iDzh$x{vm()CQ$#${(GOUY-DmAWWZf<_BZMYJV zmR6wMev^OzpKw-DP=Hoj`vMajHM+-{6)NJ-pV>A2$~Njes|GcqnDkAV)}fSijhU|Q zE3|l5po*@oN2IC4%$E6qHss^tj0h=jcPnhPz$%7>tF(CA=Z?Vy<^5O|MYo#c6BWQl zI8(nObny&PMa_~X-6iu&ub*Tah4s_Z-$YWhtry`9_YY8Ye%UpMe&c!Kjv{(!N-c72$E*>i z|2F{9BE#?n!TCu(U5iu}i7lZUFr&0O)_&^9S4t5ttBcMxKZYo+CJsh&4y+L*!HMC70hiUt;jF* zq@;)nr*@P^O>*<_@o+;3_!us&CifZvnN)um)O(S+yDuqvQRmc*a$uw-fFIr;cxF-RhzrqlF(jY-si zlZ5*`@;P_cuQxo0BP{}E6^h9vK!Zr4?r;1s zYjd!-55Nu-dGaKun^BSD(&P2fl@Iy5(rLu=RPVL{xbD_Wp)cI~GOLz4hrd)@59Leo zMOa|KwPOHp8-D8kXPQv2Ovv%7a{X`%K$Du{4gxfmv~GH9`!`I*E`Ze=xYS)8Lwkm-VyVL*Lg%n+qw z&SQ#E_lpB!-XbGme}nXM<{fk;)73Db4qnzxY@(@Qbmo^E84?EChBr=|yPVsT&QCW1 zFgV?x{QUgT#q!0cFNRZ#2L8#SmTlvEMzn%F>axbY?lmX-h~3+UyNfD?oss5ytBzms zR-JMATsk_p3%Bdx53M(=zI{_vPc-f!thicm`JxH7Rflp=Y5r>3$~ip*`jW|!2g>y9J~6WY4VHWkP2sxOZyXP$RmD+Iz`PtnxgYWr^2 z`=$rWQDXrB<&$_t7mY^~jR)nGKmesk(_NS@eRtHRI+c~#dKPd6y+Oh_OK1N`+nG(J^3DQiK&9Dyb(-dRYrSQF*J}A5hDE`jaF0UWSqbzxB z2fKDL!aA`(Z?{tl!X&JN-vHuPd?*K2$1{C!2dDG_KgNoHiUV!C^uT1DJMSELG%2k4cMkb*1DdvcV~KO{dhJoFo1AN>QJQ-(!CzIVj8Z)+I5oGyCDooz3Ddq%8~_C zAeXHId~JXx5Lb6|L{P(EHR8bTCmoJD^Kr8PWfPwjD_1j+2KWkzn@FNUa3i=@^eeE| z95TKFu$ib34fPRiJJy`Bq#vn@ANLO_{`93P9i$WjOBC0J5otvB0o0o4MV&h(@=e!Q zlOCSajqH`mM^SdHw=hkpNp{7(JM7VEvm_8*fExw!UFiFLYAUb#>Go@keHumwvf-9} zXzemsjh<&TAO878+Nfe#n#V}GxV*i+Z5aafl!oqZA>g!3ftXYXlz74?aKsw{a9!r{ zAAOcvy_qZ4cvFo*qchf9p!2O?R9Fr%qQ0(^-}b;mbh5I(cqFHbfCx9z>BI9M5>9Xy z6MVJ^CQKg>)%1GC45cGJ;h1T9%>v0*{!H(LXBn1&)qTkt-S*xB%hiqWXqvlL&RE`L!qN>c{Wn zEx|`-NP8wUw1SZCwH21K1e?_UcTJ1vAqeC-i7*@pd@E8W>Hrg@o%yZGQdLs*Nvd)j%x#2}0^`9?rP zFY%d8yq3^3hi2|i@?AGRE~2LdXtduX<)zfGW}4f(kq5*QF2|xgtb1na=#3j}m$DP} zJ|1H%lAlwBQSJIREd6vJhNoh`&?L|v;22;41_aWPq;9&JVM9FCy&3u2WlLcR-IJoi z#54<;k!YWu^3U2-^*?n}{#(KId4?*NQsnA8uMD)G8oue|JEESxzPb9^cNj!pxDW%o zy7Z~f1hHR+ZgzYCpwA3Q+g-v{YCyzI~ zBI8$~$OT--kc1t{- z#s?Pi`lk7C15>l5!*;*ZxHU-NQs)B(*Tp_uRsn&X1=GnF1lu!JJT;bSFGj($xx`%S zu%Q-A#og<_c?JQk)#O_lKfEIhR{MwHa$$TH*>S$|GVIMp*gfRI^NSqxY~~O@G>IV{ z(dMe@g#%5ja0vA6OXXL1(yP=@no1}2q2c}Y3R=~laQGtK5#0&k1O0@047KBTX-RST zPg~?5e@1O<2nkBPR(z!U={mae>_c<;RphTlI9ycUE}0O&DHLmiTl{i6@(6NLC4Vqq zj=Qc}&Xn0aiQ`G$7@v>OfCY5I@Eog>1Z$J*L?Zse=N43to0B7c=%GKn=;_`#T*!O< z(XxH|I92p9p_W4F@}$HyCDd(MtFI(zv18TWmpty)ZF8zlKqVO>aW*cf`X@~xJ}!SI zGM6O#L!MB}iro_*<-G?4H6_*;Cwg&_Blwjshlvv>jVSGF;P!&>=Z_Nz$!tx7M+?>* zk;wJzFVd2qv5s86t`S+6k(gbTX<_|bfa>PvlhrIa&o=76z^Ph{=Ye9pVNn|zZfe%>^e(9cbhV4%_3@zp{L~gS7 z4YhG<=uTUJf{AY{Ged}Azxj9_WazfNM)V);Ir@xZxxYHi{YWuBx4|1e#rhk|zjCxc zdHe;_1izBW*`n+d^MmT07#850&=|whZ)G~eo8o##es?B5)5Qni1za&{2Kuq2Q?HA@ zHCb9cecZgqWxiUsX8fj)7<%H1*oqIsQh59iEJcM7_K-pX07rJdo9%1al2cRiVT>Bp zV5wP*@0H2Xz?oN>+#zOOBf@l{q#B$m9ex{3UNpZ(f zcWD$MVF7OxO@WUs_p$KOiACWfZq_VKKEkP?AVrOt|Ie8=6AB)N8y|yS^IUc|N?>{( zPnvyMtGq81e0VH?nSh%ZUXNng@p{zn^-(;1ZSd2F+hdGumpKiR$z>Civm#QbS#Q?D zPg+C*$>9w<>;;5ylKNq-_|hRNIBYP#E?k)=qZ@OrZS5knvCeJKvsh2x4*+5;$4z)G zun1&D#?OpkzU2o|s9eA>k}8?ylHBKblt2CtQNpf*vJJhBS}{TOT>3 zbc3YyNIj|#Lc|1N(;^G&gLnYqdpI??k*oyEeWs$ec!rc+W2&{4}RvX2)N}Ne0rlOUt&?{@Q|?C;#ob%mSiI(swN&1-#MVsDir#o z7jB@M08~$#GVwRy=@7{zQQDF<@BAYG@?r2*N>;|*_n!s(`?l@uc%Bh|X}kB!Ev0R3 z_jrPNn2P9_L?5^UHz%lm58~(kU$jFTW1t)~IzV3rn+bw|MjMXsMTRpj(qJ(fhTTQX zqN?{r*7mF}DA;b>%-;-S=W3@5n1`AwDk({UNIZBcsf*B0GvIpl(l6yIXewRxDZVm- zP6Hi^hLe3~xnUA1c#GdIGZfpg5)_Q2(_i~u7j2`+CC5a9P;3{Yw|Df0{&(E3?@zOy zo?q}D!6}}!Ly_(|da7eMvnqDKb6gPr`Mp|Tzyd1<1TAnuaHMVF+|D2{v@#X&{Kg72 zds6#;uO7NzvjgsQfttln)$qNPT&AlPyad6;I@+})L0t%Ams?U_!5opAQo(bzu=8w1 z33gF&rtY;u;uxRJ-d(5Jg$df$KG~}5fzuYyIijalQe#1x_X?Nu(;R;{C+@=fRqkgq zCMNUqVS*=ozd&ft^qD zN@kn}tC%*V{JWUP^r9(WD~P~~;T$T;hG5gtnldC4(I}@A@$FF(y6EuL5O7Fx*qdW) zGr4e0zMEP5I@kq8=bw&RiXsa-Oj$cG)g+CR4Lvrw$cwhz7~0HpeM{(u-VKDh^`CoF z3e+tCI)%RFCMUudx6l8D@7AQotM4EQIm18sJ#7d_`%5>I2_}8aQ_@N6n7jbzzs4L@N|^d8yr~!vM9nN#pBbzW*UxeBoGsk(70} zO>eq*&?fy3l+0no6g5(prpA{ zP3467{@bnR){U8xRC;TGXj5Z0E+3?krVNwykkB!`^p}XUgH$9S^rtoNX-Th62c{x@ z&M%*|Oz#9+DiZzalIX zeIEL3Rh0+Ni4gkDm-wYcS5Mv$>(H7uCitik*0X-L0;`T>lJ=R0zUbSiG6Lfde!dgmaN zBJF7U6=)7)Gbz^4~X;>{JRsL*?zQgEKfu;>ZJ8Kj0vow#>B1851_QKIr4;lLsjs zw~i$BF1%pQrnSh47oyCf!38kepL;qo4pXkl?H+c*{QsC3(_Zz zJ{0mF>2jkk239$|?R^{EDTeLEBBIvP>SrlOrC4=wp^W=BiHB4_*7Kc-v?3M;wJai0 z0#E(S@9>7Qh(nC`POp#YhIWK0@iv71ojx{_Uy7g+yId6OKvtUz49`Z7U(y+z{XA^$ zBB6|sC;f3Vv-BJ{6+Xtyo}I(P{(aa4az2Sih%haUA42Nm%tRsb!@4jf3$*XM69_E| zjN6^;4|QZ;TzM!E?8ODXL#ep2BXYm8K;d5XBPJoZq0_jF-$9g^`WbZ{x|8>2aQmWd zdulc`Ejz|UqH})UE>g#$J76W%)aDMs&}C*YM#-;v-ga!ApBnM6ETY?WYCVzCov9!7 zW@k|-hBeS6Ui9Bcp2ErLjO61jt57x~;SH1t{ZVCt7ReDR<2q(gK^ zGp?ek)_o0_!6;*5&PxvF?FJ3bi-;X7wBC>awsDqiJFQM8JRb9A)JgL~W6TCmBuY?w zp$~}&`Oc=iha;_MU+*ZJH-Q6@G^Y<;S&g38ap7YMOC=*u&$0!s89({aR~p}v@M+_f z`st{a?%i>;x?2GQOBE*Vm6rJ{VZG?j41u3ADE$HUQzRv_9@sm1N<;b^-WX3#Xw3)G z7}8@6`6F#crvwXn>y`Nk4DQ*>JCtF7+)MtDoYM~DxcC4AL%45jXLq9Lb~Q`yz6d1{ zpe}nNx$pVBoKMDJmYy}pp;|YbA}wAPZHmczT3K{mgweJV+YY%25oV^aO`M<{|35vw zdv2ZX)YG#~YWfGZ(J_opu1D#hFM-UoDD!{HB>?sw@SIn~L?R(pXi?Ic>^JznGAUdn z64q)d*)c+pvwgz%?+^sj=5>}!Cwe#@{VQB$oB08ThkO&m#B{&n`T889_e$lJei&vY zS|f>c{_$*}od7SfpzxG13g$i@F?nfQ^okJ*5K_l%&d&+Lu^-^;D#YzUu+MMODlT1P zkzMB>dfC704N+#*Ykmw4WhVHTEdt-2gaifKCkjp?cjS zL(tF=TYY-zgpym{#%_y&#jQFW8-jB(*PTg`dBbeAK91(-ebBpYE@Pkp!%#{kXpWPOGBMiX>aymRzl%6v= z7^n9LoTe%pm9v96g(?*qTG|4%mRo}G?qexg_U|y4_%f_qR%h?MuOwuHt9cU7f;>M@ z_k@iv!&<;1Rh({;y4ppy@F0wo-|_d=$a;~CvT}8w0DwQt%$=URqyQ~aB7Bx%Ft4_Q z^0iN-Vjkbo3-EAhY;s!TQR-zKG%Xl#d!H1QORQg%Fh3_bxM)fO#u!+5SFEWKEaoWa z&BFd_;>keRyP>{!mV-MjI;Gh<{SAGnf)kQ=`wmI3o19m?6cqB=|BeCFZXj$YCTU3$ zw1YIvBGAtCIJwzhw7e0u!S~|x1L4fz2+~8C|7lrp?vY&vY;6J3;WN$M>gt_LHwOHR z5J}dBv1krRc1sIaDlcbV`F1D)<{kYURSVij~jMI?|utHU_xx8iI!>N$RutWW7gj?&oM)l=b%0Nz>B;IaTqWJQ{ajQu;pvLGm5OpEicD z)$X?OgI>VVVy&xVY@`xbh;&LBOBzl*x{yO?Ejva+{AkdodivI@?5k!L(XT)$_Jy%5 zg5pt1auoRl3%))L1*a;O@h8>g#jo(Mh0hz$WS>7j;pxbPDizNh5+K+UF!ovmb4v3~ z)AlVRN!gWr5l??<%R~R&&c5V%| z@3lFun2$+EA+ftJP)c-!5z0baZJ#j67(N;(bxgS(-O616Ho`KX;WVcIsP zr7?jyvLpK9j?wCQP-(ZTfezAk5WZ1bMxZuP<3MUa!glVdz9!m3p7nYSEUL}xY1 zq&`H{=E&ePB*@eSVi%{UXga>nxdyl4wu{m}?q7vjQ5B)Vf3qL)Q!s4Sm3wTy{5PYf zrsK7=7*DTYgiHpO&^fR-~y{2RaI5T&7f6NYu2Uj z6h*5MhLmEQlf6Bc37`Wd;Ks}sDRMj1a>NQrp}$KN?P95RSnXVV?JSxeqf#h#89^B6 z4USlXF9AgNA_J*l8W$iK{vH5o4O4#MXJIUAAp`noTUKFKfaB3x2I+8zxaOnncKcRK zGZmLdk))Ud5nR%U$n%vr5qw6Ro*Zf33*4|)J{xn+mzmR-hc63NTiY;HqK7;fd#kX5%6lWD{l56h7&T8@Kc{RI-bC_!-Qx6WtQ@R^OUiGXh8var zoA?#e_T#a*I}t8(KlyQXj(7+{VXp!)A%| z?E@#rd{g!LMKLpIMik5fBp zcdniq(gB)80eTV4@BbEm5@{2(q-wpn@OaqU$s&Sx#^#7Ds7nJo9aRvQ4~GgQ5aQ$W z`vJA89YZ&uh6cp&2Ims#-4$Ujdl-)qmUflU`}OIF8?ai7i;Ekb)(?(W0c~9lSd5Wb zUqyaFB!yL~g0i>d!H9DX0!2d5ZYu3DuX*lTg9m6#1#}-chz^#{at^LT-=<+}_DbQGImT`YszWI-} zAX9Yu_{2>Qte7Q86(%Mj(Y9J`%-0WNmD>O20VfMYyKpA;rCF4(vl0^9Sj)scK+hde zgevI-+hkM%jNlEZ`>ItH!$Z_SrpF^V2agE1Ej4$v#j1DWEB~+1*dl_j>dBn zVuP?<0?0zoR*S|E-7jC}Ak8V2r2 zRU6WRF=W?#4{GawXlN{!W8r#6T(!5~uW`RV7&DUdbm<0R&d2LuIR*vaj@U%T^FN(Dm-`ufjHL zC|iSJWudm55?E-Z3``>J=h-uUN5Pju1IiyH2|zaXu)FawP~!FfPP!@-hWYK= z0fxz`3WXPJpxqv$o1BiDg{hmRsJV+J_yytP;S=EG<>BPz*W?uxz@a%q#G{{u$aY^neN literal 0 HcmV?d00001 diff --git a/doc/src/week42/week42.do.txt b/doc/src/week42/week42.do.txt index 900da32a2..c79a8d47e 100644 --- a/doc/src/week42/week42.do.txt +++ b/doc/src/week42/week42.do.txt @@ -340,7 +340,7 @@ w_{ij}^{(1)}=\left\{w_{00}^{(1)},w_{01}^{(1)},w_{10}^{(1)},w_{11}^{(1)}\right\} !split ===== Layout of a simple neural network with two input nodes, one hidden layer and one output node ===== -FIGURE: [figures/simplenn3.png, width=900 frac=1.0] +FIGURE: [figures/simplenn3.pdf, width=900 frac=1.0] @@ -565,9 +565,9 @@ all inputs $\bm{x}$ are given by $\bm{\tilde{y}}_i$. !split -===== Layout of a neural network with three hidden layers ===== +===== Layout of a neural network with three hidden layers (last later = $l=L=4$, first layer $l=0$) ===== -FIGURE: [figures/nn1.pdf, width=900 frac=1.0] +FIGURE: [figures/nn2.pdf, width=900 frac=1.0] !split ===== Definitions ===== @@ -613,6 +613,13 @@ a_j^l = \sigma(z_j^l) = \frac{1}{1+\exp{-(z_j^l)}}. !et +!split +===== Layout of input to first hidden layer $l=1$ from input layer $l=0$ ===== + +FIGURE: [figures/structure.pdf, width=900 frac=1.0] + + + !split ===== Derivatives and the chain rule =====