From e91a07ef13a522d162dc1013c391b9860fb73c9a Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Mon, 21 Oct 2024 09:56:07 +0200 Subject: [PATCH] typo in tensorflow --- doc/pub/week43/html/week43-bs.html | 4 +- doc/pub/week43/html/week43-reveal.html | 4 +- doc/pub/week43/html/week43-solarized.html | 4 +- doc/pub/week43/html/week43.html | 4 +- doc/pub/week43/ipynb/ipynb-week43-src.tar.gz | Bin 192 -> 191 bytes doc/pub/week43/ipynb/week43.ipynb | 1940 ++++++++++++------ doc/src/week43/week43.do.txt | 4 +- 7 files changed, 1307 insertions(+), 653 deletions(-) diff --git a/doc/pub/week43/html/week43-bs.html b/doc/pub/week43/html/week43-bs.html index f961b6351..84e06ceb1 100644 --- a/doc/pub/week43/html/week43-bs.html +++ b/doc/pub/week43/html/week43-bs.html @@ -2254,7 +2254,7 @@ lmbd_vals = np. model.add(Dense(n_neurons_layer2, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd))) model.add(Dense(n_categories, activation='softmax')) - sgd = optimizers.SGD(lr=eta) + sgd = optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model @@ -2477,7 +2477,7 @@ batch_size=100 else: #Subsequent layers are capable of automatic shape inferencing model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda))) model.add(Dense(2,activation='softmax')) #2 outputs - ordered and disordered (softmax for prob) - sgd=optimizers.SGD(lr=eta) + sgd=optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy']) return model diff --git a/doc/pub/week43/html/week43-reveal.html b/doc/pub/week43/html/week43-reveal.html index c3d577bb1..dc85537ba 100644 --- a/doc/pub/week43/html/week43-reveal.html +++ b/doc/pub/week43/html/week43-reveal.html @@ -2119,7 +2119,7 @@ lmbd_vals = np.logspace(-5, 'sigmoid', kernel_regularizer=regularizers.l2(lmbd))) model.add(Dense(n_categories, activation='softmax')) - sgd = optimizers.SGD(lr=eta) + sgd = optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model @@ -2342,7 +2342,7 @@ batch_size=100 else: #Subsequent layers are capable of automatic shape inferencing model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda))) model.add(Dense(2,activation='softmax')) #2 outputs - ordered and disordered (softmax for prob) - sgd=optimizers.SGD(lr=eta) + sgd=optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy']) return model diff --git a/doc/pub/week43/html/week43-solarized.html b/doc/pub/week43/html/week43-solarized.html index 658589538..13b670309 100644 --- a/doc/pub/week43/html/week43-solarized.html +++ b/doc/pub/week43/html/week43-solarized.html @@ -2145,7 +2145,7 @@ lmbd_vals = np.logspace(-5, 'sigmoid', kernel_regularizer=regularizers.l2(lmbd))) model.add(Dense(n_categories, activation='softmax')) - sgd = optimizers.SGD(lr=eta) + sgd = optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model @@ -2368,7 +2368,7 @@ batch_size=100 else: #Subsequent layers are capable of automatic shape inferencing model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda))) model.add(Dense(2,activation='softmax')) #2 outputs - ordered and disordered (softmax for prob) - sgd=optimizers.SGD(lr=eta) + sgd=optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy']) return model diff --git a/doc/pub/week43/html/week43.html b/doc/pub/week43/html/week43.html index 253c233f8..902ff2fcc 100644 --- a/doc/pub/week43/html/week43.html +++ b/doc/pub/week43/html/week43.html @@ -2222,7 +2222,7 @@ lmbd_vals = np. model.add(Dense(n_neurons_layer2, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd))) model.add(Dense(n_categories, activation='softmax')) - sgd = optimizers.SGD(lr=eta) + sgd = optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model @@ -2445,7 +2445,7 @@ batch_size=100 else: #Subsequent layers are capable of automatic shape inferencing model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda))) model.add(Dense(2,activation='softmax')) #2 outputs - ordered and disordered (softmax for prob) - sgd=optimizers.SGD(lr=eta) + sgd=optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy']) return model diff --git a/doc/pub/week43/ipynb/ipynb-week43-src.tar.gz b/doc/pub/week43/ipynb/ipynb-week43-src.tar.gz index 1ddce2e798239bc71af0e9a3814a7cdc351b0779..2bebe1748030daef2da0feb30f0a923856bf7818 100644 GIT binary patch delta 169 zcmV;a09OCN0lxtyABzY82MHEu00ZsM%?iRW3KI9AfacHg^7D^S{rTWs>wdN((WmF0&$ z^DFSoKXI(2h3&p~l~$m%!(8hcZiqEaBH8vTheD$rTVU|oNrNC%527fflUj*O*cyE@ YqOno<>t{UA^SrM;0Je=^EC2`q01;D7F#rGn diff --git a/doc/pub/week43/ipynb/week43.ipynb b/doc/pub/week43/ipynb/week43.ipynb index 47032b0fe..a304c4c05 100644 --- a/doc/pub/week43/ipynb/week43.ipynb +++ b/doc/pub/week43/ipynb/week43.ipynb @@ -2,8 +2,10 @@ "cells": [ { "cell_type": "markdown", - "id": "cda62a86", - "metadata": {}, + "id": "91e0a9ef", + "metadata": { + "editable": true + }, "source": [ "\n", @@ -12,8 +14,10 @@ }, { "cell_type": "markdown", - "id": "bdae1fa5", - "metadata": {}, + "id": "4f9cc347", + "metadata": { + "editable": true + }, "source": [ "# Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University\n", @@ -23,8 +27,10 @@ }, { "cell_type": "markdown", - "id": "e2c09dd5", - "metadata": {}, + "id": "50848934", + "metadata": { + "editable": true + }, "source": [ "## Plans for week 43\n", "\n", @@ -39,8 +45,10 @@ }, { "cell_type": "markdown", - "id": "7de8bfe0", - "metadata": {}, + "id": "405af306", + "metadata": { + "editable": true + }, "source": [ "## Exercises and lab session week 43\n", "**Lab sessions on Tuesday and Wednesday.**\n", @@ -54,8 +62,10 @@ }, { "cell_type": "markdown", - "id": "2664d901", - "metadata": {}, + "id": "2cfe65c3", + "metadata": { + "editable": true + }, "source": [ "## Mathematics of deep learning\n", "\n", @@ -68,8 +78,10 @@ }, { "cell_type": "markdown", - "id": "46bb5558", - "metadata": {}, + "id": "44a05354", + "metadata": { + "editable": true + }, "source": [ "## Reminder on books with hands-on material and codes\n", "* Sebastian Rashcka et al, Machine learning with Scikit-Learn and PyTorch at " @@ -77,8 +89,10 @@ }, { "cell_type": "markdown", - "id": "1c6ad86d", - "metadata": {}, + "id": "827ced01", + "metadata": { + "editable": true + }, "source": [ "## Reading recommendations\n", "\n", @@ -89,8 +103,10 @@ }, { "cell_type": "markdown", - "id": "9fbf4898", - "metadata": {}, + "id": "0ce57484", + "metadata": { + "editable": true + }, "source": [ "## Using Automatic differentiation\n", "\n", @@ -100,8 +116,10 @@ }, { "cell_type": "markdown", - "id": "135a7122", - "metadata": {}, + "id": "01d1820a", + "metadata": { + "editable": true + }, "source": [ "## Back propagation and automatic differentiation\n", "\n", @@ -115,16 +133,20 @@ }, { "cell_type": "markdown", - "id": "45236eaf", - "metadata": {}, + "id": "7ad34d96", + "metadata": { + "editable": true + }, "source": [ "## Lecture Monday October 21" ] }, { "cell_type": "markdown", - "id": "8997a10d", - "metadata": {}, + "id": "127bf9b7", + "metadata": { + "editable": true + }, "source": [ "## Setting up the back propagation algorithm and algorithm for a feed forward NN, initalizations\n", "This is a reminder from where we ended last week.\n", @@ -148,8 +170,10 @@ }, { "cell_type": "markdown", - "id": "df88cb72", - "metadata": {}, + "id": "bb46a5f5", + "metadata": { + "editable": true + }, "source": [ "## Setting up the back propagation algorithm, part 1\n", "\n", @@ -169,8 +193,10 @@ }, { "cell_type": "markdown", - "id": "46d7ebde", - "metadata": {}, + "id": "35e6220d", + "metadata": { + "editable": true + }, "source": [ "## Setting up the back propagation algorithm, part 2\n", "\n", @@ -179,8 +205,10 @@ }, { "cell_type": "markdown", - "id": "3ad237f5", - "metadata": {}, + "id": "852508e6", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^L = \\sigma'(z_j^L)\\frac{\\partial {\\cal C}}{\\partial (a_j^L)}.\n", @@ -189,16 +217,20 @@ }, { "cell_type": "markdown", - "id": "f6906530", - "metadata": {}, + "id": "43e053e6", + "metadata": { + "editable": true + }, "source": [ "Then we compute the back propagate error for each $l=L-1,L-2,\\dots,1$ as" ] }, { "cell_type": "markdown", - "id": "e9613a49", - "metadata": {}, + "id": "6777bf2e", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^l = \\sum_k \\delta_k^{l+1}w_{kj}^{l+1}\\sigma'(z_j^l).\n", @@ -207,8 +239,10 @@ }, { "cell_type": "markdown", - "id": "dba468b9", - "metadata": {}, + "id": "32ebf887", + "metadata": { + "editable": true + }, "source": [ "## Setting up the Back propagation algorithm, part 3\n", "\n", @@ -219,8 +253,10 @@ }, { "cell_type": "markdown", - "id": "21c684a1", - "metadata": {}, + "id": "0f8d87e0", + "metadata": { + "editable": true + }, "source": [ "$$\n", "w_{ij}^l\\leftarrow = w_{ij}^l- \\eta \\delta_j^la_i^{l-1},\n", @@ -229,8 +265,10 @@ }, { "cell_type": "markdown", - "id": "6acf04a3", - "metadata": {}, + "id": "50067b7c", + "metadata": { + "editable": true + }, "source": [ "$$\n", "b_j^l \\leftarrow b_j^l-\\eta \\frac{\\partial {\\cal C}}{\\partial b_j^l}=b_j^l-\\eta \\delta_j^l,\n", @@ -239,16 +277,20 @@ }, { "cell_type": "markdown", - "id": "b0558fbe", - "metadata": {}, + "id": "f5a6981a", + "metadata": { + "editable": true + }, "source": [ "with $\\eta$ being the learning rate." ] }, { "cell_type": "markdown", - "id": "6b424064", - "metadata": {}, + "id": "a954e65b", + "metadata": { + "editable": true + }, "source": [ "## Updating the gradients\n", "\n", @@ -257,8 +299,10 @@ }, { "cell_type": "markdown", - "id": "7bc86e31", - "metadata": {}, + "id": "ddb60052", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^l = \\sum_k \\delta_k^{l+1}w_{kj}^{l+1}\\sigma'(z_j^l),\n", @@ -267,16 +311,20 @@ }, { "cell_type": "markdown", - "id": "f7a10087", - "metadata": {}, + "id": "0ec68384", + "metadata": { + "editable": true + }, "source": [ "we update the weights and the biases using gradient descent for each $l=L-1,L-2,\\dots,1$ and update the weights and biases according to the rules" ] }, { "cell_type": "markdown", - "id": "2bf12837", - "metadata": {}, + "id": "f6d23a0a", + "metadata": { + "editable": true + }, "source": [ "$$\n", "w_{ij}^l\\leftarrow = w_{ij}^l- \\eta \\delta_j^la_i^{l-1},\n", @@ -285,8 +333,10 @@ }, { "cell_type": "markdown", - "id": "ad07731e", - "metadata": {}, + "id": "2c0a74f0", + "metadata": { + "editable": true + }, "source": [ "$$\n", "b_j^l \\leftarrow b_j^l-\\eta \\frac{\\partial {\\cal C}}{\\partial b_j^l}=b_j^l-\\eta \\delta_j^l,\n", @@ -295,8 +345,10 @@ }, { "cell_type": "markdown", - "id": "30c6ae90", - "metadata": {}, + "id": "6c3830ac", + "metadata": { + "editable": true + }, "source": [ "## Activation functions\n", "\n", @@ -316,8 +368,10 @@ }, { "cell_type": "markdown", - "id": "5dfa6d1c", - "metadata": {}, + "id": "0a7e4e51", + "metadata": { + "editable": true + }, "source": [ "### Activation functions, examples\n", "\n", @@ -326,8 +380,10 @@ }, { "cell_type": "markdown", - "id": "e8d0be63", - "metadata": {}, + "id": "9ec72028", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\sigma(x) = \\frac{1}{1 + e^{-x}},\n", @@ -336,16 +392,20 @@ }, { "cell_type": "markdown", - "id": "b3ec0efa", - "metadata": {}, + "id": "a7972c57", + "metadata": { + "editable": true + }, "source": [ "and the *hyperbolic tangent* function" ] }, { "cell_type": "markdown", - "id": "78c46915", - "metadata": {}, + "id": "d5cdf554", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\sigma(x) = \\tanh(x)\n", @@ -354,8 +414,10 @@ }, { "cell_type": "markdown", - "id": "87e6973b", - "metadata": {}, + "id": "a953dadd", + "metadata": { + "editable": true + }, "source": [ "## The RELU function family\n", "\n", @@ -373,8 +435,10 @@ }, { "cell_type": "markdown", - "id": "9c990bc8", - "metadata": {}, + "id": "92250bbb", + "metadata": { + "editable": true + }, "source": [ "## ELU function\n", "\n", @@ -385,8 +449,10 @@ }, { "cell_type": "markdown", - "id": "9b7760f9", - "metadata": {}, + "id": "fc882545", + "metadata": { + "editable": true + }, "source": [ "$$\n", "ELU(z) = \\left\\{\\begin{array}{cc} \\alpha\\left( \\exp{(z)}-1\\right) & z < 0,\\\\ z & z \\ge 0.\\end{array}\\right.\n", @@ -395,8 +461,10 @@ }, { "cell_type": "markdown", - "id": "47927a82", - "metadata": {}, + "id": "95f21fa0", + "metadata": { + "editable": true + }, "source": [ "## Which activation function should we use?\n", "\n", @@ -415,8 +483,10 @@ }, { "cell_type": "markdown", - "id": "52554330", - "metadata": {}, + "id": "967142c9", + "metadata": { + "editable": true + }, "source": [ "## More on activation functions, output layers\n", "\n", @@ -435,8 +505,10 @@ }, { "cell_type": "markdown", - "id": "4b7d3292", - "metadata": {}, + "id": "71f6bb9c", + "metadata": { + "editable": true + }, "source": [ "## Setting up a Multi-layer perceptron model for classification\n", "\n", @@ -461,8 +533,10 @@ }, { "cell_type": "markdown", - "id": "38715b52", - "metadata": {}, + "id": "851bbd94", + "metadata": { + "editable": true + }, "source": [ "$$\n", "P(y = 0 \\mid \\boldsymbol{x}, \\boldsymbol{\\theta}) = \\frac{1}{1 + \\exp{(- \\boldsymbol{x}})} ,\n", @@ -471,16 +545,20 @@ }, { "cell_type": "markdown", - "id": "dcc0cbfa", - "metadata": {}, + "id": "a098cd16", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "c686ecf6", - "metadata": {}, + "id": "0dbd8000", + "metadata": { + "editable": true + }, "source": [ "$$\n", "P(y = 1 \\mid \\boldsymbol{x}, \\boldsymbol{\\theta}) = 1 - P(y = 0 \\mid \\boldsymbol{x}, \\boldsymbol{\\theta}) ,\n", @@ -489,8 +567,10 @@ }, { "cell_type": "markdown", - "id": "79a8b207", - "metadata": {}, + "id": "e9617ec8", + "metadata": { + "editable": true + }, "source": [ "where $y \\in \\{0, 1\\}$ and $\\boldsymbol{\\theta}$ represents the weights and biases\n", "of our network." @@ -498,8 +578,10 @@ }, { "cell_type": "markdown", - "id": "3218b074", - "metadata": {}, + "id": "cb915df8", + "metadata": { + "editable": true + }, "source": [ "## Defining the cost function\n", "\n", @@ -508,8 +590,10 @@ }, { "cell_type": "markdown", - "id": "c0e50336", - "metadata": {}, + "id": "30c84a61", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathcal{C}(\\boldsymbol{\\theta}) = - \\ln P(\\mathcal{D} \\mid \\boldsymbol{\\theta}) = - \\sum_{i=1}^n\n", @@ -519,8 +603,10 @@ }, { "cell_type": "markdown", - "id": "79138403", - "metadata": {}, + "id": "531ea906", + "metadata": { + "editable": true + }, "source": [ "This last equality means that we can interpret our *cost* function as a sum over the *loss* function\n", "for each point in the dataset $\\mathcal{L}_i(\\boldsymbol{\\theta})$. \n", @@ -542,8 +628,10 @@ }, { "cell_type": "markdown", - "id": "1b3db35e", - "metadata": {}, + "id": "ceeff3b7", + "metadata": { + "editable": true + }, "source": [ "$$\n", "P(y_{ic} = 1 \\mid \\boldsymbol{x}_i, \\boldsymbol{\\theta}) = \\frac{\\exp{((\\boldsymbol{a}_i^{hidden})^T \\boldsymbol{w}_c)}}\n", @@ -553,8 +641,10 @@ }, { "cell_type": "markdown", - "id": "90626f76", - "metadata": {}, + "id": "c4f2895d", + "metadata": { + "editable": true + }, "source": [ "which reduces to the logistic function in the binary case. \n", "The likelihood of this $C$-class classifier\n", @@ -563,8 +653,10 @@ }, { "cell_type": "markdown", - "id": "596eb444", - "metadata": {}, + "id": "8b732327", + "metadata": { + "editable": true + }, "source": [ "$$\n", "P(\\mathcal{D} \\mid \\boldsymbol{\\theta}) = \\prod_{i=1}^n \\prod_{c=0}^{C-1} [P(y_{ic} = 1)]^{y_{ic}} .\n", @@ -573,16 +665,20 @@ }, { "cell_type": "markdown", - "id": "5529d979", - "metadata": {}, + "id": "09e2b21e", + "metadata": { + "editable": true + }, "source": [ "Again we take the negative log-likelihood to define our cost function:" ] }, { "cell_type": "markdown", - "id": "f7d2523f", - "metadata": {}, + "id": "15ddb771", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathcal{C}(\\boldsymbol{\\theta}) = - \\log{P(\\mathcal{D} \\mid \\boldsymbol{\\theta})}.\n", @@ -591,8 +687,10 @@ }, { "cell_type": "markdown", - "id": "7d8d61f1", - "metadata": {}, + "id": "a18f5a20", + "metadata": { + "editable": true + }, "source": [ "See the logistic regression lectures for a full definition of the cost function.\n", "\n", @@ -601,8 +699,10 @@ }, { "cell_type": "markdown", - "id": "9751fb82", - "metadata": {}, + "id": "2c462e09", + "metadata": { + "editable": true + }, "source": [ "## Example: binary classification problem\n", "\n", @@ -611,8 +711,10 @@ }, { "cell_type": "markdown", - "id": "3e6587c7", - "metadata": {}, + "id": "77efe900", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathcal{C}(\\boldsymbol{\\beta}) = - \\sum_{i=1}^n \\left(y_i\\log{p(y_i \\vert x_i,\\boldsymbol{\\beta})}+(1-y_i)\\log{1-p(y_i \\vert x_i,\\boldsymbol{\\beta})}\\right),\n", @@ -621,16 +723,20 @@ }, { "cell_type": "markdown", - "id": "39bb5ba4", - "metadata": {}, + "id": "983cdcf0", + "metadata": { + "editable": true + }, "source": [ "where we had defined the logistic (sigmoid) function" ] }, { "cell_type": "markdown", - "id": "fcb1f3d9", - "metadata": {}, + "id": "53e7831f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "p(y_i =1\\vert x_i,\\boldsymbol{\\beta})=\\frac{\\exp{(\\beta_0+\\beta_1 x_i)}}{1+\\exp{(\\beta_0+\\beta_1 x_i)}},\n", @@ -639,16 +745,20 @@ }, { "cell_type": "markdown", - "id": "20fc130a", - "metadata": {}, + "id": "0bff0839", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "66c7c0e9", - "metadata": {}, + "id": "083d4cb3", + "metadata": { + "editable": true + }, "source": [ "$$\n", "p(y_i =0\\vert x_i,\\boldsymbol{\\beta})=1-p(y_i =1\\vert x_i,\\boldsymbol{\\beta}).\n", @@ -657,8 +767,10 @@ }, { "cell_type": "markdown", - "id": "e757c738", - "metadata": {}, + "id": "45a8eb10", + "metadata": { + "editable": true + }, "source": [ "The parameters $\\boldsymbol{\\beta}$ were defined using a minimization method like gradient descent or Newton-Raphson's method. \n", "\n", @@ -668,8 +780,10 @@ }, { "cell_type": "markdown", - "id": "d048e79e", - "metadata": {}, + "id": "53e02290", + "metadata": { + "editable": true + }, "source": [ "$$\n", "a_i^l = y_i = \\frac{\\exp{(z_i^l)}}{1+\\exp{(z_i^l)}},\n", @@ -678,16 +792,20 @@ }, { "cell_type": "markdown", - "id": "b97827e7", - "metadata": {}, + "id": "f7d729c6", + "metadata": { + "editable": true + }, "source": [ "with" ] }, { "cell_type": "markdown", - "id": "c11528eb", - "metadata": {}, + "id": "5cc4756b", + "metadata": { + "editable": true + }, "source": [ "$$\n", "z_i^l = \\sum_{j}w_{ij}^l a_j^{l-1}+b_i^l,\n", @@ -696,8 +814,10 @@ }, { "cell_type": "markdown", - "id": "94839c96", - "metadata": {}, + "id": "c6aec342", + "metadata": { + "editable": true + }, "source": [ "where the superscript $l-1$ indicates that these are the outputs from layer $l-1$.\n", "Our cost function at the final layer $l=L$ is now" @@ -705,8 +825,10 @@ }, { "cell_type": "markdown", - "id": "694596c9", - "metadata": {}, + "id": "46531c02", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathcal{C}(\\boldsymbol{W}) = - \\sum_{i=1}^n \\left(t_i\\log{a_i^L}+(1-t_i)\\log{(1-a_i^L)}\\right),\n", @@ -715,16 +837,20 @@ }, { "cell_type": "markdown", - "id": "b4c8232e", - "metadata": {}, + "id": "b015445b", + "metadata": { + "editable": true + }, "source": [ "where we have defined the targets $t_i$. The derivatives of the cost function with respect to the output $a_i^L$ are then easily calculated and we get" ] }, { "cell_type": "markdown", - "id": "9d611a08", - "metadata": {}, + "id": "7c277ac8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial \\mathcal{C}(\\boldsymbol{W})}{\\partial a_i^L} = \\frac{a_i^L-t_i}{a_i^L(1-a_i^L)}.\n", @@ -733,16 +859,20 @@ }, { "cell_type": "markdown", - "id": "cb7847ef", - "metadata": {}, + "id": "df80557d", + "metadata": { + "editable": true + }, "source": [ "In case we use another activation function than the logistic one, we need to evaluate other derivatives." ] }, { "cell_type": "markdown", - "id": "cd5be179", - "metadata": {}, + "id": "43a1c6e0", + "metadata": { + "editable": true + }, "source": [ "## The Softmax function\n", "In case we employ the more general case given by the Softmax equation, we need to evaluate the derivative of the activation function with respect to the activation $z_i^l$, that is we need" @@ -750,8 +880,10 @@ }, { "cell_type": "markdown", - "id": "96b7fe79", - "metadata": {}, + "id": "ab0b6274", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial f(z_i^l)}{\\partial w_{jk}^l} =\n", @@ -761,16 +893,20 @@ }, { "cell_type": "markdown", - "id": "1040c590", - "metadata": {}, + "id": "dbf62557", + "metadata": { + "editable": true + }, "source": [ "For the Softmax function we have" ] }, { "cell_type": "markdown", - "id": "3162bd5e", - "metadata": {}, + "id": "be9d40ba", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f(z_i^l) = \\frac{\\exp{(z_i^l)}}{\\sum_{m=1}^K\\exp{(z_m^l)}}.\n", @@ -779,16 +915,20 @@ }, { "cell_type": "markdown", - "id": "b7f2b7b8", - "metadata": {}, + "id": "3c744f52", + "metadata": { + "editable": true + }, "source": [ "Its derivative with respect to $z_j^l$ gives" ] }, { "cell_type": "markdown", - "id": "735fd9c0", - "metadata": {}, + "id": "6148d812", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial f(z_i^l)}{\\partial z_j^l}= f(z_i^l)\\left(\\delta_{ij}-f(z_j^l)\\right),\n", @@ -797,16 +937,20 @@ }, { "cell_type": "markdown", - "id": "c3cbbb04", - "metadata": {}, + "id": "445f8ba9", + "metadata": { + "editable": true + }, "source": [ "which in case of the simply binary model reduces to having $i=j$." ] }, { "cell_type": "markdown", - "id": "6462edbb", - "metadata": {}, + "id": "5ec53511", + "metadata": { + "editable": true + }, "source": [ "## Developing a code for doing neural networks with back propagation\n", "\n", @@ -827,8 +971,10 @@ }, { "cell_type": "markdown", - "id": "7915dc1d", - "metadata": {}, + "id": "6ae556bd", + "metadata": { + "editable": true + }, "source": [ "## Collect and pre-process data\n", "\n", @@ -875,8 +1021,11 @@ { "cell_type": "code", "execution_count": 1, - "id": "eb736d79", - "metadata": {}, + "id": "ec8c7e4e", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "%matplotlib inline\n", @@ -927,8 +1076,10 @@ }, { "cell_type": "markdown", - "id": "01fcb50b", - "metadata": {}, + "id": "1a2589fe", + "metadata": { + "editable": true + }, "source": [ "## Train and test datasets\n", "\n", @@ -946,8 +1097,11 @@ { "cell_type": "code", "execution_count": 2, - "id": "e6b7f2e3", - "metadata": {}, + "id": "d83205fa", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", @@ -981,8 +1135,10 @@ }, { "cell_type": "markdown", - "id": "1ca032f8", - "metadata": {}, + "id": "a4fb72a2", + "metadata": { + "editable": true + }, "source": [ "## Define model and architecture\n", "\n", @@ -1023,8 +1179,10 @@ }, { "cell_type": "markdown", - "id": "6e4e1e18", - "metadata": {}, + "id": "57b47066", + "metadata": { + "editable": true + }, "source": [ "## Layers\n", "\n", @@ -1061,8 +1219,10 @@ }, { "cell_type": "markdown", - "id": "ab0ee6d8", - "metadata": {}, + "id": "892f5e39", + "metadata": { + "editable": true + }, "source": [ "## Weights and biases\n", "\n", @@ -1080,8 +1240,11 @@ { "cell_type": "code", "execution_count": 3, - "id": "15ad1599", - "metadata": {}, + "id": "de56a384", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# building our neural network\n", @@ -1103,8 +1266,10 @@ }, { "cell_type": "markdown", - "id": "3efd576a", - "metadata": {}, + "id": "771f0879", + "metadata": { + "editable": true + }, "source": [ "## Feed-forward pass\n", "\n", @@ -1129,8 +1294,10 @@ }, { "cell_type": "markdown", - "id": "715c5d46", - "metadata": {}, + "id": "7a648705", + "metadata": { + "editable": true + }, "source": [ "## Matrix multiplications\n", "\n", @@ -1164,8 +1331,11 @@ { "cell_type": "code", "execution_count": 4, - "id": "ecaa833e", - "metadata": {}, + "id": "36949419", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# setup the feed-forward pass, subscript h = hidden layer\n", @@ -1207,8 +1377,10 @@ }, { "cell_type": "markdown", - "id": "c0319e65", - "metadata": {}, + "id": "9418645f", + "metadata": { + "editable": true + }, "source": [ "## Choose cost function and optimizer\n", "\n", @@ -1236,8 +1408,10 @@ }, { "cell_type": "markdown", - "id": "98c6696e", - "metadata": {}, + "id": "829bd379", + "metadata": { + "editable": true + }, "source": [ "## Optimizing the cost function\n", "\n", @@ -1272,8 +1446,10 @@ }, { "cell_type": "markdown", - "id": "b5d1145a", - "metadata": {}, + "id": "0b4f4da4", + "metadata": { + "editable": true + }, "source": [ "## Regularization\n", "\n", @@ -1304,8 +1480,10 @@ }, { "cell_type": "markdown", - "id": "b73247fd", - "metadata": {}, + "id": "0ca700a3", + "metadata": { + "editable": true + }, "source": [ "## Matrix multiplication\n", "\n", @@ -1343,8 +1521,11 @@ { "cell_type": "code", "execution_count": 5, - "id": "3d61438f", - "metadata": {}, + "id": "08360580", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# to categorical turns our integer vector into a onehot representation\n", @@ -1419,8 +1600,10 @@ }, { "cell_type": "markdown", - "id": "6a20a495", - "metadata": {}, + "id": "cc8b5d31", + "metadata": { + "editable": true + }, "source": [ "## Improving performance\n", "\n", @@ -1438,8 +1621,10 @@ }, { "cell_type": "markdown", - "id": "717b63a1", - "metadata": {}, + "id": "32ba0d69", + "metadata": { + "editable": true + }, "source": [ "## Full object-oriented implementation\n", "\n", @@ -1450,8 +1635,11 @@ { "cell_type": "code", "execution_count": 6, - "id": "9d6c9929", - "metadata": {}, + "id": "240e3453", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "class NeuralNetwork:\n", @@ -1557,8 +1745,10 @@ }, { "cell_type": "markdown", - "id": "e70667c5", - "metadata": {}, + "id": "6f767a02", + "metadata": { + "editable": true + }, "source": [ "## Evaluate model performance on test data\n", "\n", @@ -1574,8 +1764,11 @@ { "cell_type": "code", "execution_count": 7, - "id": "083dfe41", - "metadata": {}, + "id": "6fe65567", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "epochs = 100\n", @@ -1598,8 +1791,10 @@ }, { "cell_type": "markdown", - "id": "bffeadd3", - "metadata": {}, + "id": "030b9ab0", + "metadata": { + "editable": true + }, "source": [ "## Adjust hyperparameters\n", "\n", @@ -1610,8 +1805,11 @@ { "cell_type": "code", "execution_count": 8, - "id": "6513ac74", - "metadata": {}, + "id": "0d506b1b", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "eta_vals = np.logspace(-5, 1, 7)\n", @@ -1638,8 +1836,10 @@ }, { "cell_type": "markdown", - "id": "a6e305f6", - "metadata": {}, + "id": "3d42f513", + "metadata": { + "editable": true + }, "source": [ "## Visualization" ] @@ -1647,8 +1847,11 @@ { "cell_type": "code", "execution_count": 9, - "id": "d920e285", - "metadata": {}, + "id": "f7a563f7", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# visual representation of grid search\n", @@ -1688,8 +1891,10 @@ }, { "cell_type": "markdown", - "id": "0f1aaadd", - "metadata": {}, + "id": "ca1a70a2", + "metadata": { + "editable": true + }, "source": [ "## scikit-learn implementation\n", "\n", @@ -1709,8 +1914,11 @@ { "cell_type": "code", "execution_count": 10, - "id": "156764f3", - "metadata": {}, + "id": "dbbc8b38", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from sklearn.neural_network import MLPClassifier\n", @@ -1733,8 +1941,10 @@ }, { "cell_type": "markdown", - "id": "2e5fec48", - "metadata": {}, + "id": "03187c8b", + "metadata": { + "editable": true + }, "source": [ "## Visualization" ] @@ -1742,8 +1952,11 @@ { "cell_type": "code", "execution_count": 11, - "id": "ace47e70", - "metadata": {}, + "id": "13526c92", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# optional\n", @@ -1784,8 +1997,10 @@ }, { "cell_type": "markdown", - "id": "9d327f9b", - "metadata": {}, + "id": "4b8b9f7b", + "metadata": { + "editable": true + }, "source": [ "## Building neural networks in Tensorflow and Keras\n", "\n", @@ -1800,8 +2015,10 @@ }, { "cell_type": "markdown", - "id": "2280cbce", - "metadata": {}, + "id": "286ce732", + "metadata": { + "editable": true + }, "source": [ "## Tensorflow\n", "\n", @@ -1833,8 +2050,11 @@ { "cell_type": "code", "execution_count": 12, - "id": "dff81198", - "metadata": {}, + "id": "07721723", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "pip3 install tensorflow" @@ -1842,8 +2062,10 @@ }, { "cell_type": "markdown", - "id": "e9bf9cac", - "metadata": {}, + "id": "0a9abdfe", + "metadata": { + "editable": true + }, "source": [ "and/or if you use **anaconda**, just write (or install from the graphical user interface)\n", "(current release of CPU-only TensorFlow)" @@ -1852,8 +2074,11 @@ { "cell_type": "code", "execution_count": 13, - "id": "1ccb8dba", - "metadata": {}, + "id": "ba43c53c", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "conda create -n tf tensorflow\n", @@ -1862,8 +2087,10 @@ }, { "cell_type": "markdown", - "id": "efc3f4e2", - "metadata": {}, + "id": "5fad471e", + "metadata": { + "editable": true + }, "source": [ "To install the current release of GPU TensorFlow" ] @@ -1871,8 +2098,11 @@ { "cell_type": "code", "execution_count": 14, - "id": "01556b01", - "metadata": {}, + "id": "dd745d96", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "conda create -n tf-gpu tensorflow-gpu\n", @@ -1881,8 +2111,10 @@ }, { "cell_type": "markdown", - "id": "b900f015", - "metadata": {}, + "id": "95529f77", + "metadata": { + "editable": true + }, "source": [ "## Using Keras\n", "\n", @@ -1894,8 +2126,11 @@ { "cell_type": "code", "execution_count": 15, - "id": "3fd1f12f", - "metadata": {}, + "id": "43f73d04", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "conda install keras" @@ -1903,8 +2138,10 @@ }, { "cell_type": "markdown", - "id": "20ec35c5", - "metadata": {}, + "id": "7bba578e", + "metadata": { + "editable": true + }, "source": [ "You can look up the [instructions here](https://keras.io/) for more information.\n", "\n", @@ -1913,8 +2150,10 @@ }, { "cell_type": "markdown", - "id": "249e56f7", - "metadata": {}, + "id": "40f5b8c5", + "metadata": { + "editable": true + }, "source": [ "## Collect and pre-process data\n", "\n", @@ -1924,8 +2163,11 @@ { "cell_type": "code", "execution_count": 16, - "id": "0ff66efc", - "metadata": {}, + "id": "8d856ac1", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# import necessary packages\n", @@ -1976,8 +2218,11 @@ { "cell_type": "code", "execution_count": 17, - "id": "2806c62b", - "metadata": {}, + "id": "7eed3ecb", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from tensorflow.keras.layers import Input\n", @@ -2002,8 +2247,11 @@ { "cell_type": "code", "execution_count": 18, - "id": "3bf4c3f6", - "metadata": {}, + "id": "b85a7170", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\n", @@ -2020,7 +2268,7 @@ " model.add(Dense(n_neurons_layer2, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd)))\n", " model.add(Dense(n_categories, activation='softmax'))\n", " \n", - " sgd = optimizers.SGD(lr=eta)\n", + " sgd = optimizers.SGD(learning_rate=eta)\n", " model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])\n", " \n", " return model" @@ -2029,8 +2277,11 @@ { "cell_type": "code", "execution_count": 19, - "id": "81a46485", - "metadata": {}, + "id": "488b3cba", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "DNN_keras = np.zeros((len(eta_vals), len(lmbd_vals)), dtype=object)\n", @@ -2053,8 +2304,11 @@ { "cell_type": "code", "execution_count": 20, - "id": "6e619c14", - "metadata": {}, + "id": "3909625f", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# optional\n", @@ -2092,8 +2346,10 @@ }, { "cell_type": "markdown", - "id": "c303d97b", - "metadata": {}, + "id": "5d5033d8", + "metadata": { + "editable": true + }, "source": [ "## The Breast Cancer Data, now with Keras" ] @@ -2101,8 +2357,11 @@ { "cell_type": "code", "execution_count": 21, - "id": "03a57bfd", - "metadata": {}, + "id": "d6491f10", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\n", @@ -2215,7 +2474,7 @@ " else: #Subsequent layers are capable of automatic shape inferencing\n", " model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda)))\n", " model.add(Dense(2,activation='softmax')) #2 outputs - ordered and disordered (softmax for prob)\n", - " sgd=optimizers.SGD(lr=eta)\n", + " sgd=optimizers.SGD(learning_rate=eta)\n", " model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])\n", " return model\n", "\n", @@ -2275,8 +2534,10 @@ }, { "cell_type": "markdown", - "id": "f9ad1085", - "metadata": {}, + "id": "ca360d19", + "metadata": { + "editable": true + }, "source": [ "## Building a neural network code\n", "\n", @@ -2292,8 +2553,10 @@ }, { "cell_type": "markdown", - "id": "cf4f3c14", - "metadata": {}, + "id": "bd4df429", + "metadata": { + "editable": true + }, "source": [ "### Learning rate methods\n", "\n", @@ -2312,8 +2575,11 @@ { "cell_type": "code", "execution_count": 22, - "id": "467ea7a2", - "metadata": {}, + "id": "112ed568", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -2450,8 +2716,10 @@ }, { "cell_type": "markdown", - "id": "6af64d5a", - "metadata": {}, + "id": "84a8ae95", + "metadata": { + "editable": true + }, "source": [ "### Usage of the above learning rate schedulers\n", "\n", @@ -2464,8 +2732,11 @@ { "cell_type": "code", "execution_count": 23, - "id": "290e0d00", - "metadata": {}, + "id": "dee9678f", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "momentum_scheduler = Momentum(eta=1e-3, momentum=0.9)\n", @@ -2474,8 +2745,10 @@ }, { "cell_type": "markdown", - "id": "d1f43cc5", - "metadata": {}, + "id": "36918ec3", + "metadata": { + "editable": true + }, "source": [ "Here is a small example for how a segment of code using schedulers\n", "could look. Switching out the schedulers is simple." @@ -2484,8 +2757,11 @@ { "cell_type": "code", "execution_count": 24, - "id": "0def5cec", - "metadata": {}, + "id": "a7ba41e5", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "weights = np.ones((3,3))\n", @@ -2503,8 +2779,10 @@ }, { "cell_type": "markdown", - "id": "9b27dcd7", - "metadata": {}, + "id": "08351356", + "metadata": { + "editable": true + }, "source": [ "### Cost functions\n", "\n", @@ -2517,8 +2795,11 @@ { "cell_type": "code", "execution_count": 25, - "id": "4217839c", - "metadata": {}, + "id": "37e20b2a", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -2552,8 +2833,10 @@ }, { "cell_type": "markdown", - "id": "ef5f9fe9", - "metadata": {}, + "id": "8b85b5a2", + "metadata": { + "editable": true + }, "source": [ "Below we give a short example of how these cost function may be used\n", "to obtain results if you wish to test them out on your own using\n", @@ -2563,8 +2846,11 @@ { "cell_type": "code", "execution_count": 26, - "id": "305e1479", - "metadata": {}, + "id": "f025e297", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from autograd import grad\n", @@ -2581,8 +2867,10 @@ }, { "cell_type": "markdown", - "id": "1499b193", - "metadata": {}, + "id": "3006ef10", + "metadata": { + "editable": true + }, "source": [ "### Activation functions\n", "\n", @@ -2595,8 +2883,11 @@ { "cell_type": "code", "execution_count": 27, - "id": "e7b59b25", - "metadata": {}, + "id": "a6b8cc95", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -2650,8 +2941,10 @@ }, { "cell_type": "markdown", - "id": "57599619", - "metadata": {}, + "id": "c5e3a652", + "metadata": { + "editable": true + }, "source": [ "Below follows a short demonstration of how to use an activation\n", "function. The derivative of the activation function will be important\n", @@ -2663,8 +2956,11 @@ { "cell_type": "code", "execution_count": 28, - "id": "5ee68acd", - "metadata": {}, + "id": "650379b6", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "z = np.array([[4, 5, 6]]).T\n", @@ -2681,8 +2977,10 @@ }, { "cell_type": "markdown", - "id": "2d3295d9", - "metadata": {}, + "id": "a4b048f2", + "metadata": { + "editable": true + }, "source": [ "### The Neural Network\n", "\n", @@ -2703,8 +3001,11 @@ { "cell_type": "code", "execution_count": 29, - "id": "c4cfc50a", - "metadata": {}, + "id": "6b0c4506", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import math\n", @@ -3172,8 +3473,10 @@ }, { "cell_type": "markdown", - "id": "2e6e81c1", - "metadata": {}, + "id": "24ce7b3e", + "metadata": { + "editable": true + }, "source": [ "Before we make a model, we will quickly generate a dataset we can use\n", "for our linear regression problem as shown below" @@ -3182,8 +3485,11 @@ { "cell_type": "code", "execution_count": 30, - "id": "0d4f1ff8", - "metadata": {}, + "id": "5891af38", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -3223,8 +3529,10 @@ }, { "cell_type": "markdown", - "id": "a8f1940f", - "metadata": {}, + "id": "772f6025", + "metadata": { + "editable": true + }, "source": [ "Now that we have our dataset ready for the regression, we can create\n", "our regressor. Note that with the seed parameter, we can make sure our\n", @@ -3237,8 +3545,11 @@ { "cell_type": "code", "execution_count": 31, - "id": "ff6bd29b", - "metadata": {}, + "id": "ce9c6ccd", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "input_nodes = X_train.shape[1]\n", @@ -3249,8 +3560,10 @@ }, { "cell_type": "markdown", - "id": "a7b906df", - "metadata": {}, + "id": "46b49fd9", + "metadata": { + "editable": true + }, "source": [ "We then fit our model with our training data using the scheduler of our choice." ] @@ -3258,8 +3571,11 @@ { "cell_type": "code", "execution_count": 32, - "id": "db7a5bf2", - "metadata": {}, + "id": "a053fb83", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "linear_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", @@ -3270,8 +3586,10 @@ }, { "cell_type": "markdown", - "id": "508c351b", - "metadata": {}, + "id": "d879966a", + "metadata": { + "editable": true + }, "source": [ "Due to the progress bar we can see the MSE (train_error) throughout\n", "the FFNN's training. Note that the fit() function has some optional\n", @@ -3284,8 +3602,11 @@ { "cell_type": "code", "execution_count": 33, - "id": "6b0562b2", - "metadata": {}, + "id": "5b74ce75", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "linear_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", @@ -3295,8 +3616,10 @@ }, { "cell_type": "markdown", - "id": "f03172d9", - "metadata": {}, + "id": "87efa6e2", + "metadata": { + "editable": true + }, "source": [ "We see that given more epochs to train on, the regressor reaches a lower MSE.\n", "\n", @@ -3308,8 +3631,11 @@ { "cell_type": "code", "execution_count": 34, - "id": "26d1d1c3", - "metadata": {}, + "id": "50443d01", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from sklearn.datasets import load_breast_cancer\n", @@ -3331,8 +3657,11 @@ { "cell_type": "code", "execution_count": 35, - "id": "6a78c633", - "metadata": {}, + "id": "5f967df0", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "input_nodes = X_train.shape[1]\n", @@ -3343,8 +3672,10 @@ }, { "cell_type": "markdown", - "id": "5feaced2", - "metadata": {}, + "id": "f0d2f379", + "metadata": { + "editable": true + }, "source": [ "We will now make use of our validation data by passing it into our fit function as a keyword argument" ] @@ -3352,8 +3683,11 @@ { "cell_type": "code", "execution_count": 36, - "id": "6a9b1538", - "metadata": {}, + "id": "89c8755b", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "logistic_regression.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", @@ -3364,8 +3698,10 @@ }, { "cell_type": "markdown", - "id": "90c7f8f3", - "metadata": {}, + "id": "0d3c3b9b", + "metadata": { + "editable": true + }, "source": [ "Finally, we will create a neural network with 2 hidden layers with activation functions." ] @@ -3373,8 +3709,11 @@ { "cell_type": "code", "execution_count": 37, - "id": "58954140", - "metadata": {}, + "id": "6f000ad2", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "input_nodes = X_train.shape[1]\n", @@ -3390,8 +3729,11 @@ { "cell_type": "code", "execution_count": 38, - "id": "89536ca9", - "metadata": {}, + "id": "97e0a5a6", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "neural_network.reset_weights() # reset weights such that previous runs or reruns don't affect the weights\n", @@ -3402,8 +3744,10 @@ }, { "cell_type": "markdown", - "id": "803f4791", - "metadata": {}, + "id": "4ba64928", + "metadata": { + "editable": true + }, "source": [ "### Multiclass classification\n", "\n", @@ -3415,8 +3759,11 @@ { "cell_type": "code", "execution_count": 39, - "id": "50405951", - "metadata": {}, + "id": "38060eca", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from sklearn.datasets import load_digits\n", @@ -3449,8 +3796,10 @@ }, { "cell_type": "markdown", - "id": "d906edf4", - "metadata": {}, + "id": "28cbf454", + "metadata": { + "editable": true + }, "source": [ "## Testing the XOR gate and other gates\n", "\n", @@ -3460,8 +3809,11 @@ { "cell_type": "code", "execution_count": 40, - "id": "414ee306", - "metadata": {}, + "id": "67b20164", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "X = np.array([ [0, 0], [0, 1], [1, 0],[1, 1]],dtype=np.float64)\n", @@ -3480,16 +3832,20 @@ }, { "cell_type": "markdown", - "id": "982d1747", - "metadata": {}, + "id": "f0e3c9f6", + "metadata": { + "editable": true + }, "source": [ "Not bad, but the results depend strongly on the learning reate. Try different learning rates." ] }, { "cell_type": "markdown", - "id": "616eae3e", - "metadata": {}, + "id": "7efdec2b", + "metadata": { + "editable": true + }, "source": [ "## Solving ODEs with Deep Learning\n", "\n", @@ -3513,8 +3869,10 @@ }, { "cell_type": "markdown", - "id": "43fb869c", - "metadata": {}, + "id": "2ec15c81", + "metadata": { + "editable": true + }, "source": [ "## Ordinary Differential Equations\n", "\n", @@ -3525,8 +3883,10 @@ }, { "cell_type": "markdown", - "id": "4855aef1", - "metadata": {}, + "id": "da662c57", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3540,8 +3900,10 @@ }, { "cell_type": "markdown", - "id": "d31c26fc", - "metadata": {}, + "id": "0f9a508c", + "metadata": { + "editable": true + }, "source": [ "where $g(x)$ is the function to find, and $g^{(n)}(x)$ is the $n$-th derivative of $g(x)$.\n", "\n", @@ -3554,8 +3916,10 @@ }, { "cell_type": "markdown", - "id": "08d629ec", - "metadata": {}, + "id": "7f8f2c49", + "metadata": { + "editable": true + }, "source": [ "## The trial solution\n", "\n", @@ -3564,8 +3928,10 @@ }, { "cell_type": "markdown", - "id": "b2cfd7bd", - "metadata": {}, + "id": "851b1e9d", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3580,8 +3946,10 @@ }, { "cell_type": "markdown", - "id": "f4cd106f", - "metadata": {}, + "id": "e1375451", + "metadata": { + "editable": true + }, "source": [ "where $h_1(x)$ is a function that makes $g_t(x)$ satisfy a given set\n", "of conditions, $N(x,P)$ a neural network with weights and biases\n", @@ -3599,8 +3967,10 @@ }, { "cell_type": "markdown", - "id": "74da655f", - "metadata": {}, + "id": "8ee29eac", + "metadata": { + "editable": true + }, "source": [ "## Minimization process\n", "\n", @@ -3614,8 +3984,10 @@ }, { "cell_type": "markdown", - "id": "efe0ec18", - "metadata": {}, + "id": "a07434a0", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C\\left(x, P\\right) = \\big(f\\left(x, \\, g(x), \\, g'(x), \\, g''(x), \\, \\dots \\, , \\, g^{(n)}(x)\\right)\\big)^2\n", @@ -3624,8 +3996,10 @@ }, { "cell_type": "markdown", - "id": "f9c1643a", - "metadata": {}, + "id": "31045ebd", + "metadata": { + "editable": true + }, "source": [ "If $N$ inputs are given as a vector $\\boldsymbol{x}$ with elements $x_i$ for $i = 1,\\dots,N$,\n", "the cost function becomes" @@ -3633,8 +4007,10 @@ }, { "cell_type": "markdown", - "id": "9b900e26", - "metadata": {}, + "id": "b833234d", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3648,8 +4024,10 @@ }, { "cell_type": "markdown", - "id": "82e8e341", - "metadata": {}, + "id": "942d77d7", + "metadata": { + "editable": true + }, "source": [ "The neural net should then find the parameters $P$ that minimizes the cost function in\n", "([3](#cost)) for a set of $N$ training samples $x_i$." @@ -3657,8 +4035,10 @@ }, { "cell_type": "markdown", - "id": "b89f2f97", - "metadata": {}, + "id": "87898b0c", + "metadata": { + "editable": true + }, "source": [ "## Minimizing the cost function using gradient descent and automatic differentiation\n", "\n", @@ -3671,8 +4051,10 @@ }, { "cell_type": "markdown", - "id": "cc38c37f", - "metadata": {}, + "id": "d069ecf7", + "metadata": { + "editable": true + }, "source": [ "## Example: Exponential decay\n", "\n", @@ -3681,8 +4063,10 @@ }, { "cell_type": "markdown", - "id": "3c5f0410", - "metadata": {}, + "id": "834f15b8", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3696,8 +4080,10 @@ }, { "cell_type": "markdown", - "id": "4531d4a4", - "metadata": {}, + "id": "139fc20d", + "metadata": { + "editable": true + }, "source": [ "with $g(0) = g_0$ for some chosen initial value $g_0$.\n", "\n", @@ -3706,8 +4092,10 @@ }, { "cell_type": "markdown", - "id": "d580caea", - "metadata": {}, + "id": "d014cf33", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3722,16 +4110,20 @@ }, { "cell_type": "markdown", - "id": "6f384ec5", - "metadata": {}, + "id": "2978822e", + "metadata": { + "editable": true + }, "source": [ "Having an analytical solution at hand, it is possible to use it to compare how well a neural network finds a solution of ([4](#solve_expdec))." ] }, { "cell_type": "markdown", - "id": "e650af49", - "metadata": {}, + "id": "0826e8bf", + "metadata": { + "editable": true + }, "source": [ "## The function to solve for\n", "\n", @@ -3740,8 +4132,10 @@ }, { "cell_type": "markdown", - "id": "68e3a73b", - "metadata": {}, + "id": "60fcd2a1", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3755,8 +4149,10 @@ }, { "cell_type": "markdown", - "id": "61f7dce9", - "metadata": {}, + "id": "65810a45", + "metadata": { + "editable": true + }, "source": [ "where $g(0) = g_0$ with $\\gamma$ and $g_0$ being some chosen values.\n", "\n", @@ -3765,8 +4161,10 @@ }, { "cell_type": "markdown", - "id": "47e985ed", - "metadata": {}, + "id": "f70cb891", + "metadata": { + "editable": true + }, "source": [ "## The trial solution\n", "To begin with, a trial solution $g_t(t)$ must be chosen. A general trial solution for ordinary differential equations could be" @@ -3774,8 +4172,10 @@ }, { "cell_type": "markdown", - "id": "b5787755", - "metadata": {}, + "id": "e2bad0ef", + "metadata": { + "editable": true + }, "source": [ "$$\n", "g_t(x, P) = h_1(x) + h_2(x, N(x, P))\n", @@ -3784,16 +4184,20 @@ }, { "cell_type": "markdown", - "id": "efd0d663", - "metadata": {}, + "id": "2af084db", + "metadata": { + "editable": true + }, "source": [ "with $h_1(x)$ ensuring that $g_t(x)$ satisfies some conditions and $h_2(x,N(x, P))$ an expression involving $x$ and the output from the neural network $N(x,P)$ with $P $ being the collection of the weights and biases for each layer. For now, it is assumed that the network consists of one input layer, one hidden layer, and one output layer." ] }, { "cell_type": "markdown", - "id": "de4cfa12", - "metadata": {}, + "id": "40057844", + "metadata": { + "editable": true + }, "source": [ "## Setup of Network\n", "\n", @@ -3810,8 +4214,10 @@ }, { "cell_type": "markdown", - "id": "4de8eeb1", - "metadata": {}, + "id": "cbba9aed", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3825,8 +4231,10 @@ }, { "cell_type": "markdown", - "id": "c3076a5d", - "metadata": {}, + "id": "136ee921", + "metadata": { + "editable": true + }, "source": [ "## Reformulating the problem\n", "\n", @@ -3842,8 +4250,10 @@ }, { "cell_type": "markdown", - "id": "00459420", - "metadata": {}, + "id": "67949baa", + "metadata": { + "editable": true + }, "source": [ "$$\n", "g_t(x, P) = g_0 + x \\cdot N(x, P)\n", @@ -3852,16 +4262,20 @@ }, { "cell_type": "markdown", - "id": "493318e4", - "metadata": {}, + "id": "4a701182", + "metadata": { + "editable": true + }, "source": [ "has been chosen such that it already solves the condition $g(0) = g_0$. What remains, is to find $P$ such that" ] }, { "cell_type": "markdown", - "id": "b1cd6594", - "metadata": {}, + "id": "556d52ba", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3875,16 +4289,20 @@ }, { "cell_type": "markdown", - "id": "bf2aa493", - "metadata": {}, + "id": "908fcca0", + "metadata": { + "editable": true + }, "source": [ "is fulfilled as *best as possible*." ] }, { "cell_type": "markdown", - "id": "5eda172b", - "metadata": {}, + "id": "cfbc4378", + "metadata": { + "editable": true + }, "source": [ "## More technicalities\n", "\n", @@ -3897,8 +4315,10 @@ }, { "cell_type": "markdown", - "id": "2bfc8161", - "metadata": {}, + "id": "f781283d", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\min_{P}\\Big\\{ \\big(g_t'(x, P) - ( -\\gamma g_t(x, P) \\big)^2 \\Big\\}\n", @@ -3907,8 +4327,10 @@ }, { "cell_type": "markdown", - "id": "9af6a125", - "metadata": {}, + "id": "4c8ec707", + "metadata": { + "editable": true + }, "source": [ "(the notation $\\min_{P}\\{ f(x, P) \\}$ means that we desire to find $P$ that yields the minimum of $f(x, P)$)\n", "\n", @@ -3917,8 +4339,10 @@ }, { "cell_type": "markdown", - "id": "bea03f10", - "metadata": {}, + "id": "8487f782", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\min_{P_{\\text{hidden} }, \\ P_{\\text{output} }}\\Big\\{ \\big(g_t'(x, \\{ P_{\\text{hidden} }, P_{\\text{output} }\\}) - ( -\\gamma g_t(x, \\{ P_{\\text{hidden} }, P_{\\text{output} }\\}) \\big)^2 \\Big\\}\n", @@ -3927,16 +4351,20 @@ }, { "cell_type": "markdown", - "id": "352fb11e", - "metadata": {}, + "id": "b16f3d82", + "metadata": { + "editable": true + }, "source": [ "for an input value $x$." ] }, { "cell_type": "markdown", - "id": "98d5219e", - "metadata": {}, + "id": "22a46cf9", + "metadata": { + "editable": true + }, "source": [ "## More details\n", "\n", @@ -3945,8 +4373,10 @@ }, { "cell_type": "markdown", - "id": "83d7b299", - "metadata": {}, + "id": "32043e4d", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3960,16 +4390,20 @@ }, { "cell_type": "markdown", - "id": "1ba53d11", - "metadata": {}, + "id": "ed757b9b", + "metadata": { + "editable": true + }, "source": [ "Letting $\\boldsymbol{x}$ be a vector with elements $x_i$ and $C(\\boldsymbol{x}, P) = \\frac{1}{N} \\sum_i \\big(g_t'(x_i, P) - ( -\\gamma g_t(x_i, P) \\big)^2$ denote the cost function, the minimization problem that our network must solve, becomes" ] }, { "cell_type": "markdown", - "id": "3953e778", - "metadata": {}, + "id": "14e578bc", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\min_{P} C(\\boldsymbol{x}, P)\n", @@ -3978,8 +4412,10 @@ }, { "cell_type": "markdown", - "id": "706a1978", - "metadata": {}, + "id": "29f15e0d", + "metadata": { + "editable": true + }, "source": [ "In terms of $P_{\\text{hidden} }$ and $P_{\\text{output} }$, this could also be expressed as\n", "\n", @@ -3990,8 +4426,10 @@ }, { "cell_type": "markdown", - "id": "8ff57d1a", - "metadata": {}, + "id": "7f36927f", + "metadata": { + "editable": true + }, "source": [ "## A possible implementation of a neural network\n", "\n", @@ -4004,8 +4442,10 @@ }, { "cell_type": "markdown", - "id": "fb487d3d", - "metadata": {}, + "id": "3f1a8713", + "metadata": { + "editable": true + }, "source": [ "## Technicalities\n", "\n", @@ -4014,8 +4454,10 @@ }, { "cell_type": "markdown", - "id": "1d6ac6a6", - "metadata": {}, + "id": "669a42f3", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4034,8 +4476,10 @@ }, { "cell_type": "markdown", - "id": "fee86c29", - "metadata": {}, + "id": "b3fe20ff", + "metadata": { + "editable": true + }, "source": [ "## Final technicalities I\n", "\n", @@ -4044,8 +4488,10 @@ }, { "cell_type": "markdown", - "id": "317dd762", - "metadata": {}, + "id": "d5eada96", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4065,8 +4511,10 @@ }, { "cell_type": "markdown", - "id": "4641c716", - "metadata": {}, + "id": "230a03b4", + "metadata": { + "editable": true + }, "source": [ "## Final technicalities II\n", "\n", @@ -4079,8 +4527,10 @@ }, { "cell_type": "markdown", - "id": "a28fd5d3", - "metadata": {}, + "id": "5d217f51", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f(z) = \\frac{1}{1 + \\exp{(-z)}}\n", @@ -4089,8 +4539,10 @@ }, { "cell_type": "markdown", - "id": "d2c6ce13", - "metadata": {}, + "id": "662333b5", + "metadata": { + "editable": true + }, "source": [ "It is possible to use other activations functions for the hidden layer also.\n", "\n", @@ -4111,8 +4563,10 @@ }, { "cell_type": "markdown", - "id": "5bf0304c", - "metadata": {}, + "id": "ede63c89", + "metadata": { + "editable": true + }, "source": [ "## Final technicalities III\n", "\n", @@ -4121,8 +4575,10 @@ }, { "cell_type": "markdown", - "id": "f4ee906f", - "metadata": {}, + "id": "10656448", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4140,8 +4596,10 @@ }, { "cell_type": "markdown", - "id": "454cd32f", - "metadata": {}, + "id": "6d4a14d4", + "metadata": { + "editable": true + }, "source": [ "## Final technicalities IV\n", "\n", @@ -4150,8 +4608,10 @@ }, { "cell_type": "markdown", - "id": "49b52638", - "metadata": {}, + "id": "c256571e", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{z}_{1}^{\\text{output}} =\n", @@ -4167,16 +4627,20 @@ }, { "cell_type": "markdown", - "id": "75d1e0dc", - "metadata": {}, + "id": "bdff1c30", + "metadata": { + "editable": true + }, "source": [ "In this case we seek a continuous range of values since we are approximating a function. This means that after computing $\\boldsymbol{z}_{1}^{\\text{output}}$ the neural network has finished its feed forward step, and $\\boldsymbol{z}_{1}^{\\text{output}}$ is the final output of the network." ] }, { "cell_type": "markdown", - "id": "243508b9", - "metadata": {}, + "id": "f68afcd0", + "metadata": { + "editable": true + }, "source": [ "## Back propagation\n", "\n", @@ -4187,8 +4651,10 @@ }, { "cell_type": "markdown", - "id": "cfaa5264", - "metadata": {}, + "id": "5d895510", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C(\\boldsymbol{x}, P) = \\frac{1}{N} \\sum_i \\big(g_t'(x_i, P) - ( -\\gamma g_t(x_i, P) \\big)^2\n", @@ -4197,8 +4663,10 @@ }, { "cell_type": "markdown", - "id": "188a58e4", - "metadata": {}, + "id": "1293c131", + "metadata": { + "editable": true + }, "source": [ "In order to minimize the cost function, an optimization method must be chosen.\n", "\n", @@ -4207,8 +4675,10 @@ }, { "cell_type": "markdown", - "id": "6f3c7251", - "metadata": {}, + "id": "45946356", + "metadata": { + "editable": true + }, "source": [ "## Gradient descent\n", "\n", @@ -4222,8 +4692,10 @@ }, { "cell_type": "markdown", - "id": "b48662af", - "metadata": {}, + "id": "0d7ffa2d", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{\\omega}_{\\text{new} } = \\boldsymbol{\\omega} - \\lambda \\nabla_{\\boldsymbol{\\omega}} C(\\boldsymbol{x}, \\boldsymbol{\\omega})\n", @@ -4232,8 +4704,10 @@ }, { "cell_type": "markdown", - "id": "46cc1b9b", - "metadata": {}, + "id": "9cc683f4", + "metadata": { + "editable": true + }, "source": [ "for a number of iterations or until $ \\big|\\big| \\boldsymbol{\\omega}_{\\text{new} } - \\boldsymbol{\\omega} \\big|\\big|$ becomes smaller than some given tolerance.\n", "\n", @@ -4252,8 +4726,10 @@ }, { "cell_type": "markdown", - "id": "018877f6", - "metadata": {}, + "id": "c01a7433", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4265,8 +4741,10 @@ }, { "cell_type": "markdown", - "id": "35e348bd", - "metadata": {}, + "id": "5899086f", + "metadata": { + "editable": true + }, "source": [ "## The code for solving the ODE" ] @@ -4274,8 +4752,11 @@ { "cell_type": "code", "execution_count": 41, - "id": "dafe8d25", - "metadata": {}, + "id": "fb60da3a", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -4426,8 +4907,10 @@ }, { "cell_type": "markdown", - "id": "65e4f300", - "metadata": {}, + "id": "f91702fa", + "metadata": { + "editable": true + }, "source": [ "## The network with one input layer, specified number of hidden layers, and one output layer\n", "\n", @@ -4439,8 +4922,11 @@ { "cell_type": "code", "execution_count": 42, - "id": "5ef5f766", - "metadata": {}, + "id": "4dedb3fb", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -4605,8 +5091,10 @@ }, { "cell_type": "markdown", - "id": "c7ad45ef", - "metadata": {}, + "id": "d44f06bd", + "metadata": { + "editable": true + }, "source": [ "## Example: Population growth\n", "\n", @@ -4616,8 +5104,10 @@ }, { "cell_type": "markdown", - "id": "2d1376bc", - "metadata": {}, + "id": "47302c1e", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -4631,8 +5121,10 @@ }, { "cell_type": "markdown", - "id": "1f039785", - "metadata": {}, + "id": "37e45c88", + "metadata": { + "editable": true + }, "source": [ "where $g(t)$ is the population density at time $t$, $\\alpha > 0$ the growth rate and $A > 0$ is the maximum population number in the environment.\n", "Also, at $t = 0$ the population has the size $g(0) = g_0$, where $g_0$ is some chosen constant.\n", @@ -4645,8 +5137,10 @@ }, { "cell_type": "markdown", - "id": "8dfcfb5e", - "metadata": {}, + "id": "51408078", + "metadata": { + "editable": true + }, "source": [ "## Setting up the problem\n", "\n", @@ -4656,8 +5150,10 @@ }, { "cell_type": "markdown", - "id": "3cb5e674", - "metadata": {}, + "id": "299678ea", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -4671,8 +5167,10 @@ }, { "cell_type": "markdown", - "id": "56581c9f", - "metadata": {}, + "id": "f6cb8b1b", + "metadata": { + "editable": true + }, "source": [ "where $g(0) = g_0$.\n", "\n", @@ -4681,8 +5179,10 @@ }, { "cell_type": "markdown", - "id": "ebf7032a", - "metadata": {}, + "id": "493ae9c0", + "metadata": { + "editable": true + }, "source": [ "## The trial solution\n", "\n", @@ -4706,8 +5206,10 @@ }, { "cell_type": "markdown", - "id": "a943c5c6", - "metadata": {}, + "id": "2c5f825a", + "metadata": { + "editable": true + }, "source": [ "## The program using Autograd\n", "\n", @@ -4717,8 +5219,11 @@ { "cell_type": "code", "execution_count": 43, - "id": "a3620769", - "metadata": {}, + "id": "4f16ff94", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -4888,8 +5393,10 @@ }, { "cell_type": "markdown", - "id": "9a21afc2", - "metadata": {}, + "id": "9263374a", + "metadata": { + "editable": true + }, "source": [ "## Using forward Euler to solve the ODE\n", "\n", @@ -4906,8 +5413,10 @@ }, { "cell_type": "markdown", - "id": "8a8ad46c", - "metadata": {}, + "id": "559b83cb", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4919,8 +5428,10 @@ }, { "cell_type": "markdown", - "id": "7b83336b", - "metadata": {}, + "id": "3b57d5e9", + "metadata": { + "editable": true + }, "source": [ "along with the condition that $g(0) = g_0$.\n", "\n", @@ -4931,8 +5442,10 @@ }, { "cell_type": "markdown", - "id": "ea68eaac", - "metadata": {}, + "id": "3b470395", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -4945,16 +5458,20 @@ }, { "cell_type": "markdown", - "id": "c8a2f92c", - "metadata": {}, + "id": "6c2db077", + "metadata": { + "editable": true + }, "source": [ "Now, if $g_i = g(t_i)$ then" ] }, { "cell_type": "markdown", - "id": "4f1a2187", - "metadata": {}, + "id": "936911cf", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -4973,8 +5490,10 @@ }, { "cell_type": "markdown", - "id": "cb87fc93", - "metadata": {}, + "id": "02788d12", + "metadata": { + "editable": true + }, "source": [ "for $i \\geq 1$ and $g_0 = g(t_0) = g(0) = g_0$.\n", "\n", @@ -4985,8 +5504,11 @@ { "cell_type": "code", "execution_count": 44, - "id": "f8c976e7", - "metadata": {}, + "id": "8a950397", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# Assume that all function definitions from the example program using Autograd\n", @@ -5058,8 +5580,10 @@ }, { "cell_type": "markdown", - "id": "23756e15", - "metadata": {}, + "id": "83ceed34", + "metadata": { + "editable": true + }, "source": [ "## Example: Solving the one dimensional Poisson equation\n", "\n", @@ -5068,8 +5592,10 @@ }, { "cell_type": "markdown", - "id": "3a68185f", - "metadata": {}, + "id": "1197e0d6", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -5083,8 +5609,10 @@ }, { "cell_type": "markdown", - "id": "dc784286", - "metadata": {}, + "id": "ffd53983", + "metadata": { + "editable": true + }, "source": [ "where $f(x)$ is a given function for $x \\in (0,1)$.\n", "\n", @@ -5093,8 +5621,10 @@ }, { "cell_type": "markdown", - "id": "e0f7d593", - "metadata": {}, + "id": "4ee6046d", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -5106,8 +5636,10 @@ }, { "cell_type": "markdown", - "id": "4cacfaeb", - "metadata": {}, + "id": "41bf19ac", + "metadata": { + "editable": true + }, "source": [ "This equation can be solved numerically using programs where e.g Autograd and TensorFlow are used.\n", "The results from the networks can then be compared to the analytical solution.\n", @@ -5116,8 +5648,10 @@ }, { "cell_type": "markdown", - "id": "69ba705a", - "metadata": {}, + "id": "9e36552a", + "metadata": { + "editable": true + }, "source": [ "## The specific equation to solve for\n", "\n", @@ -5126,8 +5660,10 @@ }, { "cell_type": "markdown", - "id": "995cfbc9", - "metadata": {}, + "id": "b338aaed", + "metadata": { + "editable": true + }, "source": [ "$$\n", "-g''(x) = f(x),\\qquad x \\in (0,1)\n", @@ -5136,16 +5672,20 @@ }, { "cell_type": "markdown", - "id": "769f9670", - "metadata": {}, + "id": "855d72eb", + "metadata": { + "editable": true + }, "source": [ "where $f(x)$ is a given function, along with the chosen conditions" ] }, { "cell_type": "markdown", - "id": "855cddbb", - "metadata": {}, + "id": "b96b47d7", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -5159,8 +5699,10 @@ }, { "cell_type": "markdown", - "id": "7006cb54", - "metadata": {}, + "id": "68d4f973", + "metadata": { + "editable": true + }, "source": [ "In this example, we consider the case when $f(x) = (3x + x^2)\\exp(x)$.\n", "\n", @@ -5169,8 +5711,10 @@ }, { "cell_type": "markdown", - "id": "5216ca22", - "metadata": {}, + "id": "8217f1b1", + "metadata": { + "editable": true + }, "source": [ "$$\n", "g_t(x) = x \\cdot (1-x) \\cdot N(P,x)\n", @@ -5179,16 +5723,20 @@ }, { "cell_type": "markdown", - "id": "b77670be", - "metadata": {}, + "id": "11ae80da", + "metadata": { + "editable": true + }, "source": [ "The analytical solution for this problem is" ] }, { "cell_type": "markdown", - "id": "bf2bed23", - "metadata": {}, + "id": "c7c4d772", + "metadata": { + "editable": true + }, "source": [ "$$\n", "g(x) = x(1 - x)\\exp(x)\n", @@ -5197,8 +5745,10 @@ }, { "cell_type": "markdown", - "id": "a22ccb78", - "metadata": {}, + "id": "0dd52013", + "metadata": { + "editable": true + }, "source": [ "## Solving the equation using Autograd" ] @@ -5206,8 +5756,11 @@ { "cell_type": "code", "execution_count": 45, - "id": "7b3b5f5a", - "metadata": {}, + "id": "ff65a33a", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -5364,8 +5917,10 @@ }, { "cell_type": "markdown", - "id": "6668b7ef", - "metadata": {}, + "id": "4332a0d6", + "metadata": { + "editable": true + }, "source": [ "## Comparing with a numerical scheme\n", "\n", @@ -5384,8 +5939,10 @@ }, { "cell_type": "markdown", - "id": "ec1adda2", - "metadata": {}, + "id": "456bca1b", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -5399,16 +5956,20 @@ }, { "cell_type": "markdown", - "id": "33486af4", - "metadata": {}, + "id": "166994c0", + "metadata": { + "editable": true + }, "source": [ "If $x_i = i \\Delta x = x_{i-1} + \\Delta x$ and $g_i = g(x_i)$ for $i = 1,\\dots N_x - 2$ with $N_x$ being the number of values for $x$, ([15](#approx)) becomes" ] }, { "cell_type": "markdown", - "id": "42c3ef84", - "metadata": {}, + "id": "098e9594", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -5420,16 +5981,20 @@ }, { "cell_type": "markdown", - "id": "f454dbea", - "metadata": {}, + "id": "64d2c25b", + "metadata": { + "editable": true + }, "source": [ "Since we know from our problem that" ] }, { "cell_type": "markdown", - "id": "7411a707", - "metadata": {}, + "id": "5ded4126", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -5441,8 +6006,10 @@ }, { "cell_type": "markdown", - "id": "c47c3822", - "metadata": {}, + "id": "bf63e28b", + "metadata": { + "editable": true + }, "source": [ "along with the conditions $g(0) = g(1) = 0$,\n", "the following scheme can be used to find an approximate solution for $g(x)$ numerically:" @@ -5450,8 +6017,10 @@ }, { "cell_type": "markdown", - "id": "1cca824c", - "metadata": {}, + "id": "087e5dd6", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -5468,8 +6037,10 @@ }, { "cell_type": "markdown", - "id": "82ecd5a1", - "metadata": {}, + "id": "798206b9", + "metadata": { + "editable": true + }, "source": [ "for $i = 1, \\dots, N_x - 2$ where $g_0 = g_{N_x - 1} = 0$ and $f(x_i) = (3x_i + x_i^2)\\exp(x_i)$, which is given for our specific problem.\n", "\n", @@ -5478,8 +6049,10 @@ }, { "cell_type": "markdown", - "id": "39f5fe07", - "metadata": {}, + "id": "62b32e23", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{aligned}\n", @@ -5513,16 +6086,20 @@ }, { "cell_type": "markdown", - "id": "5ac54c35", - "metadata": {}, + "id": "a480287d", + "metadata": { + "editable": true + }, "source": [ "which makes it possible to solve for the vector $\\boldsymbol{g}$." ] }, { "cell_type": "markdown", - "id": "79ad3ebc", - "metadata": {}, + "id": "4355063e", + "metadata": { + "editable": true + }, "source": [ "## Setting up the code\n", "\n", @@ -5532,8 +6109,11 @@ { "cell_type": "code", "execution_count": 46, - "id": "be03bf2d", - "metadata": {}, + "id": "490f065b", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -5730,8 +6310,10 @@ }, { "cell_type": "markdown", - "id": "faa0daa4", - "metadata": {}, + "id": "0f02abd9", + "metadata": { + "editable": true + }, "source": [ "## Partial Differential Equations\n", "\n", @@ -5745,8 +6327,10 @@ }, { "cell_type": "markdown", - "id": "fb7b7ff6", - "metadata": {}, + "id": "74d30dde", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -5760,16 +6344,20 @@ }, { "cell_type": "markdown", - "id": "71da1640", - "metadata": {}, + "id": "f0b35ea9", + "metadata": { + "editable": true + }, "source": [ "where $f$ is an expression involving all kinds of possible mixed derivatives of $g(x_1,\\dots,x_N)$ up to an order $n$. In order for the solution to be unique, some additional conditions must also be given." ] }, { "cell_type": "markdown", - "id": "c77e6225", - "metadata": {}, + "id": "c47496ab", + "metadata": { + "editable": true + }, "source": [ "## Type of problem\n", "\n", @@ -5781,8 +6369,10 @@ }, { "cell_type": "markdown", - "id": "98f82ea2", - "metadata": {}, + "id": "fa710cfa", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -5793,8 +6383,10 @@ }, { "cell_type": "markdown", - "id": "bc33501c", - "metadata": {}, + "id": "f84378dd", + "metadata": { + "editable": true + }, "source": [ "where $h_1(x_1,\\dots,x_N)$ is a function that ensures $g_t(x_1,\\dots,x_N)$ satisfies some given conditions.\n", "The neural network $N(x_1,\\dots,x_N,P)$ has weights and biases described by $P$ and $h_2(x_1,\\dots,x_N,N(x_1,\\dots,x_N,P))$ is an expression using the output from the neural network in some way.\n", @@ -5804,8 +6396,10 @@ }, { "cell_type": "markdown", - "id": "818e8900", - "metadata": {}, + "id": "664e9f0b", + "metadata": { + "editable": true + }, "source": [ "## Network requirements\n", "\n", @@ -5822,8 +6416,10 @@ }, { "cell_type": "markdown", - "id": "526d5428", - "metadata": {}, + "id": "a4a96aa1", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C\\left(x_1, \\dots, x_N, P\\right) = \\left( f\\left(x_1, \\, \\dots \\, , x_N, \\frac{\\partial g(x_1,\\dots,x_N) }{\\partial x_1}, \\dots , \\frac{\\partial g(x_1,\\dots,x_N) }{\\partial x_N}, \\frac{\\partial g(x_1,\\dots,x_N) }{\\partial x_1\\partial x_2}, \\, \\dots \\, , \\frac{\\partial^n g(x_1,\\dots,x_N) }{\\partial x_N^n} \\right) \\right)^2\n", @@ -5832,8 +6428,10 @@ }, { "cell_type": "markdown", - "id": "279b9296", - "metadata": {}, + "id": "a3d0209e", + "metadata": { + "editable": true + }, "source": [ "## More details\n", "\n", @@ -5842,8 +6440,10 @@ }, { "cell_type": "markdown", - "id": "bcb8ce40", - "metadata": {}, + "id": "39bbd3f6", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C\\left(\\boldsymbol{x}, P\\right) = f\\left( \\left( \\boldsymbol{x}, \\frac{\\partial g(\\boldsymbol{x}) }{\\partial x_1}, \\dots , \\frac{\\partial g(\\boldsymbol{x}) }{\\partial x_N}, \\frac{\\partial g(\\boldsymbol{x}) }{\\partial x_1\\partial x_2}, \\, \\dots \\, , \\frac{\\partial^n g(\\boldsymbol{x}) }{\\partial x_N^n} \\right) \\right)^2\n", @@ -5852,16 +6452,20 @@ }, { "cell_type": "markdown", - "id": "6549ea7b", - "metadata": {}, + "id": "691d78c7", + "metadata": { + "editable": true + }, "source": [ "If we also have $M$ different sets of values for $x_1, \\dots, x_N$, that is $\\boldsymbol{x}_i = \\big(x_1^{(i)}, \\dots, x_N^{(i)}\\big)$ for $i = 1,\\dots,M$ being the rows in matrix $X$, the cost function can be generalized into" ] }, { "cell_type": "markdown", - "id": "0956062e", - "metadata": {}, + "id": "eaf497e5", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C\\left(X, P \\right) = \\sum_{i=1}^M f\\left( \\left( \\boldsymbol{x}_i, \\frac{\\partial g(\\boldsymbol{x}_i) }{\\partial x_1}, \\dots , \\frac{\\partial g(\\boldsymbol{x}_i) }{\\partial x_N}, \\frac{\\partial g(\\boldsymbol{x}_i) }{\\partial x_1\\partial x_2}, \\, \\dots \\, , \\frac{\\partial^n g(\\boldsymbol{x}_i) }{\\partial x_N^n} \\right) \\right)^2.\n", @@ -5870,8 +6474,10 @@ }, { "cell_type": "markdown", - "id": "8fca0166", - "metadata": {}, + "id": "23273cdb", + "metadata": { + "editable": true + }, "source": [ "## Example: The diffusion equation\n", "\n", @@ -5880,8 +6486,10 @@ }, { "cell_type": "markdown", - "id": "58ad6f25", - "metadata": {}, + "id": "5b241ad8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial g(x,t)}{\\partial t} = \\frac{\\partial^2 g(x,t)}{\\partial x^2}\n", @@ -5890,16 +6498,20 @@ }, { "cell_type": "markdown", - "id": "e6e5728e", - "metadata": {}, + "id": "851b6ae9", + "metadata": { + "editable": true + }, "source": [ "where a possible choice of conditions are" ] }, { "cell_type": "markdown", - "id": "6921faa1", - "metadata": {}, + "id": "39620fd4", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -5912,16 +6524,20 @@ }, { "cell_type": "markdown", - "id": "80e632ef", - "metadata": {}, + "id": "83e43549", + "metadata": { + "editable": true + }, "source": [ "with $u(x)$ being some given function." ] }, { "cell_type": "markdown", - "id": "a2711421", - "metadata": {}, + "id": "7ae737e9", + "metadata": { + "editable": true + }, "source": [ "## Defining the problem\n", "\n", @@ -5930,8 +6546,10 @@ }, { "cell_type": "markdown", - "id": "981da4af", - "metadata": {}, + "id": "ef5d7fee", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -5945,16 +6563,20 @@ }, { "cell_type": "markdown", - "id": "c1b8c16d", - "metadata": {}, + "id": "01280fa8", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "b8ed6509", - "metadata": {}, + "id": "b8945521", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -5967,8 +6589,10 @@ }, { "cell_type": "markdown", - "id": "200050ac", - "metadata": {}, + "id": "86ee3290", + "metadata": { + "editable": true + }, "source": [ "with $u(x) = \\sin(\\pi x)$.\n", "\n", @@ -5979,8 +6603,10 @@ }, { "cell_type": "markdown", - "id": "5dca99d2", - "metadata": {}, + "id": "05812bd1", + "metadata": { + "editable": true + }, "source": [ "## Setting up the network using Autograd\n", "\n", @@ -5996,8 +6622,11 @@ { "cell_type": "code", "execution_count": 47, - "id": "85f4b843", - "metadata": {}, + "id": "2d0e1c98", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "def sigmoid(z):\n", @@ -6048,8 +6677,10 @@ }, { "cell_type": "markdown", - "id": "9dd68ff8", - "metadata": {}, + "id": "09a8211f", + "metadata": { + "editable": true + }, "source": [ "## Setting up the network using Autograd; The trial solution\n", "\n", @@ -6076,8 +6707,10 @@ }, { "cell_type": "markdown", - "id": "65f68c5a", - "metadata": {}, + "id": "d02f7c09", + "metadata": { + "editable": true + }, "source": [ "## Why the jacobian?\n", "\n", @@ -6103,8 +6736,11 @@ { "cell_type": "code", "execution_count": 48, - "id": "91612f4f", - "metadata": {}, + "id": "e2e322e4", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# Set up the trial function:\n", @@ -6147,8 +6783,10 @@ }, { "cell_type": "markdown", - "id": "ff9df4ba", - "metadata": {}, + "id": "c7a4bf59", + "metadata": { + "editable": true + }, "source": [ "## Setting up the network using Autograd; The full program\n", "\n", @@ -6171,8 +6809,11 @@ { "cell_type": "code", "execution_count": 49, - "id": "e6451b9e", - "metadata": {}, + "id": "12b32fb2", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -6402,8 +7043,10 @@ }, { "cell_type": "markdown", - "id": "7f867cf2", - "metadata": {}, + "id": "25a4c55c", + "metadata": { + "editable": true + }, "source": [ "## Example: Solving the wave equation with Neural Networks\n", "\n", @@ -6412,8 +7055,10 @@ }, { "cell_type": "markdown", - "id": "ded83be1", - "metadata": {}, + "id": "c7d06ad8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial^2 g(x,t)}{\\partial t^2} = c^2\\frac{\\partial^2 g(x,t)}{\\partial x^2}\n", @@ -6422,8 +7067,10 @@ }, { "cell_type": "markdown", - "id": "f0aba0c9", - "metadata": {}, + "id": "4e808405", + "metadata": { + "editable": true + }, "source": [ "with $c$ being the specified wave speed.\n", "\n", @@ -6432,8 +7079,10 @@ }, { "cell_type": "markdown", - "id": "28a48fe7", - "metadata": {}, + "id": "a6014180", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\begin{align*}\n", @@ -6447,16 +7096,20 @@ }, { "cell_type": "markdown", - "id": "61ded78d", - "metadata": {}, + "id": "e057a79a", + "metadata": { + "editable": true + }, "source": [ "where $\\frac{\\partial g(x,t)}{\\partial t} \\Big |_{t = 0}$ means the derivative of $g(x,t)$ with respect to $t$ is evaluated at $t = 0$, and $u(x)$ and $v(x)$ being given functions." ] }, { "cell_type": "markdown", - "id": "990695ea", - "metadata": {}, + "id": "fbad6b58", + "metadata": { + "editable": true + }, "source": [ "## The problem to solve for\n", "\n", @@ -6465,8 +7118,10 @@ }, { "cell_type": "markdown", - "id": "58841089", - "metadata": {}, + "id": "26e59766", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -6480,8 +7135,10 @@ }, { "cell_type": "markdown", - "id": "303eb900", - "metadata": {}, + "id": "88e5bc6a", + "metadata": { + "editable": true + }, "source": [ "where $c$ is the given wave speed.\n", "The chosen conditions for this equation are" @@ -6489,8 +7146,10 @@ }, { "cell_type": "markdown", - "id": "ce5c5167", - "metadata": {}, + "id": "a8bab835", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -6507,16 +7166,20 @@ }, { "cell_type": "markdown", - "id": "609a7556", - "metadata": {}, + "id": "b43249ae", + "metadata": { + "editable": true + }, "source": [ "In this example, let $c = 1$ and $u(x) = \\sin(\\pi x)$ and $v(x) = -\\pi\\sin(\\pi x)$." ] }, { "cell_type": "markdown", - "id": "5edb8797", - "metadata": {}, + "id": "a483fdd5", + "metadata": { + "editable": true + }, "source": [ "## The trial solution\n", "Setting up the network is done in similar matter as for the example of solving the diffusion equation.\n", @@ -6539,8 +7202,10 @@ }, { "cell_type": "markdown", - "id": "c6be7518", - "metadata": {}, + "id": "70d3377b", + "metadata": { + "editable": true + }, "source": [ "## The analytical solution\n", "\n", @@ -6553,8 +7218,10 @@ }, { "cell_type": "markdown", - "id": "368bb540", - "metadata": {}, + "id": "0308aa0e", + "metadata": { + "editable": true + }, "source": [ "## Solving the wave equation - the full program using Autograd" ] @@ -6562,8 +7229,11 @@ { "cell_type": "code", "execution_count": 50, - "id": "65085f11", - "metadata": {}, + "id": "1ee90ae8", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -6790,8 +7460,10 @@ }, { "cell_type": "markdown", - "id": "41011c90", - "metadata": {}, + "id": "761c699e", + "metadata": { + "editable": true + }, "source": [ "## Resources on differential equations and deep learning\n", "\n", @@ -6805,25 +7477,7 @@ ] } ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.15" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/src/week43/week43.do.txt b/doc/src/week43/week43.do.txt index 9c5bfe858..8325ddfe6 100644 --- a/doc/src/week43/week43.do.txt +++ b/doc/src/week43/week43.do.txt @@ -1406,7 +1406,7 @@ def create_neural_network_keras(n_neurons_layer1, n_neurons_layer2, n_categories model.add(Dense(n_neurons_layer2, activation='sigmoid', kernel_regularizer=regularizers.l2(lmbd))) model.add(Dense(n_categories, activation='softmax')) - sgd = optimizers.SGD(lr=eta) + sgd = optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) return model @@ -1582,7 +1582,7 @@ def NN_model(inputsize,n_layers,n_neuron,eta,lamda): else: #Subsequent layers are capable of automatic shape inferencing model.add(Dense(n_neuron,activation='relu',kernel_regularizer=regularizers.l2(lamda))) model.add(Dense(2,activation='softmax')) #2 outputs - ordered and disordered (softmax for prob) - sgd=optimizers.SGD(lr=eta) + sgd=optimizers.SGD(learning_rate=eta) model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy']) return model