diff --git a/doc/pub/week40/html/._week40-bs000.html b/doc/pub/week40/html/._week40-bs000.html index 79fcc3f5f..76cb74266 100644 --- a/doc/pub/week40/html/._week40-bs000.html +++ b/doc/pub/week40/html/._week40-bs000.html @@ -334,7 +334,7 @@ MathJax.Hub.Config({
-

Nov 3, 2021

+

Nov 5, 2021


diff --git a/doc/pub/week40/html/._week40-bs032.html b/doc/pub/week40/html/._week40-bs032.html index 0f332f3cb..41aad2c83 100644 --- a/doc/pub/week40/html/._week40-bs032.html +++ b/doc/pub/week40/html/._week40-bs032.html @@ -391,9 +391,9 @@ theta = np.for epoch in range(n_epochs): # Can you figure out a better way of setting up the contributions to each batch? for i in range(m): - random_index = np.random.randint(m) - xi = X[random_index*M:random_index*M+M] - yi = y[random_index*M:random_index*M+M] + random_index = M*np.random.randint(m) + xi = X[random_index:random_index+M] + yi = y[random_index:random_index+M] gradients = (2.0/M)*training_gradient(yi, xi, theta) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients diff --git a/doc/pub/week40/html/week40-bs.html b/doc/pub/week40/html/week40-bs.html index 79fcc3f5f..76cb74266 100644 --- a/doc/pub/week40/html/week40-bs.html +++ b/doc/pub/week40/html/week40-bs.html @@ -334,7 +334,7 @@ MathJax.Hub.Config({
-

Nov 3, 2021

+

Nov 5, 2021


diff --git a/doc/pub/week40/html/week40-reveal.html b/doc/pub/week40/html/week40-reveal.html index 89a271df5..defd8db62 100644 --- a/doc/pub/week40/html/week40-reveal.html +++ b/doc/pub/week40/html/week40-reveal.html @@ -184,7 +184,7 @@ MathJax.Hub.Config({
-

Nov 3, 2021

+

Nov 5, 2021


@@ -1633,9 +1633,9 @@ theta = np.random.randn(2,for epoch in range(n_epochs): # Can you figure out a better way of setting up the contributions to each batch? for i in range(m): - random_index = np.random.randint(m) - xi = X[random_index*M:random_index*M+M] - yi = y[random_index*M:random_index*M+M] + random_index = M*np.random.randint(m) + xi = X[random_index:random_index+M] + yi = y[random_index:random_index+M] gradients = (2.0/M)*training_gradient(yi, xi, theta) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients diff --git a/doc/pub/week40/html/week40-solarized.html b/doc/pub/week40/html/week40-solarized.html index eab5aa203..ac962d117 100644 --- a/doc/pub/week40/html/week40-solarized.html +++ b/doc/pub/week40/html/week40-solarized.html @@ -267,7 +267,7 @@ MathJax.Hub.Config({
-

Nov 3, 2021

+

Nov 5, 2021


@@ -1651,9 +1651,9 @@ theta = np.random.randn(2,for epoch in range(n_epochs): # Can you figure out a better way of setting up the contributions to each batch? for i in range(m): - random_index = np.random.randint(m) - xi = X[random_index*M:random_index*M+M] - yi = y[random_index*M:random_index*M+M] + random_index = M*np.random.randint(m) + xi = X[random_index:random_index+M] + yi = y[random_index:random_index+M] gradients = (2.0/M)*training_gradient(yi, xi, theta) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients diff --git a/doc/pub/week40/html/week40.html b/doc/pub/week40/html/week40.html index 1e6243108..77c973a2d 100644 --- a/doc/pub/week40/html/week40.html +++ b/doc/pub/week40/html/week40.html @@ -344,7 +344,7 @@ MathJax.Hub.Config({
-

Nov 3, 2021

+

Nov 5, 2021


@@ -1728,9 +1728,9 @@ theta = np.for epoch in range(n_epochs): # Can you figure out a better way of setting up the contributions to each batch? for i in range(m): - random_index = np.random.randint(m) - xi = X[random_index*M:random_index*M+M] - yi = y[random_index*M:random_index*M+M] + random_index = M*np.random.randint(m) + xi = X[random_index:random_index+M] + yi = y[random_index:random_index+M] gradients = (2.0/M)*training_gradient(yi, xi, theta) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients diff --git a/doc/pub/week40/ipynb/ipynb-week40-src.tar.gz b/doc/pub/week40/ipynb/ipynb-week40-src.tar.gz index 4a02258ed..d0e5c99f0 100644 Binary files a/doc/pub/week40/ipynb/ipynb-week40-src.tar.gz and b/doc/pub/week40/ipynb/ipynb-week40-src.tar.gz differ diff --git a/doc/pub/week40/ipynb/week40.ipynb b/doc/pub/week40/ipynb/week40.ipynb index 0839ac1a6..926b5c027 100644 --- a/doc/pub/week40/ipynb/week40.ipynb +++ b/doc/pub/week40/ipynb/week40.ipynb @@ -2,8 +2,10 @@ "cells": [ { "cell_type": "markdown", - "id": "2e064e52", - "metadata": {}, + "id": "90cdfc12", + "metadata": { + "editable": true + }, "source": [ "\n", @@ -12,21 +14,25 @@ }, { "cell_type": "markdown", - "id": "74a3ac3e", - "metadata": {}, + "id": "474f8807", + "metadata": { + "editable": true + }, "source": [ "# Week 40: From Stochastic Gradient Descent to Neural networks\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo, Norway and Department of Physics and Astronomy and Facility for Rare Ion Beams, Michigan State University, USA\n", "\n", - "Date: **Nov 3, 2021**\n", + "Date: **Nov 5, 2021**\n", "\n", "Copyright 1999-2021, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license" ] }, { "cell_type": "markdown", - "id": "337b794a", - "metadata": {}, + "id": "6c5cc3d9", + "metadata": { + "editable": true + }, "source": [ "## Plan for week 40\n", "\n", @@ -45,8 +51,10 @@ }, { "cell_type": "markdown", - "id": "8e6d5b80", - "metadata": {}, + "id": "6e613d6d", + "metadata": { + "editable": true + }, "source": [ "## Overview video on Stochastic Gradient Descent\n", "\n", @@ -55,8 +63,10 @@ }, { "cell_type": "markdown", - "id": "c0aa61eb", - "metadata": {}, + "id": "acdc41b1", + "metadata": { + "editable": true + }, "source": [ "## Batches and mini-batches\n", "\n", @@ -75,8 +85,10 @@ }, { "cell_type": "markdown", - "id": "a26c735b", - "metadata": {}, + "id": "312926d8", + "metadata": { + "editable": true + }, "source": [ "## Stochastic Gradient Descent (SGD)\n", "\n", @@ -105,8 +117,10 @@ }, { "cell_type": "markdown", - "id": "49c015d0", - "metadata": {}, + "id": "44cf9842", + "metadata": { + "editable": true + }, "source": [ "## Stochastic Gradient Descent\n", "\n", @@ -120,8 +134,10 @@ }, { "cell_type": "markdown", - "id": "2ffd2cee", - "metadata": {}, + "id": "6ebea4b5", + "metadata": { + "editable": true + }, "source": [ "$$\n", "C(\\mathbf{\\beta}) = \\sum_{i=1}^n c_i(\\mathbf{x}_i,\n", @@ -131,8 +147,10 @@ }, { "cell_type": "markdown", - "id": "a0b744d6", - "metadata": {}, + "id": "3f6389d9", + "metadata": { + "editable": true + }, "source": [ "## Computation of gradients\n", "\n", @@ -142,8 +160,10 @@ }, { "cell_type": "markdown", - "id": "0b5f0d51", - "metadata": {}, + "id": "22cdd63e", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\nabla_\\beta C(\\mathbf{\\beta}) = \\sum_i^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", @@ -153,8 +173,10 @@ }, { "cell_type": "markdown", - "id": "1aa15c93", - "metadata": {}, + "id": "4a2d3f21", + "metadata": { + "editable": true + }, "source": [ "Stochasticity/randomness is introduced by only taking the\n", "gradient on a subset of the data called minibatches. If there are $n$\n", @@ -165,8 +187,10 @@ }, { "cell_type": "markdown", - "id": "5a81f7c7", - "metadata": {}, + "id": "8bf174ef", + "metadata": { + "editable": true + }, "source": [ "## SGD example\n", "\n", @@ -186,8 +210,10 @@ }, { "cell_type": "markdown", - "id": "c96985f2", - "metadata": {}, + "id": "46312c82", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\nabla_{\\beta}\n", @@ -199,8 +225,10 @@ }, { "cell_type": "markdown", - "id": "e7e6836b", - "metadata": {}, + "id": "23212add", + "metadata": { + "editable": true + }, "source": [ "## The gradient step\n", "\n", @@ -209,8 +237,10 @@ }, { "cell_type": "markdown", - "id": "aca06f53", - "metadata": {}, + "id": "b967218f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\beta_{j+1} = \\beta_j - \\gamma_j \\sum_{i \\in B_k}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n", @@ -220,8 +250,10 @@ }, { "cell_type": "markdown", - "id": "1cf73b37", - "metadata": {}, + "id": "55739075", + "metadata": { + "editable": true + }, "source": [ "where $k$ is picked at random with equal\n", "probability from $[1,n/M]$. An iteration over the number of\n", @@ -232,8 +264,10 @@ }, { "cell_type": "markdown", - "id": "d9aa3662", - "metadata": {}, + "id": "0d959af0", + "metadata": { + "editable": true + }, "source": [ "## Simple example code" ] @@ -241,8 +275,11 @@ { "cell_type": "code", "execution_count": 1, - "id": "d979ba35", - "metadata": {}, + "id": "5dbc2eb6", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np \n", @@ -263,8 +300,10 @@ }, { "cell_type": "markdown", - "id": "42081dc1", - "metadata": {}, + "id": "9f8fd45d", + "metadata": { + "editable": true + }, "source": [ "Taking the gradient only on a subset of the data has two important\n", "benefits. First, it introduces randomness which decreases the chance\n", @@ -277,8 +316,10 @@ }, { "cell_type": "markdown", - "id": "0ac64f4a", - "metadata": {}, + "id": "04fcf804", + "metadata": { + "editable": true + }, "source": [ "## When do we stop?\n", "\n", @@ -296,8 +337,10 @@ }, { "cell_type": "markdown", - "id": "bc68f86a", - "metadata": {}, + "id": "5ffbe7df", + "metadata": { + "editable": true + }, "source": [ "## Slightly different approach\n", "\n", @@ -317,8 +360,11 @@ { "cell_type": "code", "execution_count": 2, - "id": "f8c5407d", - "metadata": {}, + "id": "92865897", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np \n", @@ -349,55 +395,33 @@ }, { "cell_type": "markdown", - "id": "ea2cf5bb", - "metadata": {}, + "id": "e0434bc9", + "metadata": { + "editable": true + }, "source": [ "We note that we have defined several hyperparameters. These are now the number of epochs, the number of mini-batches and the parameters $t_0$ and $t_1$." ] }, { "cell_type": "markdown", - "id": "0f1e90ab", - "metadata": {}, + "id": "1fc573b6", + "metadata": { + "editable": true + }, "source": [ "## Program for stochastic gradient" ] }, { "cell_type": "code", - "execution_count": 1, - "id": "9599012b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Own inversion\n", - "[[4.17281579]\n", - " [2.88341961]]\n", - "Eigenvalues of Hessian Matrix:[0.30657583 4.43654855]\n", - "theta from own gd\n", - "[[4.17281579]\n", - " [2.88341961]]\n", - "theta from own sdg\n", - "[[4.18670611]\n", - " [2.89245361]]\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEWCAYAAAB1xKBvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAoeElEQVR4nO3de5wcZZ3v8c8vk0ySSUIukxAiZCawSrgptwkIchlFF+Smnn0twgkqiifrjRVFEM0KKEZdcRddV9eNXD2Zg7ddXRHdFWUGkRDCBBJIIJFbBrmEBELuCZkkv/NH1SSdTl+qu6urunu+79erX9NdXV31VHVPfauep+opc3dERESKGZJ2AUREpD4oMEREJBIFhoiIRKLAEBGRSBQYIiISiQJDREQiUWBIwzKz68xsXtrliJOZdZrZ82mXQwYnBYYkysxWmtlWM9tkZqvM7DYzG512uUSkOAWGpOE8dx8NHAMcC3wh3eIMTmY2NO0ySH1RYEhq3H0V8D8EwQGAmV1tZk+b2UYze9zM3pfx3iVm9icz+5aZvWZmz5rZuzPeP9jM7g0/ezcwMXN+Zna+mS0zs3Vm1mNmh2e8t9LMrjSzR81ss5ndbGaTzey34fR+b2bjcy3HQDWRmV1hZqvN7CUz+3DG+z1m9tHs5ch47Wb2CTN7MpzX9Wb2V2Y238w2mNlPzaw5a55fNLNXwnLPzBg+PFw/z5nZy2b2AzMbmVXOz5vZKuBWM5toZr8O18laM7vPzLRdkJz0w5DUmNlBwLuBpzIGPw2cCowFvgzMM7MpGe+fCKwgCINvAjebmYXv/T9gUfje9cCHMuZ1KHAHcDkwCfgNcGfWhvhvgHcBhwLnAb8FvhiOPwT4+wKLc0BY5gOBS4Hv5QuYPM4EjgfeClwFzAUuBqYCRwEXZc1rYjivDwFzzWx6+N43wvIfA7wxHOearM9OANqBWcAVwPPhMk4Ol1f9BUlOCgxJwy/NbCPwF2A1cO3AG+7+M3d/0d13uftPgCeBEzI+2+fuP3T3ncDtwBRgspm1ATOAL7n76+7+R+DOjM+9H7jL3e92937gW8BI4OSMcb7r7i+7+wvAfcCD7v6Iu28DfkFQfZZPP/AVd+93998Am4DpBcbP9k133+Duy4ClwO/c/Rl3X08QXNnzHljOe4G7gAvC4JwFfMbd17r7RuBrwIUZn9sFXBt+dmtY7ilAe1j2+1wdzEkeCgxJw3vdfQzQCRxGRtWRmX3QzBaHVSTrCPauM6uWVg08cfct4dPRwBuA19x9c8a4fRnP35D52t13EQTWgRnjvJzxfGuO14Ua51919x0Zr7cUGT9bKfPOtZxvIDhKaAEWZay//w6HD1gTBuCAGwiO8H5nZs+Y2dUllFkGGQWGpCbcO76NYG8fM2sHfgh8Cmh193EEe9uWZxKZXgLGm9mojGFtGc9fJKiGIZyXEVT3vFD+EkS2mWBDPuCACqeXazlfBF4hCJcj3X1c+BgbnmAwYK+jB3ff6O5XuPshwPnAZ83sjArLJw1KgSFp+zbwLjM7GhhFsEFbAxA2HB8VZSLu3gf0Al82s2YzO4WgHWLAT4FzzOwMMxtGUHf/OjA/rgUpYDHwv8ysxczeSNDGUamB5TwVOBf4WXjU9EPgRjPbH8DMDjSzM/NNxMzONbM3hgG6HthJUG0lsg8FhqTK3dcAPwKucffHgX8CHiCoknkzcH8Jk/vfBI3iawnaRX6UMZ8VBI3I3yXYEz+P4PTe7TEsRjE3AtsJlul2oKvC6a0CXiM4qugCPubuy8P3Pk9QxbTAzDYAv6dwW8qbwnE2Eaz377t7d4XlkwZlat8SEZEodIQhIiKRKDBERCQSBYaIiESiwBARkUjqqvOxiRMn+rRp09IuhohIXVm0aNEr7j6p+JiF1VVgTJs2jd7e3rSLISJSV8ysr/hYxalKSkREIlFgiIhIJAoMERGJRIEhIiKRKDBERCQSBYaIiESiwBARkUgUGCIiEokCQ0REIlFgiIhIJFUPDDO7xcxWm9nSHO9dYWZuZhOrXQ4REalMEkcYtwFnZQ80s6nAXwPPJVAGERGpUNUDw93/SHCP5Ww3AlcBukesiEgdSKUNw8zeA7zg7ksijDvLzHrNrHfNmjUJlE5ERHJJPDDMrAX4InBNlPHdfa67d7h7x6RJFXfnLiIiZUrjCOOvgIOBJWa2EjgIeNjMDkihLCIiElHiN1By98eA/Qdeh6HR4e6vJF0WERGJLonTau8AHgCmm9nzZnZptecpIiLxq/oRhrtfVOT9adUug4iIVE5XeouISCQKDBERiUSBISIikSgwREQkEgWGiIhEosAQEZFIFBgiIhKJAkNERCJRYIiISCQKDBERiUSBISIikSgwREQkEgWGiIhEosAQEZFIFBgiIhKJAkNERCJRYIiISCQKDBERiUSBISIikSgwREQkkqoHhpndYmarzWxpxrAbzGy5mT1qZr8ws3HVLoeIiFQmiSOM24CzsobdDRzl7m8B/gx8IYFyiIhIBaoeGO7+R2Bt1rDfufuO8OUC4KBql0NEpKq6umDaNBgyJPjb1ZV2iWI3NO0CAB8BfpLvTTObBcwCaGtrS6pMIiLRdXXBrFmwZUvwuq8veA0wc2Z65YpZqo3eZjYb2AHkjWJ3n+vuHe7eMWnSpOQKJyIS1ezZe8JiwJYtwfAGktoRhpldApwLnOHunlY5REQq9txzpQ2vU6kcYZjZWcBVwPnuvqXY+CIiNS1fdXmDVaMncVrtHcADwHQze97MLgX+FRgD3G1mi83sB9Uuh4hI1cyZAy0tew9raQmGN5CqV0m5+0U5Bt9c7fmKiCRmoGF79uygGqqtLQiLBmrwhto4S0pEpP7NnNlwAZFNXYOIiEgkCgwREYlEgSEiIpEoMEREGkWVuydRo7eISCMo1D1JTHSEISIyoJ47EEygexIFhogI7NlD7+sD9z176JWERpIBlED3JAoMERGIfw+9GgFUSALdkygwREQg/j30pHuwTaB7EgWGiAjEv4eedA+2M2fC3LnQ3g5mwd+5c2O9+lyBISIC8e+hp9GD7cyZsHIl7NoV/I25qxIFhogIxL+HHjWA6ujMLF2HISIyIM4OBKP0YFtnt3a1errZXUdHh/f29qZdDBGRynR1BUHS15f7/fb2oEopJma2yN07Kp2OqqREBpM6qv5oWJmn2+ZTqGE8xe9QVVIig0WdVX80rFyn22bL1zCe8neoKimRwWLatNx7tTFXf0gRQ4YEF/Ll09KSv7G9zO9QVVIiUpqkrwuQ3AqdVtvaWvjMrJS/QwWGyGCRxnUBtaYW2nDmzAlO281l9OjCVUv5vqsJExJZrqoHhpndYmarzWxpxrAJZna3mT0Z/h1f7XKIDHoJdB1R05Lu2ymfmTPzV0kVO1LI9R02N8OGDYksVxJHGLcBZ2UNuxr4g7u/CfhD+FpEqimBriNqWtJ9OxXS3p57eLGjvVzf4Zgx0N+/93hVWq5EGr3NbBrwa3c/Kny9Auh095fMbArQ4+7Ti01Hjd4iUrZ8jc1mQVcaSco+2wkKN3YXUmS5tq3bxsjxI+u60Xuyu78UPl8FTM43opnNMrNeM+tds2ZNMqUTkcZTS204cR7t5Sn/uub96Ry3mHExVvin3ujtwSFO3sMcd5/r7h3u3jFp0qQESyYiDSXpNpxiDewxdRS4ffZ17Bw2Yq9hm2nhk6/fwKb+Zj55/IKypptLWoHxclgVRfh3dUrlEJHBIsk2nLgb2DPCZ9fUNpbNnMO1p/fQOW4x+826kA/030QfbezCWNs8mRXv/TzfW3k+vZuP4J96O2NbrLTaMG4AXnX3b5jZ1cAEd7+q2HTUhiEiiRno7ylfx4GFxHiR5Pa5t9L0qU/Q1L9t97DNtDCLH7Ci5Xg6D19N59ktnHLpdMa1j805jbgu3Kt6YJjZHUAnMBF4GbgW+CXwU6AN6AMucPe1xaalwBCRRFTaKF1BA/vWtVtZcNtyev5rPT1LxvGj9e+hnX1Pt9114FSGPB/tgr26CYw4KTBEJBGVHiGU8PnsgFiw/nC2M5wh7OS4lhUs3HIUlquZt4Szu+IKDHU+KCKSrdIuOObMyX2EMmcOW9du5YFbBwJiPA9uOIztHLs7IP6+4wE6zxnFKR85lLFtR8C0ttzhk8LZXamfJSUiDSbJ7jeqNa9KT8HNaGB3M7aOm8LPpn6G0z5xFONah3DG545lzn2nsm3nUD494wHuuu4h1vZt4qHNR3DDQ52cc90MxraF7RG1dIW+u9fN4/jjj3cRicm8ee7t7e5mwd958+KZZkuLe1CDHzxaWuKZdpLzqmDam9ds9j9862H/0qndfup+i72ZbQ7uQ9jhM0Yt9StndPtd1y30dX3rSitPlO8qz3hAr8ewDU49BEp5KDBEYlKtjW17+97THHi0t8dR6mTnFXEjvXnNZv/9Nxf5P5zS7afst9iH8XrOgFj/l/XxlKtQefN8p3EFhhq9RQajat0bI8nuN1Lq6mPLK1uCNohfbaDn0fE8uOFw+mlmCDvpGPUEl0/+Me999WZGbHgZK/V03EoU+E6tr0+N3iJSpmrdV6EtwQbahOaVOyCOo4kdHD9qOZ85YT6d54zibR+Zzn73LoFZN6ZzR7wE7pWhwBAZjKq1sS1wdlDsqjSvLa9sYf4tAwExgYUbD8sfEAcdtfeHC/WIW+3AKPSdFrp/eCniqNdK6qE2DJFQpQ3W1W4wjrsxvYrz2vTyJr/7Hxf57Ld1+9vGLNndBtFEv5846jG/6oRu/82XI7ZBmOVuVzEruVwlS6ANI/UQKOWhwBDx+Db2SW7Ya0ixgPj8id3+2+sf8g0vbChtwvPmuTc15Q6MajT65ytDFc+SUqO3SL2pVoN1g9q8enNQxXTnRnoem8DCjYezg2E0sYOOUcvpPOoVOs8dzds+Mp0xbxhT3kxydSUyoNz7XMRIXYOIDFa1dCOgWhJ2FujPPce2sZO5c9KlfGfVBXsFxIzRT3D6ka9WHhDZ8oV4UxPcfnvlYVFJR4ioaxCRwSvJM5HqwObVm3nmshuZ/vOv0rzrdQwYuW4V56y7kWXDR3PaW9fSee5oTv7wdMa84c3RJ1zKRjrfmUi7dsUTFplHL0meeZUtjnqtpB5qwxDxZK+mrkEbX9ro//O1Xv/CSd1+0uhHfSjb/Vna4207KHUdV+siwpjaRYipDUNHGCL1ZmCvsoIqinqyadUm5t+6ImyDaOWhTYexg+N3VzF97q33074g5msQSjk9tqsLNm3adxqVnuI7cGSxc2fu92O8viIqtWGISE3JHRDDGEo/M0Yvp/PNr9J53hhO/vB0Rh8wOvhQ3CcCRG0nytfY3doK3/lOZSGeb5kGlLBsasMQkYawadUm7r95OT2/3kTP0lZ6wyOIgYC48qT7MwIiTxtE3BfxRW0nynUkAjB6dOVHfIWOIFLqrVaBISKJyh0QHfsExGljlzDim1+BBc/Bi23QVqDaLe5quqgBVM3uOPKFVlNTeqfpxtEQktRDjd4i9WfjSxv9v7/6kF/91m5/6+hHvYl+B/ehbPeTxyzxL57c7b/7eq9venlT8IF589xbW/dt5E26YT/KhY3V7DE3xpMb0JXeIjnU+9XL9V5+d9/wwgb/7fUP+edP7PYTRz1WPCAy5dpIpnHFdFTVPmMtpt+DAkMkW72fblqn5c8XEBdzu79kU3wX5lvGHeBbv39z8Ynl22NPsk+mUtVByMcVGKmeJWVmnwE+CjjwGPBhd9+Wb3ydJSUF1XuXGXVS/o0vbuT+W1aEbRAT6d18GDsZyjC2c8KY5XS+ZS3vP+h+jvrVHGzr1j0fjNJFRr6zkwbU2LqoOXkuNqz7rkHM7EDgT8AR7r7VzH4K/Mbdb8v3GQWGFFTvXWbUaPk3vriRP920nJ67NtOzbCKLcgRE5/n7cdIl0xm1/6jgQ+WGX6FTSWugT6aalusU33Cd2cUXN8RptUOBkWbWD7QAL6ZcHklKhX3j5FTvXWbUSPlzB8SM3QFx9dv+lBEQb8k9kXLPHsp1dhLEc11Doyt0sWFc4qjXKvcBfBrYBKwBuvKMMwvoBXrb2triqdCTdFWrrr5O2wB2S6n86/+y3n/z5YV+1QndfkJGG8QwXvdT9lvs/3BKt//+m4t885rN0SdaqC2iUD1/9hlSra3xLn8dtDeUrcC9OKj3Rm9gPHAPMAkYBvwSuLjQZ9To3SCqfSpioQ1CrW8wEijf+r+s97uuW+hXzogxILIVO9spVxAmccZRPe9QFFPg/6oRAuNvgZszXn8Q+H6hzygwGkRadyVr9A1GHpkBMWPUUh/CjvgDIpeB8MsXGk1NewdjNXck3Ks//bQ18h33gBOBZQRtFwbcDlxW6DMKjAaR1j9uvW4wSjzqKBQQp+632L90arf/4VsPxx8Q+eTbQcjesFX7VNo0b5+alCrfcS+1wAiWgS8Dy4GlwP8FhhcaX4HRINLa0y+04arVo4wI62pd3zr/9bUL/XMdNRIQ2YpdW5F5xKEjjKpoiMAo9TEoA6PW69zLlcZyFdpwRQ2spMudp8ybxx7gn+vo9o6WZbsDoplttREQ2Yq1ZxQ60lAbRiwSCwzgbuDoOGZW6WPQBcYg/oFXRaXdTqTxfeQ5KtqJeTPb/LSxj/g1p3X7Pf/0sG95dUv1ylGpzKAtdCRR7UBu1B2wIpIMjOOAbuBWYEocMy33MegCYxAfQlfNvHn5A6NYXXah7yPGDVFmFdOLNiXnPLeMn1LbAVGIdoQSl3iVFPA3wBLgWmBkHDMv9THoAmMwNNKlodwgLtQGUsEGcF3fOr/zSw/6Fcd3+/FZVUxfHjnHXx8yvPE2roN0Tz8tcQXGkCgX95mZASuAfwMuA540sw9E+axUIN8VvvVy5XI1dHUF3UcMGRL87eoqfRpz5gRdJmSKckOafOu9qSn3FbYf+lDOcq7rW8+vr1nI5zp66Bj1OBPaR3Pe9Sfw3UUnMXrYdr50+n1037iYda/u4pofttM8fvSe6ba21mf3GNnfGwRdhOzaFfytt+UZrIolCnA/QZcddwPXA+cCbwS+C8yNI7WiPgbdEYYO3fcW5/ooZw833/wjNOb2Dxvh86bN3ucI4vSxj/i1p3d7942P7FvF1Cjff6MsRx0jwTaMIwk7Kczx3hNxFCLqY9AFhrsO3TPF2aZT7nrN9bmIp42uZGrhgKjm8qapUZajjsUVGBX1Vmtmh7j7MxUc4JREvdUOcnH15lqgV89yqkY2f+cmhl95GUP78/bMD4CbYaWUs0Z7ry1ZoyxHHYure/NIbRj5JBkWUgVxtAckKa42nQp79VzXt547v7SQKzp6OL7lCcZc/hE+2H8TfUxlF8bOPP9WVmo5oy5vrX+PaotrHHEcpiT1GJRVUtVSj/XKcZW5xLPP1j7zmv/XFxf4Z47r9mNHPu7GTgf34Wz1znEP+3Wd3d7z7Ud862tb4y3nvHnuzc17T6e52f3jH99TLdba6j5sWG1/j/X4W2sw6EpvqUi91ivH0aZTZNkLBcTbcwVEtco5b96+YTBkyL4hUg/fY5xtcWrXK1lcgZHqLVpLpTaMGA3meuUcbRg7ho7gxwdewT+vnsnirdNxhjCcbZw87gk6j11P5/vGc8IHpjNi3Ijkylno7nPFNOr3GHP702BRE20YUkey67knTMg93iCoV37t5HPofedVrB02mV0YK2njgztu4qN9/8C44du47u1/5N5/WcK61+Ce147lmns6Oe2yo5MNCyh+d7pCGvV7TOKucpJX2rdolSRk75X19UFzMwwbBv39e8aLcvFarmnHfavVmK19+jXuu/nP9Pz3NnqWT2bJ1kNxrmUEV3Hy+Cc4/ZgNfOx947nlAzBi3LFpF3ePfLdsLaac77FelHvrV4lHHPVaST3UhlGmfHX2ra2V1QXXaGPmq0+t9V9+YYFffmyPHzPyid1tECPY4u8Yv8i/8o5u/+N3F/u29dvyT6QW6slzrd9hw3I3hLe2Do46/Xpte0sZavSuA7Ww0XGvXp9UcfzzxrCOXn1qrf/i6gX+6WN6/OgRy8sLiOwy1UoQ5lo/mcNaWwdPWLjX1ndTRxQYta6WftjV2isrFERRgqDMdVQoIM7IFxClBFO+9dXUVFsbplr6jSWpVnbE6ogCo9bVQjcWmZ+vxoalUFVXlPlFXEfFAuL6M7r9vu8tyX8EkWv5B8Iu1/os1ittrWygVD0jESkwal1c1UBxXgQW915ZvrK1tkbbkOVZR7vAN+93gN8+dba/ZcTy3W+NZHO0gMhWrK+n7PVZbPxa2SDHWdWovfaGpsCodXHt/dX6XmSuDU2UDdm8efnvvBY+NjHS57R8tfSAyFboiCHX+ix2Z75auR9JXL+NwVq1NYgoMGpdSt1YpK5QEIQbsg3f+nfvHzqi+EY8rmCM0pts9vqMsBypi+s3Vus7JVKxhggMYBzwc2A58ARwUqHxyw6MtA63E+jGoirzLFeBPfP+oSP8tqmz/c0jVviz5FmmagVjsSOGfOuzHva84/i+622nRErWKIFxO/DR8HkzMK7Q+GUFRq3+00f9Ry+l/Gkva55w66fJL2Kej2Szv3NCr+8iQhVR3Hu5mfetyN5AFlpHg6FuX0cYDa/uAwMYCzxLnpsz5XqUFRi1+M9Q6oY96kYrpWVds/wV/48rH8gbBLswv/8Hj/rrG18vXM5SNuRRFbuOIXN9DoZwyCXtHQ2pukYIjGOAhcBtwCPATcCoHOPNAnqB3ra2ttLXVC0ebqdxXcSAGDaKqx9f4z//3Hz/1Jt7/Kjhf949mz6mRluufBuozG6749hg19PRWdoGa1gOEo0QGB3ADuDE8PV3gOsLfaZhjjDSuvK6zI1ivoBoYZO/a0Kvz3lXt9//g0e9/+bbSttAV3svv5TvPunfiTbQkqBGCIwDgJUZr08F7ir0mYZpw6jWxqnYskacb6GA+OvWh/xrf93t8/89o4opuwyZ12G0tkZf17nKX0k/SYVOp82eVpJHorX4m5SGVveBESwD9wHTw+fXATcUGr+mz5IqZR7V3GAUKke+C+XM/Gefne+fLCcg4ly2KKe/lrKeSrlgL8kjjFo86pWG1iiBcUzYPvEo8EtgfKHxa/Y6jHI2kmlUSeTZUD1Le86A2L55e2zziLQxLBYWpW5YSzmdNsm9/lpsV5OG1hCBUeqjZgOjxvcYX1662n/22fl+60GzfQt7XzC3k6Arjm0Tpnj/LbdXPrNKNoZFrvwua8OaeTptsWklFeI1/nuRxqPAKEe1Ngi1sMeYsWw7phzkD5x1rX/iqB4/MqOKaRQb/eujvuLrRkz2XWFVVOx71LV0hBFXueKmNgxJmAKjVNX8J015Y7TuG//m/UOH7zXvTbT4JdzsZ7Y+5F8/s9sf+OFje1cxpdXwXkjcbRhxlasadJaUJEiBUapqbtQT3hi9vHS1//Qz8/0TR/X4EcOfzNvVxq6B5ctVjmoeFZW7MYz7LKm4yiVS5xQYuZRxhlDRDWSUjUwlp5JGkB0QmVVMZ01cWLyrjQruRZE4bdRFYqfAyBbTNQglTTPqOCVa9dhq/8nl9/vHj7zXD29+avdkR7PBz5q40L9xVrcvuCmjiilKVU7Uq621gRZpOAqMbNW4yjlKyMSwp14oIN49aaH/47u7/cFblnr/1v7cE4hy+miuIyntzYsMCgqMbNXoRynKNMuo6io5IKJWixU60qi1+1GLSGIUGNmqUScf0xHGS0te9h///f3+sSPu9cPCgLiIed7HVN+J+boRk/3J//ON3EcQ5fRsm+9ooxGqnHRUJFIyBUa2atTJl9mGsXPESJ9/5rV7BQS4j2G9nz1pof/q6Nm+o3lEtLKW2/ZS63eLK4faXUTKosDIpRp7nxGm+drXvu+bxkz2XZg/zxv8IubtFRDfPLvbF962bM8RRCkhUO7ZXaW0ZdSLWj2zS6TGxRUYFkyrPnR0dHhvb2/axeClxS9z7y1P0/P7HfQ8fRArth8CwBg2cOqkFXTO2EznBftz7PsPZeiIoftOYMiQYFOXzQx27dp72LRp0Ne377jt7bBy5d7Durpg9uzc4xf6XL0oZb2JyG5mtsjdOyqdTo6tmWTLHRCTGcMGTtt/OZd2PJcREDOKT7CtLfdGva1t32Fz5sCsWbBly55hLS3B8ExdXfuOl81s38/Vk1LWm4jEL47DlKQeSXU++OIjq/yOy+73vzv8Xp/e/PRebRDn7P+g33BOtz90+7L8p7kWU05DdrGqtijXYkB55a0VasMQKQuqkorPiw+v4t5bnwmPIKby5/6DAdiP9Zy6/wo6Z2yh84L9OeaCPFVM5RioPnruuWAPec4cmDmz/Onlq67JVM/VUQPiXm8ig0BcVVKDMjAKBcRp+6/g9GoERLXla+sY0NICc+dq4yoyCKkNowS5A+KA3QEx64Q+Ot8/mWMuOJSm5hOCD3V1wWFn18+ebK62DrPgqKO9vfbLLyI1ryED48WHV9Fz89P03LOTnqen8mSUgMiU3YDc1xe8htrd6A6US9U1IlIlDVEl9ULvS8ERxF4BAWNZz2mTl3P6jK0ZAdFUfEalnMoqIlLjBnWVVO6AmLI7ID52Yh+dF0zm6L89lKbmE0ufwXPPlTZcRGQQqKvA6Fu2iUObny09IEo9s0bn+4uI7CP1wDCzJqAXeMHdzy007mvbRnLS5NWlHUGU0x4R9WI5EZFBJPU2DDP7LNAB7FcsMDqO7/DeRSWeVltue4TO9xeRBhFXG8aQOApTLjM7CDgHuCnaB8qYSbntETNnBoGya1fwV2EhIoNcqoEBfBu4Csjbc5yZzTKzXjPrXbNmTelzyNfuoPYIEZGSpBYYZnYusNrdFxUaz93nunuHu3dMmjSp9BnNmRO0P2RSe4SISMnSPMJ4G3C+ma0Efgy8w8zmxT6XmTODLjHa24Mrn9vb1UWGiEgZUgsMd/+Cux/k7tOAC4F73P3iqsxM7RHx6OoKTiIYMiT429WVdolEJEGpn1YrdaIeu0sRkVilflptKWrljnuDkrpLEalbDXFardQRdZciMugpMCQanZ4sMugpMCQanZ4sMugpMCQanZ4sMujpLCmJbuZMBYTIIKYjDBERiUSBISIikSgwREQkkvoKjEWL1CWFiEhK6iswYE+XFAoNEZFE1V9gQNCf0ezZaZdCRGRQqc/AAHVJISKSsPoNDHVJISKSqPoMDHVJISKSuPoLjKamPW0Y5TZ860ZAIiIlq6+uQYYMgZ07g+fl3sBHNwISESlLfd1Aycz3uX1SqTfw0Y2ARGSQ0Q2UBpR6tpRuBCQiUpb6D4xSz5bSjYBERMpSX4ExJKu45ZwtpRsBiYiUJbXAMLOpZtZtZo+b2TIz+3TRD7W3V34DH90ISESkLKk1epvZFGCKuz9sZmOARcB73f3xfJ/p6Ojw3t59mr1FRKSAum/0dveX3P3h8PlG4AngwLTKIyIihdVEG4aZTQOOBR7M8d4sM+s1s941a9YkXjYREQmkHhhmNhr4D+Byd9+Q/b67z3X3DnfvmDRpUvIFFBERIOXAMLNhBGHR5e7/mWZZRESksDTPkjLgZuAJd//ntMohIiLRpHmE8TbgA8A7zGxx+Dg7xfKIiEgBqXU+6O5/Aiyt+YuISGlSb/QWEZH6oMAQEZFIFBgiIhKJAkNERCJRYIiISCQKDBERiUSBISIikSgwREQkEgWGiIhEosAQEZFIFBgiIhKJAkNERCJRYIiISCQKDBERiUSBISIikSgwREQkEgWGiIhEosAQEZFIFBgiIhKJAkNERCJJNTDM7CwzW2FmT5nZ1WmWRURECkstMMysCfge8G7gCOAiMzsirfKIiEhhaR5hnAA85e7PuPt24MfAe1Isj4iIFDA0xXkfCPwl4/XzwInZI5nZLGBW+PJ1M1uaQNkqNRF4Je1CRKByxqceyggqZ9zqpZzT45hImoERibvPBeYCmFmvu3ekXKSiVM541UM566GMoHLGrZ7KGcd00qySegGYmvH6oHCYiIjUoDQD4yHgTWZ2sJk1AxcCv0qxPCIiUkBqVVLuvsPMPgX8D9AE3OLuy4p8bG71SxYLlTNe9VDOeigjqJxxG1TlNHePYzoiItLgdKW3iIhEosAQEZFIaiYwinUTYmbDzewn4fsPmtm0jPe+EA5fYWZnpljGz5rZ42b2qJn9wczaM97baWaLw0dVG/cjlPMSM1uTUZ6PZrz3ITN7Mnx8KOVy3phRxj+b2bqM9xJZn2Z2i5mtznf9jwX+JVyGR83suIz3klyXxco5MyzfY2Y238yOznhvZTh8cVynX1ZQzk4zW5/x3V6T8V5iXQlFKOeVGWVcGv4eJ4TvJbI+zWyqmXWH25xlZvbpHOPE+/t099QfBI3eTwOHAM3AEuCIrHE+AfwgfH4h8JPw+RHh+MOBg8PpNKVUxrcDLeHzjw+UMXy9qYbW5SXAv+b47ATgmfDv+PD5+LTKmTX+ZQQnRiS9Pk8DjgOW5nn/bOC3gAFvBR5Mel1GLOfJA/Mn6I7nwYz3VgITa2R9dgK/rvT3Uu1yZo17HnBP0usTmAIcFz4fA/w5x/96rL/PWjnCiNJNyHuA28PnPwfOMDMLh//Y3V9392eBp8LpJV5Gd+929y3hywUE15YkrZIuV84E7nb3te7+GnA3cFaNlPMi4I4qlSUvd/8jsLbAKO8BfuSBBcA4M5tCsuuyaDndfX5YDkjvtxllfeaTaFdCJZYzrd/mS+7+cPh8I/AEQQ8amWL9fdZKYOTqJiR7wXeP4+47gPVAa8TPJlXGTJcSJPuAEWbWa2YLzOy9VSjfgKjl/JvwEPXnZjZwAWVS67KkeYVVewcD92QMTmp9FpNvOZJcl6XK/m068DszW2RBVzxpO8nMlpjZb83syHBYTa5PM2sh2ND+R8bgxNenBVX0xwIPZr0V6++z5rsGqUdmdjHQAZyeMbjd3V8ws0OAe8zsMXd/Op0Scidwh7u/bmZ/R3Dk9o6UyhLFhcDP3X1nxrBaWp91w8zeThAYp2QMPiVcl/sDd5vZ8nAPOw0PE3y3m8zsbOCXwJtSKksU5wH3u3vm0Uii69PMRhME1uXuvqFa84HaOcKI0k3I7nHMbCgwFng14meTKiNm9k5gNnC+u78+MNzdXwj/PgP0EOwNVEPRcrr7qxlluwk4PupnkyxnhgvJOuRPcH0Wk285aq7rGzN7C8H3/R53f3VgeMa6XA38gupU6Ubi7hvcfVP4/DfAMDObSA2uz1Ch32bV16eZDSMIiy53/88co8T7+6x2w0zExpuhBI0uB7OnQevIrHE+yd6N3j8Nnx/J3o3ez1CdRu8oZTyWoGHuTVnDxwPDw+cTgSepUoNdxHJOyXj+PmCB72kIezYs7/jw+YS0yhmOdxhBI6KlsT7DeUwjfyPtOezdqLgw6XUZsZxtBO17J2cNHwWMyXg+HzgrxXIeMPBdE2xonwvXbaTfS1LlDN8fS9DOMSqN9Rmulx8B3y4wTqy/z6qt7DIW/myCVv6ngdnhsK8Q7KkDjAB+Fv7oFwKHZHx2dvi5FcC7Uyzj74GXgcXh41fh8JOBx8If+WPApSmvy68Dy8LydAOHZXz2I+E6fgr4cJrlDF9fB3wj63OJrU+CvceXgH6Cet5LgY8BHwvfN4IbgT0dlqUjpXVZrJw3Aa9l/DZ7w+GHhOtxSfibmJ1yOT+V8dtcQEbA5fq9pFXOcJxLCE64yfxcYuuToFrRgUczvtezq/n7VNcgIiISSa20YYiISI1TYIiISCQKDBERiUSBISIikSgwREQkEgWGiIhEosAQEZFIFBgiFQjvR/Cu8PlXzey7aZdJpFrU+aBIZa4FvhJ2NHcscH7K5RGpGl3pLVIhM7sXGA10enBfApGGpCopkQqY2ZsJ7ny2XWEhjU6BIVKm8M5lXQR3NdtkZlW7o55ILVBgiJQhvNPafwJXuPsTwPUE7RkiDUttGCIiEomOMEREJBIFhoiIRKLAEBGRSBQYIiISiQJDREQiUWCIiEgkCgwREYnk/wMYMor9sJsXawAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], + "execution_count": 3, + "id": "09c35f49", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], "source": [ "%matplotlib inline\n", "\n", @@ -474,8 +498,10 @@ }, { "cell_type": "markdown", - "id": "1a445879", - "metadata": {}, + "id": "64ed24bb", + "metadata": { + "editable": true + }, "source": [ "## Momentum based GD\n", "\n", @@ -487,8 +513,10 @@ }, { "cell_type": "markdown", - "id": "e50f5198", - "metadata": {}, + "id": "6dae73b6", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t) \\nonumber\n", @@ -497,8 +525,10 @@ }, { "cell_type": "markdown", - "id": "9b1f5752", - "metadata": {}, + "id": "831c5807", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -513,8 +543,10 @@ }, { "cell_type": "markdown", - "id": "55dc76ca", - "metadata": {}, + "id": "3922d4f8", + "metadata": { + "editable": true + }, "source": [ "where we have introduced a momentum parameter $\\gamma$, with\n", "$0\\le\\gamma\\le 1$, and for brevity we dropped the explicit notation to\n", @@ -530,8 +562,10 @@ }, { "cell_type": "markdown", - "id": "1a922557", - "metadata": {}, + "id": "d5068c23", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\Delta \\boldsymbol{\\theta}_{t+1} = \\gamma \\Delta \\boldsymbol{\\theta}_t -\\ \\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t),\n", @@ -540,16 +574,20 @@ }, { "cell_type": "markdown", - "id": "8cda1681", - "metadata": {}, + "id": "c46fe71c", + "metadata": { + "editable": true + }, "source": [ "where we have defined $\\Delta \\boldsymbol{\\theta}_{t}= \\boldsymbol{\\theta}_t-\\boldsymbol{\\theta}_{t-1}$." ] }, { "cell_type": "markdown", - "id": "1f3f99f4", - "metadata": {}, + "id": "6692fe1e", + "metadata": { + "editable": true + }, "source": [ "## More on momentum based approaches\n", "\n", @@ -562,8 +600,10 @@ }, { "cell_type": "markdown", - "id": "f16258e5", - "metadata": {}, + "id": "ade434cc", + "metadata": { + "editable": true + }, "source": [ "$$\n", "m {d^2 \\mathbf{w} \\over dt^2} + \\mu {d \\mathbf{w} \\over dt }= -\\nabla_w E(\\mathbf{w}).\n", @@ -572,16 +612,20 @@ }, { "cell_type": "markdown", - "id": "714e253d", - "metadata": {}, + "id": "fb546dff", + "metadata": { + "editable": true + }, "source": [ "We can discretize this equation in the usual way to get" ] }, { "cell_type": "markdown", - "id": "ba9c27ca", - "metadata": {}, + "id": "330ecc06", + "metadata": { + "editable": true + }, "source": [ "$$\n", "m { \\mathbf{w}_{t+\\Delta t}-2 \\mathbf{w}_{t} +\\mathbf{w}_{t-\\Delta t} \\over (\\Delta t)^2}+\\mu {\\mathbf{w}_{t+\\Delta t}- \\mathbf{w}_{t} \\over \\Delta t} = -\\nabla_w E(\\mathbf{w}).\n", @@ -590,16 +634,20 @@ }, { "cell_type": "markdown", - "id": "9ada2b55", - "metadata": {}, + "id": "e5f8c0be", + "metadata": { + "editable": true + }, "source": [ "Rearranging this equation, we can rewrite this as" ] }, { "cell_type": "markdown", - "id": "1dab5b45", - "metadata": {}, + "id": "c1e8fcb0", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\Delta \\mathbf{w}_{t +\\Delta t}= - { (\\Delta t)^2 \\over m +\\mu \\Delta t} \\nabla_w E(\\mathbf{w})+ {m \\over m +\\mu \\Delta t} \\Delta \\mathbf{w}_t.\n", @@ -608,8 +656,10 @@ }, { "cell_type": "markdown", - "id": "af0c4c94", - "metadata": {}, + "id": "7f3bd58d", + "metadata": { + "editable": true + }, "source": [ "## Momentum parameter\n", "\n", @@ -622,8 +672,10 @@ }, { "cell_type": "markdown", - "id": "a99f4367", - "metadata": {}, + "id": "39cfb115", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\gamma= {m \\over m +\\mu \\Delta t }, \\qquad \\eta = {(\\Delta t)^2 \\over m +\\mu \\Delta t}.\n", @@ -632,8 +684,10 @@ }, { "cell_type": "markdown", - "id": "33e67d82", - "metadata": {}, + "id": "f1401af5", + "metadata": { + "editable": true + }, "source": [ "Thus, as the name suggests, the momentum parameter is proportional to\n", "the mass of the particle and effectively provides inertia.\n", @@ -663,8 +717,10 @@ }, { "cell_type": "markdown", - "id": "8e88cb9c", - "metadata": {}, + "id": "fa7fc1b4", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma \\mathbf{v}_{t-1}) \\nonumber\n", @@ -673,8 +729,10 @@ }, { "cell_type": "markdown", - "id": "85d24114", - "metadata": {}, + "id": "4bbf3232", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -689,16 +747,20 @@ }, { "cell_type": "markdown", - "id": "96cbc7ad", - "metadata": {}, + "id": "1cee2641", + "metadata": { + "editable": true + }, "source": [ "One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of $\\gamma$." ] }, { "cell_type": "markdown", - "id": "09587ea5", - "metadata": {}, + "id": "54339b58", + "metadata": { + "editable": true + }, "source": [ "## Second moment of the gradient\n", "\n", @@ -726,8 +788,10 @@ }, { "cell_type": "markdown", - "id": "a4fab14c", - "metadata": {}, + "id": "4d210ce2", + "metadata": { + "editable": true + }, "source": [ "## RMS prop\n", "\n", @@ -739,8 +803,10 @@ }, { "cell_type": "markdown", - "id": "9d1e9528", - "metadata": {}, + "id": "cadaff5b", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -755,8 +821,10 @@ }, { "cell_type": "markdown", - "id": "b983c2e4", - "metadata": {}, + "id": "0e5a7835", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbf{s}_t =\\beta \\mathbf{s}_{t-1} +(1-\\beta)\\mathbf{g}_t^2 \\nonumber\n", @@ -765,8 +833,10 @@ }, { "cell_type": "markdown", - "id": "a83cdb71", - "metadata": {}, + "id": "325cbf1a", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\mathbf{g}_t \\over \\sqrt{\\mathbf{s}_t +\\epsilon}}, \\nonumber\n", @@ -775,8 +845,10 @@ }, { "cell_type": "markdown", - "id": "87ba5403", - "metadata": {}, + "id": "cb7f472b", + "metadata": { + "editable": true + }, "source": [ "where $\\beta$ controls the averaging time of the second moment and is\n", "typically taken to be about $\\beta=0.9$, $\\eta_t$ is a learning rate\n", @@ -791,8 +863,10 @@ }, { "cell_type": "markdown", - "id": "34368494", - "metadata": {}, + "id": "623c3e4f", + "metadata": { + "editable": true + }, "source": [ "## ADAM optimizer\n", "\n", @@ -812,8 +886,10 @@ }, { "cell_type": "markdown", - "id": "a52d1737", - "metadata": {}, + "id": "bca8e3c2", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -828,8 +904,10 @@ }, { "cell_type": "markdown", - "id": "0520dbc3", - "metadata": {}, + "id": "956129dd", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbf{m}_t = \\beta_1 \\mathbf{m}_{t-1} + (1-\\beta_1) \\mathbf{g}_t \\nonumber\n", @@ -838,8 +916,10 @@ }, { "cell_type": "markdown", - "id": "7e43fae4", - "metadata": {}, + "id": "c2ec8ca8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbf{s}_t =\\beta_2 \\mathbf{s}_{t-1} +(1-\\beta_2)\\mathbf{g}_t^2 \\nonumber\n", @@ -848,8 +928,10 @@ }, { "cell_type": "markdown", - "id": "52dde39c", - "metadata": {}, + "id": "1e8f5983", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{\\mathbf{m}}_t={\\mathbf{m}_t \\over 1-\\beta_1^t} \\nonumber\n", @@ -858,8 +940,10 @@ }, { "cell_type": "markdown", - "id": "2b5dd7dc", - "metadata": {}, + "id": "cffa531b", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{\\mathbf{s}}_t ={\\mathbf{s}_t \\over1-\\beta_2^t} \\nonumber\n", @@ -868,8 +952,10 @@ }, { "cell_type": "markdown", - "id": "4c8fd34a", - "metadata": {}, + "id": "fd755a07", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\boldsymbol{\\mathbf{m}}_t \\over \\sqrt{\\boldsymbol{\\mathbf{s}}_t} +\\epsilon}, \\nonumber\n", @@ -878,8 +964,10 @@ }, { "cell_type": "markdown", - "id": "76f7a92f", - "metadata": {}, + "id": "885b4ca2", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -893,8 +981,10 @@ }, { "cell_type": "markdown", - "id": "dda15856", - "metadata": {}, + "id": "cc8084d2", + "metadata": { + "editable": true + }, "source": [ "where $\\beta_1$ and $\\beta_2$ set the memory lifetime of the first and\n", "second moment and are typically taken to be $0.9$ and $0.99$\n", @@ -910,8 +1000,10 @@ }, { "cell_type": "markdown", - "id": "8aeda58f", - "metadata": {}, + "id": "42257d44", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\Delta \\theta_{t+1}= -\\eta_t { \\boldsymbol{m}_t \\over \\sqrt{\\sigma_t^2 + m_t^2 }+\\epsilon}.\n", @@ -920,8 +1012,10 @@ }, { "cell_type": "markdown", - "id": "13cad70d", - "metadata": {}, + "id": "25bbb3d9", + "metadata": { + "editable": true + }, "source": [ "## Practical tips\n", "\n", @@ -938,8 +1032,10 @@ }, { "cell_type": "markdown", - "id": "419c2c61", - "metadata": {}, + "id": "ec2c7d6f", + "metadata": { + "editable": true + }, "source": [ "## Automatic differentiation\n", "\n", @@ -974,8 +1070,10 @@ }, { "cell_type": "markdown", - "id": "6ddf2971", - "metadata": {}, + "id": "0b8c3ecf", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f(x) = \\sin\\left(2\\pi x + x^2\\right)\n", @@ -984,16 +1082,20 @@ }, { "cell_type": "markdown", - "id": "a60c1bc2", - "metadata": {}, + "id": "c32d1c75", + "metadata": { + "editable": true + }, "source": [ "which has the following derivative" ] }, { "cell_type": "markdown", - "id": "b0771c96", - "metadata": {}, + "id": "77bc33e9", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f'(x) = \\cos\\left(2\\pi x + x^2\\right)\\left(2\\pi + 2x\\right)\n", @@ -1002,38 +1104,23 @@ }, { "cell_type": "markdown", - "id": "490ba19d", - "metadata": {}, + "id": "771e53af", + "metadata": { + "editable": true + }, "source": [ "Using **autograd** we have" ] }, { "cell_type": "code", - "execution_count": 2, - "id": "a4638b4a", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdwAAAEWCAYAAADM0CYnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABDoklEQVR4nO3dd3gU5drH8e+dAiEkhEBC70gnECAI6BEsWLCAYsMGVizH+loPNjzHip5jV+wFu6CIoiKoWAFJIITeWwglBAKhhLTn/WMmuMZ0kn223J/rypXdndnZ3zw7M/e0nRFjDEoppZSqXSG2AyillFLBQAuuUkop5QVacJVSSikv0IKrlFJKeYEWXKWUUsoLtOAqpZRSXmCl4IrIcSKyspaGPU5EXq+NYQc6EXlbRB4up/vDIrJTRLZ5M1cgEpHjRSTddg6bRMSIyFFHOIylInJ8Od1ni8jVRzD88SLyXnXf7y1H0pYi8o2IjDnCz79cRH6tQv8bRGRoNT+rjYjsE5HQ6ry/gmFfIiLf1fRwi1VYcN2GOSgiOSKSLSK/i8h1IlLtYm2M+cUY06W67/fI9reFljHmUWNMtWcwf1YTC7Byht0GuB3oboxpVhufUckcl7vjeWEV31fuyoTyT8aYHsaY2XDkxTEYVoJKayNjzDBjzDu2MlWVMWaTMSbKGFN4JMMRkXbusiTMY9jvG2NOOfKUpats0TzLGBMNtAUeB+4G3qjOB3qOnPIrbYAsY8yO0jp68XsdA+wCRnvp8yqk07S2gfIOv5/OjDHl/gEbgKElXjsaKAJ6us/rAk8Bm4DtwESgntvteCAdp0hvAyYVv+Z2vxuYXGL4zwLPuY+vAJYDOcA64Fr39frAQTfHPvevBTAeeM/t5xvgxhLDXgSMdB93BWbiLMBXAheU0w6NgLeADGA3MNWj2zXAGnc404AWHt0McAOw2h2H/wAdgd+BvcAnQJ0SbTUO2Om2/SUew5oNXO3x/HLgV/fxz+5n7Xfb4kL39TOBVCDb/cxeHu/vAyxwc30MfAQ8XMq4Dy3R1m8D7dzPu8r93n/GWYG7D9gI7ADeBWLcYRT3fwWw2W3D64D+QJqb74UKpsW2boZzgQKgWWltUaLtjwLGAvlAnpv/S7d7N7dNs4GlwHCP9zYGvnS/o/nAw57Dd4f9T/d7Xe8x3W5235MCHOfRfz233XYDy4A7ceeBMsa1B39Om9uBcR7z2jM402GG+7huiennLrf9twJnA6cDq9xhjfP4jPHAZPe7z3Gnhd4e3e8B1rrdlgHnlGjv34CngSy3fcpcDrjvudPNlAFcWfz9lDLuJwCLPZ7PBOZ7PP8FONtz+QSc5n6/+e53vMhjnvmPmzUH+A6IK+Uzy1uefIIzLee400mSx/taAFOATGA9cHM53+kZwEJ3+tgMjPfo1s5tjzFu++0E7i2xzJ2DM61uBV7AXW6UmNb7u20f6tFtJM5yr7w28lyuXMOfy9xlQN9KTg+/ljPul+EsF7KAe/GoKzjLjeJhZ7nt3ahEu3guZ4pfCwMuBJJLfNZtwLRKtPkmdzjF3/cg/rpMfRl4qsSwvwD+r6rf/eH3V9hDKQXXI+z17uOncQpNIyAaZ0H1mMdCoAB4AmeGrMdfC25b4AAQ7T4PdSeogR4N1hEQYIjbb1+PYaeXyDWePwvuaOA3j27dcSbYujgz2GacAhCGU3x24uwyLa0dpuMsmGKBcGCI+/qJ7vv6usN9Hvi5xIzwBdAAZyF6CPge6ADE4Ey4Y0q01f/cYQ3BKaBdypgxDk8cnjOdx/M+OAveAW67jnG/z7pAHZwZ4DZ3fM7DmRH/VnBLa2v+nOjfdduyHs5CdI07blHAZ8CkEv1PBCKAU4BcYCrQBGjpZh1SzrR4P/CH+3gxcHt5M7xne+AUu4c9uoW7Wce5bXEizoKkuK0/cv8icaabzaW09Uycab545fJSnEIdhrP7fRsQ4XZ7HKdQNAJaA0soo+DizENb3WFEuM8HuN3+Dcx12yweZyXqPyWmnwfc8bsGZ2HwgTuMHjhFpb3HvJLvfvfhwB04C45wt/v5OAuVEJwF236guUd7FwA3ueNbj/KXA6fhFIKeONPLB5RdcOu500acm2s7sMUdZj13HBqXXD7hMe97DGs2zoK8s/ve2cDjlZnGPYaZi7PSEgo8Bsz1KBQpbnvXwZnu1wGnljP8BPd9vdzxKl5xaOe2x2tuzt44y4pubvd+wEC3rdvhFMRby5jWlwHDPLp9jjuvlNNGV3t851twCrfgFPG2lZweSi24OPPPPmAwzrLnfzjTTvH3dgvONN3K7f4K8GE5y5ni18Jw5s8coJPH580HRlWhzcNKW464eTcD4j6PxZn2itug0t/94eGX17HkBF3i9bk4ayriNnxHj26D+HOt/3ictaqIchbevwKj3ccnA2vLyTMVuKWCGaS44Ea72YonmEeAN93HFwK/lHjvK8CDpXxmc5w139hSur0BTPB4HoWzEGvnMSMc69E9Bbjb4/l/gWdKLDDre3T/BLi/5IxR2kTO3wvuy7gLY4/XVuIU8sE4Wxri0e13ql5wO3i89j1wg8fzLm5bhHn039Kjexbulrj7fAoeC5FSMqwu7g78C3cNvawZnvIL7nE4BTHE47UP3ekn1M3dxaNbaVu4J1Yw7+zG3WLEmRlP8+g2lrIL7kXAwjK6rQVO93h+KrDB4zs6iLt1gzP9G9xi7TH9FS9wxuMWD/d5CE6hP66Mz04FRni09yaPbhUtB97Eo9DhFMBSC67b/RecLbOBOFuln+AU7ROANI/+NlBxwb3P4/kNwLeVmcY9hjnL43l34KD7eIBnG3hMl2+VN1149PsM8HSJ+amVR/c/cAtHKe+9Ffi8jGn9buB993EjnI2U5h7jU17BnYG7fK1E/pLTQ1kF9wHgI4/n9XFqQvH3thw4yaN7c/6+3PBczhS/FuY+fw94wH3cCacAR1ahzcsquIKzYTnYfX4N8MORfPdHcpZyS5xdVPE4axkp7klV2cC37uvFMo0xueUM6wOchQzAxe5zAERkmIjMFZFd7rBPx1nzrZAxJgdny3SU+9JFwPvu47bAgOLM7rAvAUo7Iag1sMsYs7uUbi1wthSLP3MfTiFp6dHPdo/HB0t5HuXxfLcxZr/H843uZ1RHW+D2EuPY2h1eC2CLcacUj8+qqs0ej//SFu7jMKCpx2tVaYvDRORYoD3OVic400iCiCRWI3Nx1s3GmKISeVviTLth/HXcPB+X+pqI3CEiy0Vkj9vWMfw5rbYo0X95bd0ap7CWlbtkG3tOH1nmz5NJDrr/y2vjw5nctkgvHp6IjBaRVI9ppyd/nfc8x6ei5UBVxh/gJ5wCONh9PBtnRXGI+7wqPM+qP0AZ01gV3h/hHktsC7QoMX+N46/T+2EiMkBEfhSRTBHZg3NIpeSyrNSsItJZRL4SkW0ishd4tJT3FnsPOEtE6gMX4GxYbK3kuJY57VVieijLX757d/mW5dG9LfC5x3CXA4X8tR1Lm/+KlawfU40xB9zMlWnzUrnLxo9KDNuzflT6uy9WrYIrIv1xFky/4uxOPQj0MMY0dP9ijDGeE7UpbTgePgWOF5FWwDm4BVdE6uJs9TwFNDXGNAS+xlnzqMxwwdlquUhEBuHsnvvRfX0z8JNH5obGOfPt+lKGsRloJCINS+mWgdP4uJnr4+xW3FKJbKWJdYdRrI37GeBsQUR6dKvobOHNwCMlxjHSGPMhzpZMSxERj/7bVCOv53fwl7Zwh1fAXxf41TUG53tPdX+WNM/jdSjRNiJSsm1KTisZQOsSZ9u3wfneMt3crTy6tS4l0+FhishxOMdOL8DZE9IQ2MOf0+rWEsMor6034+yiKk1pbZxRRr+VcTiT2xatgAwRaYuze/NGnN23DXF2g3tOL55tWtFyoCrjD38vuD9RccGtzPKgPFV9/2acLXjP+SvaGHN6Gf1/gLPLvbUxJgbn8IqU0W9JLwMrcHadNsBZuJf6XmPMFpzjvSNxjp1O8uxciXHqWPLFSk4PZfnLdy8ikTjLSM/PHFaiHSPc8ahM7plAvLvyfREeG2yU3+aVrR/nueM/AKceFWeuyncPVLHgikgDETkTp+q/Z4xZ7K4VvwY8LSJN3P5aisiplR2uMSYTZw32LXcklrud6uDs088ECkRkGM6xv2LbgcYiElPO4L/GWUD9G/jYY4vmK6CziFwmIuHuX38R6VZKvq04J2C9JCKxbr+D3c4fAleISKK7gvAoMM8Ys6Gy41+Kh0SkjrsQPxNnhQScXTgjRSTS/fnPVSXet52/LqhfA65z1/JEROqLyBkiEo0zQxYAN7vjMxLnxIwj8SFwm4i0F5EonLb42BhTcCQDFZEInEI2Fkj0+LsJuNjd2lgE9HC/hwicXWeeSrbNPJwtiLvc8T8eOAtn11chzvHn8W5bd6Xis6KjcdozEwgTkQdwjtsX+wT4lzv9tHKzl+UroLmI3CoidUUkWkQGuN0+BO4TkXgRicPZXXckvxPtJyIj3Ta8Fee44Vyc3X7GHR9E5AqcLZpSVWI58AlwuYh0dxe4D1aQ63ecQxJH4xy3X4q7VwrnxJnSbAfaSfV/sliZ5YmnP4AcEblbROqJSKiI9HQ3SEoTjbOnLFdEjsbZYqqsaJwTf/a502NpGwae3sVZAUzAmZaLVdRGrwN3iEg/d5lxlFtsqjQ9lDAZOFNE/iEidXCWxZ6fPxF4xP0c3Gl7RCWHjTEmH2cZ+STOLvSZHp3La/NMnEOFZa3cYoxZiLMy+TowwxiT7Xaq6ncPVL7gfikiOThV/V6cg95XeHS/G+cElLni7O6YhTOzVMUHOGcbHl47cXcJ34wzs+7GaaxpHt1X4CyA1rmb9X/b9WqMOYQzwZU27FNwdjdn4OzKKT6xqzSX4RxXWIFzcs+t7nBm4ZzMMwVnTa4jf+7Cro5tOOOagbP74jp3PME5KSUPZ6Z5hz93bxQbD7zjtsUFxphknOMOL7jDXINzjAJjTB7OGvDlOIcGLuSvM2Z1vImzNv0zzsk3uZRfWCrrbJytp3eNMduK/9zPC8M5NroKZ0aehXOst+SP8N8AurttM9Ud/7OAYTgz1Es45xEUt/WNOLuEi8+s/xCnGJVlBs4u1FU4u0tz+etusIfc19fjHJOcVHIAxdxp82Q33zZ3fE5wOz8MJOOc2b0Y58ziI/l98Rc43/1unGl8pDEm3xizDOf8gjk401sCzpm+5SlzOWCM+Qbn+NkPbj8/lDcgd7fjAmCp+13hZtloyvhpGn+umGaJyIIKspb2mRUuT0r0X4izQpyI870WL5jLKtg3AP92l6UP4CzXKusOnOVfDs6KzccV9P857q7a4t2rrnLbyBjzKc65Lh+4nzUV54zh6kwPxcNcinNG/wc4y8jdOIcuij2Ls1z/zm2buTgrVlVRXD8+LbGCX2abu+3yCPCb+30PrGDYnvWjqt898OfZV8oHuFtZ7xljWlXQq/IyEXkC52dIYyrs2U+IyHicE20utZ1F1TwRWYvzM8pZtrMoh15LWalSiEhXEenl7lY7Gmf3/ee2cylVGSJyLs4u4HL3JCjv8u+rdihVe6Jxdi+2wNmF9l+c3a9K+TQRmY3z86XLSpyFryzTXcpKKaWUF+guZaWUUsoLdJdyOeLi4ky7du1sx1BKKb+SkpKy0xgTX3GfwUULbjnatWtHcnKy7RhKKeVXRKQ6V60LeLpLWSmllPICLbhKKaWUF2jBVUoppbxAC65SSinlBVpwlVJKKS/QgquUUkp5QVAVXBG5TUSWisgSEfnQvY2bUkopVeuC5ne4ItIS51Z/3Y0xB0XkE5zb6L1tNZhSKugU5OexccUCdq1bQGHODkzeAaRuFGENmxPfsR+tjupFaFjQLJ6DRrB9o2FAPRHJByJx7jmrlFK1zhQVsfinz8hb+CFd9/xGRzlIx9J6nA9ZxLCm8fHEHnslnfse7+WkqrYETcE1xmwRkaeATTg3M//OGPNdyf5EZCwwFqBNmzbeDamUCjimqIgF375Do+Sn6VW0kWyiWNboREI6DCG+8wBim7Ymsn40+/ftJWvLWnau/oPQtbNI2PktkdO+YOm3CYSe8hBdk06yPSrqCAXN3YJEJBaYAlwIZAOfApONMe+V9Z6kpCSjl3ZUSlXXplWp7J1yCz0PpbIhpA2Zva+n92lXUqduxaeP7Nu7myVfPkfH1W8Rz27+aHg6XcY8T0xsnBeSHxkRSTHGJNnO4WuC6aSpocB6Y0ymMSYf+Aw4xnImpVQAMkVF/DHlaZq8P5Q2h1Yzr9s4Wo9bSP8RN1Sq2AJENYhl4CUPEnl7KnOaX0qf3TM48OxAVsyfVcvpVW0JpoK7CRgoIpEiIsBJwHLLmZRSASb34H6Snx3F0YvHszqiJ3nXzWXAhXdX+ySo+tENGXTti6w7azJGhI5fXcD8qS/WcGrlDUFTcI0x84DJwAJgMc64v2o1lFIqoOzZlcm6p0+l/54ZzGlzLd3vnEVcs5o5F6RL0onUv3kOKyMS6J86jjlv3oUpKqqRYSvvCJpjuNWhx3CVUpW1c9tm9r16Oi0Kt5B29BMknXFNrXxO3qFcFr00hv57vmVOi9EMvPpZJMS3tp30GG7pfOtbUkopP7Q7cys5r55Bk8LtrBr6Vq0VW4A6dSNIuuVD5jUewaCMd5n79j219lmqZmnBVUqpI7Bn906yJp5Bi8IM1g19jZ7Hjaj1z5SQEPrf8BbzGw5j0KZXmPfx47X+merIacFVSqlqyjuUS/rL59CmYAMrjn/ZK8W2WEhoKH1vfI+FkceQtOxx0n6c7LXPVtWjBVcpparBFBWR+vIV9MhLI63fo/Q+4XyvZwgNC6Pz9R+yIaw97WffyMblKV7PoCpPC65SSlXDvA8e4ujsr5nb6iqShl9nLUf96IZEXTGZQ1IXPh3D/pxsa1lU+bTgKqVUFS35dRpHr36WBVFDOPqKJ23HoWmrjmwb+jytC9NZ9vo1+nMhH6UFVymlqmDnts00m3UTm0Nb0uXadwkJDbUdCYCe/xjOvLbX0H/Pd8z/4gXbcVQptOAqpVQlFRUWsvWt0USZ/RSd+yb1oxvajvQXR49+jKV1Euie+ijbNq22HUeVoAVXKaUq6Y9J95NwaAFpCeNo32OA7Th/ExoWRsNRrxFCEZnv665lX6MFVymlKmFt2u/0Wz+RlOgT6D/yVttxytSyQzcW97iDhEML+WPK/2zHUR604CqlVAXyDuXCFzewR6LpOGaiz11KsaT+597OkrqJ9FzyJNvT19qOo1y+PdUopZQPSHnvPjoWrmfzMY/QMK6Z7TgVCgkNJfbCiYRSSPpH/2c7jnJpwVVKqXKsTfudpE1vktxgKH1OudR2nEpr2aEbC9teSb99s1n88xe24yi04CqlVJkKCwoonHYLeySaTmNesh2nyvpc9CDp0oyGP97DodwDtuMEPS24SilVhuTPn6FzwSo29P0XMY2b2o5TZRH16rNr8CO0Nhks/OQx23GCXpjtAIFo3qf/JWr1VMKLcgkvOkSBhJMfWo9D4TEcimqNNGpPw479addzIHUjIm3HVUqVYteOLXRd+j+W1ulFvzPH2o5Tbb1OOI/UP16j+7rXyd75T784Bh2otODWhsI8QkwhuWEN2B9SlxCTT1jhQRod3EjTffOI2J4PyyHvyzCW1u1GTtuTaT3oPFp26GE7uVLKteaDO+hjcok65xmfPyu5IrHDH6X+Rycx/5P7GXjDa7bjBC0xxtjO4DUi0hB4HegJGOBKY8ycsvpPSkoyycnJNZrBFBWxfcs6ti77jUPr59F0x2+0L9oAwLLwnuzvcTEJp4whIjKqRj9XKVV5K/6YSdevz2NO80sZdO2LtuPUiD+eu5TErK/JHPNLra/ci0iKMSapVj/EDwVbwX0H+MUY87qI1AEijTHZZfVfGwW3NBnrV7Dxl/dptX4yrU0GO2nImqOuIGHErT536TilAl1RYSFrHhtITEEWUbcvCJh5cGfGRiJf6c+K6IH0vWNarX6WFtzS+fd+kioQkRhgMPAGgDEmr7xi600t2ndl0Oj/0Or+pSwZOoltddsxcM3T5P03gXkfP05Bfp7tiEoFjQXfvEHnglVs6nNHwBRbgLgWbVnUZjR99/3EqgU/2Y4TlIJmC1dEEoFXgWVAbyAFuMUYs79Ef2OBsQBt2rTpt3HjRi8ndayYP4uCWf+h56FU1oe048DQx+hxzOlWsigVLHIP7mf3E73ZH9qADuPm+8ydgGpKzp5dFD6dwMZ63el998xa+xzdwi1d0Gzh4pwg1hd42RjTB9gP3FOyJ2PMq8aYJGNMUnx8vLczHta1/1B63P0jCwc9R0TRfnp8dxHzXriCA/v2WMukVKBL/fRxmpNJ7gnjA67YAkTHNGJ5+zH0PvgHK5N/sB0n6ARTwU0H0o0x89znk3EKsM+SkBD6nDqG2LtSmdv0Ivpnfs6u/w5gxfxZtqMpFXB2Z26lx9rXWFRvAD3/Mdx2nFqTcM6d7CaaQ7MetR0l6ARNwTXGbAM2i0gX96WTcHYv+7yIyCgGXj+R5ad+QAiFdPzqAuZ++IjeekupGrTqk/uIJJeGwwP7AhFRDWJZ0eEKeuXO15V3Lwuaguu6CXhfRNKARMCvVvF6HHM6UbfMZUn9AQxcOYGUp8/TXcxK1YCMDSvps+NzkhufRdtu/WzHqXW9zrmd3TQg73u/WgT6vaAquMaYVPf4bC9jzNnGmN22M1VVg4aN6X37V8xt90/67v2BjKdPYOe2TbZjKeXX0qeOxxBC+3P/bTuKV9SPbsiKDpfTKzeF1am/2I4TNIKq4AaKkNBQBl7+KIuHTKRFQTr5E09k44oFtmMp5Zc2r1lM393fsrDpOTRp2d52HK/pMfxWckw99s56ynaUoKEF14/1PnEUW86eTDj5xH50ph6PUaoatk8bTz5hHDXyfttRvKpBw8YsaXEuiTk/sWXdUttxgoIWXD/Xqc9g8i7/jr3SgNZfXcLyeTNsR1LKb2xcnkLfPd+zqPn5xDVrYzuO1x111p0UEkr69Am2owQFLbgBoEW7LtS5+huyQhvT9uvLWPrbdNuRlPILWdMf4iB16XJucG3dFotv0Y7URqfSe+d0sran244T8LTgBogmLdsTOfZbdoQ2ocN3l2vRVaoC65fOo+++n0hrdRGx8c1tx7Gm2bC7qEMBq6bpsdzapgU3gMQ1a0OD62awPbQpbb+7Ss8+VKocu759nP0mgu4j/2U7ilVtOieyKOpYemz5RH9mWMu04AaYRk1aEnnVl+RIFHFTL2bTqlTbkZTyOZvXLCZx74+ktTiPmMZNbcexLmLwzTRgP4u/0Xvl1iYtuAGoScv2FFz6OUUIdT44l+3pa21HUsqnbJ3+GPmE0WnE3y6nHpS69j+ZNaEdabr8bb2CXS3SghugWh+VQPbIj4gy+9n31nnsz8m2HUkpn7Bt8xoSd33LoviziGvW2nYcnyAhIezqeQXtijaz9LcvbccJWFpwA1jHXsew7vgXaFewnlUvX0RhQYHtSEpZt+HLJxCgzVm6deup12lXsosG5M+ZaDtKwNKCG+B6nXAeyd3ups+B35n/+s224yhl1a4dW+i9fSqpsafQvG2Xit8QRCLq1Wdlq/PovX8OW9Yttx0nIGnBDQIDRv2LeXHnMnDb+/wx5RnbcZSyZuUXE6hLPk1P163b0nQcdjOFhLB5xrO2owQkLbhBot+1E0mLSCIx7T+sWvCT7ThKed3e7Cx6pn9MavRg2nROtB3HJzVp2Z60BkPovv0LPe+jFmjBDRJh4XVoe80HZEksDaZdye7MrbYjKeVVy758lmg5SIOT77IdxafVH/xPGnCApbMm2Y4ScLTgBpGYxk3Zd/ZbxJo9pL9+sZ5EpYJGft4h2q99j6V1enNU73/YjuPTuvQ7kWWnfkTS8BtsRwk4WnCDTKfE41jU614SDi1g/tu6pq+Cw6Jv36IpWRQM1CJSEQkJofugYYSEhtqOEnCCruCKSKiILBSRr2xnsaX/ObfwR8PTGZj+Bot//tx2HKVqlSkqouGiV9kY0oqEIefbjqOCWNAVXOAWIKjPeZeQEBKueZWNIa1p9sNt7NqxxXYkpWrN0jnTOapwLdu7X6VbbcqqoCq4ItIKOAN43XYW2+rVj6bgnNeJMTlsevsqvZybCliFvz7PLhrQ64xrbUdRQS6oCi7wDHAXoNUF6JgwkAVdbiPxwBz++PRJ23GUqnEbVyyg98F5rGwzioh69W3HUUEuaAquiJwJ7DDGpFTQ31gRSRaR5MzMTC+ls2fAqHEsiuhP72VPsn7ZfNtxlKpR27/7H7kmnC5n3mo7ilLBU3CBY4HhIrIB+Ag4UUTeK9mTMeZVY0ySMSYpPj7e2xm9TkJCaHn5W+yXSMzkq8k7lGs7klI1Imt7Or2zvmVR42E0atLSdhylgqfgGmP+ZYxpZYxpB4wCfjDGXGo5lk+Ia9aazcc+ToeiDaS8d6/tOErViFXTn6Ou5NPs1NttR1EKCKKCq8qXePLFJDc4maRNb7Fm0W+24yh1RPIO5dJp08ekRfSnbZdE23GUAoK04BpjZhtjzrSdw9d0GvMieySakC9u0F3Lyq+lzZpEHNmYo8fajqLUYUFZcFXpYho3Jf0fzq7lBZPG2Y6jVLVFpb5JujQnYci5tqModZgWXPUXiUMvYn7MKSRtfos1i361HUepKluz6Fe65i8jvdMleqEL5VO04Kq/6TzmJXZLDDLtRgry82zHUapKdv/4AgdMXbqfrtdNVr5FC676m5hG8aQPeoiOhetJ/vhR23GUqrRdO7bQa/csFscNo0HDxrbjKPUXWnBVqRJPvoyFkcfQa/VLZGxYaTuOUpWy8psXnZ8CDb3ZdhSl/kYLriqVhITQfNRzAGR+fKNea1n5vIL8PNqv/5gldRNp262f7ThK/Y0WXFWmZm06kdb5Rnof/IOFM96xHUepci3+4UOasZP8ftfYjqJUqbTgqnIlXXAPa0I70nreQ+zNzrIdR6ky1Ul5nQxpQq8TR9mOolSptOCqcoWF18Gc9SyNTDbL37vDdhylSrV+6Tx65KWxqcNFhIaF2Y6jVKm04KoKdUo8juT4kSRlfs7axXNtx1Hqb3Z8/wK5Jpyuw/SnQMp3acFVldL14ifYK9Ecmna7nkClfMqe3TtJyJpBWqNTaBjXzHYcpcqkBVdVSkyjeFb3vI3u+UtImf6a7ThKHbZixmtEyiFih1xvO4pS5dKCqyqt39k3szr0KNqkPM7+nGzbcZTCFBXRdNUHrArrTKfE42zHUapcWnBVpYWGhVE4bAJN2EXaB/fbjqMUK+fPol3RJrK7XWI7ilIV0oKrqqRr0knMjzmNfhnvs3n1IttxVJDb99ur5Jh69Dz1CttRlKqQFlxVZe0vepI86rBrip5ApezJ3rmNhD2zWRZ/OpFRMbbjKFUhLbiqyuKatWFJ5+vpnTuftNmf2I6jgtSKGa9QV/JpcqKeLKX8Q9AUXBFpLSI/isgyEVkqIrfYzuTP+p53N5ulBQ1/fVhv4ae8zhQV0XLNRywP70777v1tx1GqUoKm4AIFwO3GmO7AQOCfItLdcia/VaduBDsH3Uvbos2kfP6M7TgqyCz9/Stamwz29bzMdhSlKi1oCq4xZqsxZoH7OAdYDrS0m8q/JQ69mKV1Eui87Hm9zrLyqkNz3yCbKBJOHm07ilKVFjQF15OItAP6APMsR/FrEhJCndMfI5a9LP14vO04Kkjs3LaZXjm/sKLpWURERtmOo1SlBV3BFZEoYApwqzFmbyndx4pIsogkZ2Zmej+gn+mUeBzzY06lX8YHeqN65RWrZ7xMuBTS4qTrbEdRqkqCquCKSDhOsX3fGPNZaf0YY141xiQZY5Li4+O9G9BPtTn/MQoJYeuUe2xHUQGuqLCQths+ZWmd3rTpnGg7jlJVEjQFV0QEeANYboz5n+08gaRpq46ktr6Ufjk/sCL5e9txVABb8vPntDA7yE0cYzuKUlUWNAUXOBa4DDhRRFLdv9NthwoUCRc8wE4awrf36sUwVK0pnP8GWcSQcJJeylH5n6ApuMaYX40xYozpZYxJdP++tp0rUEQ1iGVdz1voWrCc1JmTbMdRAWh7+loS9s9lVYsR1KkbYTuOUlUWNAVX1b6+I25kY0grGs97Qi+GoWrcuu8mEoKhzVC9ybzyT1pwVY0JC69D1oB7aFO0hQXTXrQdRwWQgvw8Om6azJJ6/WjZoZvtOEpVixZcVaP6nHwJK8O60m7xc+Qe2Gc7jgoQS2ZPpgm7KOyrdwVS/ksLrqpREhJCwYkP0oRdpE5+wnYcFSAk5U120IiEEy6wHUWpatOCq2pcj2NOZ1FEf7qve4M9u/TiIerIZGxYScLBZNa2HklYeB3bcZSqNi24qlZEnfEwUeYAyz79t+0oys9tnPkSBmh/it6GT/k3LbiqVnRMGMiCmJPok/EhO7astx1H+am8Q7l02jKVxfUH0qz1UbbjKHVEtOCqWtPinEcIoYgNU+63HUX5qcXff0gc2UjSlbajKHXEtOCqWtOifVcWNBlJ36zpbFqVajuO8kN1F73NVuLpOXik7ShKHTEtuKpWdTpvPIeow84vdCtXVc3m1YvoeSiVDe3OIzQszHYcpY6YFlxVqxo3bUVam8vou/9nVqf+YjuO8iNbZr1Mvgmlk54spQKEFlxV63qeN45sojgwQ89YVpWTe3A/XbZ/xeLoY4lr0dZ2HKVqhBZcVeuiYxqxosOV9D74Byv+mGk7jvIDS2ZOIpYcwo++2nYUpWqMFlzlFb1G3sFOGlI46z+2oyg/UH/xu6RLc3oce6btKErVGJ8suCJyk4jE2s6hak5kVAxruoylR94ilvw6zXYc5cM2LE+mW/5S0jtcQEhoqO04StUYnyy4QFNgvoh8IiKniYjYDqSOXOLZt7KdxoTNflRvUq/KtP2Hl8kzYXQ5TU+WUoHFJwuuMeY+oBPwBnA5sFpEHhWRjlaDqSMSUa8+G3veSNeC5aTN/sR2HOWDDu7PoVvm16TFHE9sfHPbcZSqUT5ZcAGMMQbY5v4VALHAZBGZUN1hulvLK0VkjYjcU0NRVRX0Gf5PtkhT6v82gaLCQttxlI9Z/N1bNOAAkcdcYzuKUjXOJwuuiNwiIinABOA3IMEYcz3QDzi3msMMBV4EhgHdgYtEpHsNRVaVFF6nLhmJt3JU4VpSZ06yHUf5mIZLJ7EhpDXdjj7FdhSlapxPFlygETDSGHOqMeZTY0w+gDGmCKjuaYtHA2uMMeuMMXnAR8CImomrqqLvGWPZGNKaxn88RWFBge04ykesWfQrnQtWsa3TRUiIry6alKo+n5yqjTEPGmM2ltFteTUH2xLY7PE83X3tL0RkrIgki0hyZqbey7U2hIaFkdX/dtoWbWbh16/ZjqN8RNZPr3DQ1KHbqWNtR1GqVvhkwbXJGPOqMSbJGJMUHx9vO07ASjxlNGtDO9Bs4TPk5x2yHUdZlrNnFwlZM1gcO5SYRjrfqcAUTAV3C9Da43kr9zVlQUhoKDnH3E0rs42F0160HUdZtmzGG0TKIRoOvtZ2FKVqTTAV3PlAJxFpLyJ1gFGAXoHBot4nXMDKsK60XfICuQf3246jLDFFRcStfJ+1oR3olDjYdhylak3QFFxjTAFwIzADWA58YoxZajdVcJOQEPKHjKMpWaROfcZ2HGXJygU/0rFwPTu7XqInS6mAFlRTtzHma2NMZ2NMR2PMI7bzKOhx7FksrdOLo1a+ysH9ObbjKAtyfnmV/SaCHqdeZTuKUrUqqAqu8j0SEkLISfcRRzaLPnvSdhzlZXt2ZZKQ/T1L4k4jqoFePl0FNi24yrpuA04lLSKJLmvfZN/e3bbjKC9a/u0rREg+cUP0ZCkV+LTgKp8QccoDxJLD4imP246ivMQUFdF8zYesDOtCx17H2I6jVK3Tgqt8Que+Q1gYeQw9Nr7Lnl16wZFgsGzut7QtSmdPj8tsR1HKK7TgKp8Rc/p4GnCAZZ89ajuK8oKDc15nL5EknHK57ShKeYUWXOUzOvQcQErU8fTa/AG7dug1SQJZ1vZ0eu2dzbL4M6hXP9p2HKW8Qguu8ilxZz5IBIdY9Zn+aiuQrZ7xCnWkkOYn6k3mVfDQgqt8StuufVnQ8BQSt37CzoxS71+h/FxhQQGt13/MsvCetO3Wz3YcpbxGC67yOS1GPEgYhaz9/N+2o6hasOTnybQ02zmYeIXtKEp5lRZc5XNadujBgsZn0GfHVLZtWm07jqppf7xOJrEkDL3UdhKlvEoLrvJJbc5+EIBNUx+ynETVpPQ1S+idO581bc6nTt0I23GU8iotuMonNWvTiYXxI+iT9TVb1uk9JgJF+swXyDehdDrtRttRlPI6LbjKZ3Uc+SCFhJDxhW7lBoKD+3Povn0aaQ0GE9eire04SnmdFlzls+JatCW1+QX0zf6OjSsW2I6jjtDib1+nAfupd+x1tqMoZYUWXOXTOo+8l1zqsvMr3cr1Z6aoiMbL3mFdSDu6HX2K7ThKWaEFV/m0Rk1aktbqIvrtm826JfNsx1HVtGL+TDoWriez22i9ybwKWjrlK5/X/dx72Uske74ebzuKqqYDv050rps87GrbUZSyJigKrog8KSIrRCRNRD4XkYa2M6nKi2kUz9K2o+lz4HdWLfjJdhxVRTszNtJr708sa3IWkVExtuMoZU1QFFxgJtDTGNMLWAX8y3IeVUUJ597DbqLJ/U6vPuVvVn/7IuFSSMtTbrIdRSmrgqLgGmO+M8YUuE/nAq1s5lFVF9UglpUdr6RXbjLL582wHUdVUn7eIY7a9AlpEf1pfVSC7ThKWRUUBbeEK4FvyuooImNFJFlEkjMz9UbovqT3yDvZSUOKvv8PpqjIdhxVCWkzJxHPbor667FbpQKm4IrILBFZUsrfCI9+7gUKgPfLGo4x5lVjTJIxJik+Pt4b0VUl1asfzZouY+mRt5ilv31pO46qhKiFr5Iuzel1/Pm2oyhlXcAUXGPMUGNMz1L+vgAQkcuBM4FLjDHGalhVbYln38p2GhP202O6levjVsyfRZeClWzpMoaQ0FDbcZSyLmAKbnlE5DTgLmC4MeaA7Tyq+iLq1WdDj3/StWA5abMn246jyrH/p+fZSyQ9z9CbzCsFQVJwgReAaGCmiKSKyETbgVT19R1xI1ukKfV/e1y3cn3Utk2r6Z3zM8uanUP96Ia24yjlE4Ki4BpjjjLGtDbGJLp/ejFXPxZepy4ZvW/mqMK1pM6cZDuOKsWGr58GoN3pt1lOopTvCIqCqwJPnzPGsimkJbHznqKwoKDiNyiv2Z+TTfdtn7MoejDN2nSyHUcpn6EFV/mlsPA6ZPb7P9oVbWLht2/ajqM8LJn+Mg04QP3jb7YdRSmfogVX+a0+p13B+pB2NE15moL8PNtxFFBUWEjLle+wMqwLXZNOsh1HKZ+iBVf5rZDQULIH3klrk8GCaS/ZjqOAtB8+ppXZyr4+Y21HUcrnaMFVfi1x6MWsDOtCu8XPcnB/ju04QS98/stsI47ep4y2HUUpn6MFV/k1CQmh8KSHaMIuUic/bjtOUFu14Cd65KWx4ahLCQuvYzuOUj5HC67ye90HDSO13kB6rH+T7J3bbMcJWvu+f4q9RNLjrFtsR1HKJ2nBVQGh4fBHqG8OsuLTB21HCUqbVy8icd8vLG1xPtExjWzHUconacFVAaFdtyRSYofRd9tkMjastB0n6Gz9egL5hNFp+J22oyjls7TgqoDR9ryHKULI+Gyc7ShBZWfGRhJ3fUtq3BnENWttO45SPksLrgoYTVt1ZGGLi0jaO4s1i36zHSdorP5yAqEU0uqMu2xHUcqnacFVAaX7BQ+STRQHvrnfdpSgsDc7i54ZU0iNHkLLDj1sx1HKp2nBVQElJjaOFZ3G0is3hcU/f2E7TsBbOu0ZouUgMSfrsVulKqIFVwWcPufeyVbiqffTQxQVFtqOE7AO7s+h07p3WVy3D0f1/oftOEr5PC24KuDUjYhkS9/bOapwLSlf6q2Pa8uiL54ljmxCj7/bdhSl/IIWXBWQ+p4xllVhnWmb+hT7c7Jtxwk4uQf303HV6yytk0D3QcNsx1HKL2jBVQEpJDSUolMfowm7SPv4IdtxAs6iac8Tz27MEN26VaqygqrgisjtImJEJM52FlX7uvYfSnL0SfTZPImtG/ViGDXlUO4B2i1/leXhPegx6AzbcZTyG0FTcEWkNXAKsMl2FuU9rS6YgEHImHyP7SgBI3XaizQli4Lj7kRCgmYRotQRC6a55WngLsDYDqK8p1nro0htM5p+OT+wYt53tuP4vbxDubRd9gorwrrR8x8jbMdRyq8ERcEVkRHAFmPMokr0O1ZEkkUkOTMz0wvpVG3rfeED7KARoTPH6c+EjtDCaS/SjEzyjr1Dt26VqqKAmWNEZJaILCnlbwQwDnigMsMxxrxqjEkyxiTFx8fXbmjlFZFRMWzqexedClaT8uXLtuP4rdwD+2i/9AVWhnUlYchI23GU8jsBU3CNMUONMT1L/gHrgPbAIhHZALQCFohIM5t5lXf1PWMsK8O60j71SfZmZ9mO45dSP3uKJuyi4IQHdOtWqWoI+LnGGLPYGNPEGNPOGNMOSAf6GmP0TuVBJCQ0lJAznyLW7GH5+3qR/arK2bOLLmteJy2iHz2O1TOTlaqOgC+4ShXrlHgcyfHnkLRjit5NqIqWTH6EWHKod5r+plmp6gq6gutu6e60nUPZ0fWSJ8mWBhR+eZueQFVJu3Zsodem91gQNZhOicfZjqOU3wq6gquCW0xsHOv63EOXgpUkf/6s7Th+YdXkh4jgEI3P+o/tKEr5NS24KugknXUdy+ok0HnJf9mdudV2HJ+WvmYJfbdPJqXR6bTtkmg7jlJ+TQuuCjoSEkK9s5+mvjnI6g9utx3Hp2V+dhcFhNHh/MdsR1HK72nBVUGpfff+pDQfxdG7p7P0t+m24/ikJb99SZ8Dv5HW/mriWrS1HUcpv6cFVwWt3pc9wRZpSsys/+Pg/hzbcXxKYUEB9X54gG3Ek3jBONtxlAoIWnBV0KpXP5rdQ/9LK7ONRZP0NnOeFkx7kY6F60jvfzcRkVG24ygVELTgqqDW89izmNdoOP23fsCqBT/ZjuMT9u3dTfu0/7EirBv9hl1lO45SAUMLrgp63UY/Q5bEEv7VTeQdyrUdx7ol791NI7OHkNMf10s4KlWDwmwH8Df5+fmkp6eTm6sL5uqIiIigVatWhIeH245yWIOGjVl33GMk/nItc967l0FX/dd2JGvWLPqN/ts/YX7cCAb0Pd52HKUCihbcKkpPTyc6Opp27dohIrbj+BVjDFlZWaSnp9O+fXvbcf4i8aRRzE+bQv9Nb7Iy+Qy6JJ1oO5LXFRYUUPTlrWRLA7pe8pTtOEoFHN1fVEW5ubk0btxYi201iAiNGzf22b0DXa6YyE5pTP3pN7A/J9t2HK9L/uxpOhesYn3fccQ00ltTKlXTtOBWgxbb6vPltmvQsDFZpzxHi6JtLHnrJttxvGrntk10W/Y/ltRNpN+ZY23HUSogacFVykOPY05nXotLGLBrGqmzPrQdxytMURGb372WuiafBuc+qydKKVVLdM4KAlOnTmXZsmVe/9zx48fz1FP+dyyw75gnWRvanja/3kVmxgbbcWpdylev0ufA7yzsdCNtOifajqNUwNKCGwRqsuAWFBTUyHB8Wd2ISMLOf5MIc4jMty6hID/PdqRaszNjI50W/JsV4d3pP+o+23GUCmh6lvIReOjLpSzL2Fujw+zeogEPntWjwv7OPvtsNm/eTG5uLrfccgtjx44lKiqKffv2ATB58mS++uorxo4dy7Rp0/jpp594+OGHmTJlCjk5OVx33XUcOHCAjh078uabbxIbG8v8+fO56qqrCAkJ4eSTT+abb75hyZIlvP3223z22Wfs27ePwsJCpk+fzogRI9i9ezf5+fk8/PDDjBgxAoBHHnmEd955hyZNmtC6dWv69etXo+3jLW279iU56T8kpdzFnDdvY9C1L9qOVONMURHp711LV5NH/QsmEhqmiwOlapNu4fqpN998k5SUFJKTk3nuuefIysoqtb9jjjmG4cOH8+STT5KamkrHjh0ZPXo0TzzxBGlpaSQkJPDQQw8BcMUVV/DKK6+QmppKaGjoX4azYMECJk+ezE8//URERASff/45CxYs4Mcff+T222/HGENKSgofffQRqampfP3118yfP7/W26E2JZ11LfMan82gre+ROvMD23Fq3B+fTiDxwBxSu9xK6069bcdRKuAFzSqtiNwE/BMoBKYbY+460mFWZku0tjz33HN8/vnnAGzevJnVq1dX6n179uwhOzubIUOGADBmzBjOP/98srOzycnJYdCgQQBcfPHFfPXVV4ffd/LJJ9OoUSPA+T3tuHHj+PnnnwkJCWHLli1s376dX375hXPOOYfIyEgAhg8fXmPja0vvq19izVOL6fDbHWzp2IeWHbrZjlQj1qb9Tp9lT7IocgADRunNCZTyhqDYwhWRE4ARQG9jTA/A/87k8TB79mxmzZrFnDlzWLRoEX369CE3N/cvP7mp6d+61q9f//Dj999/n8zMTFJSUkhNTaVp06Y++9vaIxVRrz6Rl36AAfLfO5+92aXvSfAn+3OyCZ96NdnSgNZXvKVnJSvlJcEyp10PPG6MOQRgjNlhOc8R2bNnD7GxsURGRrJixQrmzp0LQNOmTVm+fDlFRUWHt34BoqOjyclxbj8XExNDbGwsv/zyCwCTJk1iyJAhNGzYkOjoaObNmwfARx99VO7nN2nShPDwcH788Uc2btwIwODBg5k6dSoHDx4kJyeHL7/8slbG39tatO/K5pNfoWVhBhsmXujXJ1GZoiKWv3Y1rQoz2HHyczRq0tJ2JKWCRrAU3M7AcSIyT0R+EpH+ZfUoImNFJFlEkjMzM70YsfJOO+00CgoK6NatG/fccw8DBw4E4PHHH+fMM8/kmGOOoXnz5of7HzVqFE8++SR9+vRh7dq1vPPOO9x555306tWL1NRUHnjgAQDeeOMNrrnmGhITE9m/fz8xMTGlfv4ll1xCcnIyCQkJvPvuu3Tt2hWAvn37cuGFF9K7d2+GDRtG//5lNrPf6XnsWSxMuI9eufNJfu2ftuNU27wP/k3S3pnMa3ctPY89y3YcpYKKGGNsZ6gRIjILaFZKp3uBR4AfgZuB/sDHQAdTwcgnJSWZ5OTkv7y2fPlyunULjON4Je3bt4+oKOfep48//jhbt27l2WefrfHP8ec2nPvSWAbu+Ji5Xe5i4EX32o5TJWmzp9Djx6tYFHUcif83lZASJ8YpVVNEJMUYk2Q7h68JmJOmjDFDy+omItcDn7kF9g8RKQLiAN/chLVk+vTpPPbYYxQUFNC2bVvefvtt25F8Tv+xL7Hwf+kMXDmB5GmNSBp+ve1IlbJ59SLazb6JjaFt6XLdJC22SlkQMAW3AlOBE4AfRaQzUAfYaTWRD7rwwgu58MILbcfwaaFhYXS76VOWPj2MxJRxpNaPJfGkUbZjlSszYwNh759LAaFEjP6E+tENbUdSKigFyzHcN4EOIrIE+AgYU9HuZKXKElGvPm1v/IL14R3p+vONpM2eYjtSmfbsymTf68NpYHLIOvsDWrTrYjuSUkErKAquMSbPGHOpMaanMaavMeYH25mUf4tqEEvctV+yJaw1XX8cy8Lv3rMd6W9y9uxi68vDaVm4hfVDX6VT4nG2IykV1IKi4CpVG2LjmxP3z+/YEN6Bnr/dTMr0121HOmzPrky2Pn8qHfNWsvSY/9HzuBG2IykV9LTgKnUEYhrF0/ymGayu041+829n7rv3Y4qKrGbK2p5O5oun0i5/HUv/8QJ9Th1jNY9SyqEFN4i9/fbb3HjjjRX2k5GRcfj51VdfbeVWf74sOqYRHW6bQUr0CQxc9xzzn7uEvEN2rry1ftl8Dk08gVYFm1h+/EQST77YSg6l1N9pwVXlKllwX3/9dbp3724xkW+KiIyiz61TmNPqSo7O/poNTx7HlnXLvZph0Q8fEf/xWYSbfDaNmELvE8736ucrpcoXLD8Lqh3f3APbFtfsMJslwLDHK+ytrNvz3XLLLXz11VfUq1ePL774gqZNm/Lll1/y8MMPk5eXR+PGjXn//fdp2rTp4WHl5OTQq1cvVq1aRXh4OHv37qV3795MmDCB5ORkLrnkEurVq8ecOXMYNmwYTz31FElJSXz77beMGzeOwsJC4uLi+P7772u2LfxMSGgog65+mgUz+nLUnLuRd04gud94+p05tlavV3wo9wAL37yVgTs+Zk1YR6Kv+JTOrTrW2ucppapHt3D9VGm359u/fz8DBw5k0aJFDB48mNdeew2Af/zjH8ydO5eFCxcyatQoJkyY8JdhRUdHc/zxxzN9+nTAuY7yyJEjOf/880lKSuL9998nNTWVevXqHX5PZmYm11xzDVOmTGHRokV8+umn3ht5H9f31MvYf8WPbKnTnqQFd7N4wlC2rFtaK5+1fN4MMiYMZOCOj5kXdy6t7viFplpslfJJuoV7JCqxJVpbSrs9X506dTjzzDMB6NevHzNnzgQgPT2dCy+8kK1bt5KXl0f79u3/Nryrr76aCRMmcPbZZ/PWW28dLtZlmTt3LoMHDz48rOJb9ylH87ZdaHL3L8yb/CQ9lj9L+DtDmNvsXDqNvJ/GTVsd8fC3rFtOxhcP0n/PDLYRR+pxrzDAxy/AoVSw0y1cP1TW7fnCw8MP36IvNDSUgoICAG666SZuvPFGFi9ezCuvvFLqrfSOPfZYNmzYwOzZsyksLKRnz55eHadAFBoWxoBR/+LA2DmkNTyJ/ts+pt5LfZn3/GjWpv1e5eGZoiJW/DGT+U9fQNN3jqFX9g/MaX4p0ben+PzVrpRSuoXrl8q6PV95/bds6dyG7Z133imzv9GjR3PxxRdz//33H37N89Z+ngYOHMgNN9zA+vXrad++Pbt27dKt3DI0admeJrd9zKZVqWz/+nF67/yaiM++YOPUVmxtMoSIToNp3qU/TVq0/8ux3qLCQnZkrGfbqmRyV/1Iyx0/09VkcMDUJbnp+XQ8exyDWrSzN2JKqSrRguuHTjvtNCZOnEi3bt3o0qXL4dvzlWX8+PGcf/75xMbGcuKJJ7J+/fpS+7vkkku47777uOiiiw6/dvnll3PdddcdPmmqWHx8PK+++iojR46kqKiIJk2aHN6FrUrXpnMibTp/xJ5dmSz67nUi18+g79aPqLPtffgFDplw9kg0eVKXOuYQ0WYfzSSPZjjdVkX0ZFuX6+g2dDQDG8TaHh2lVBUFzO35akOw3Z5v8uTJfPHFF0yaNKlWPyeQ27Cq9udks3nZH+xZn4LJ3kxI7m5CCg9RFFqXoroxSNxRRLXsTsc+Q4ioV992XKUqRW/PVzrdwlWAc5z3m2++4euvv7YdJajUj25I1wGnwIBTbEdRStUyLbgKgOeff952BKWUCmh6lnI16G746tO2U0oFKy24VRQREUFWVpYWjmowxpCVlUVERITtKEop5XW6S7mKWrVqRXp6OpmZmbaj+KWIiAhatTryCz8opZS/0YJbReHh4aVeqUkppZQqj+5SVkoppbxAC65SSinlBVpwlVJKKS/QK02VQ0QygY3VfHscsLMG4/gDHefgoOMcHI5knNsaY+JrMkwg0IJbS0QkOdgubabjHBx0nINDMI5zbdNdykoppZQXaMFVSimlvEALbu151XYAC3Scg4OOc3AIxnGuVXoMVymllPIC3cJVSimlvEALrlJKKeUFWnCPkIicJiIrRWSNiNxTSve6IvKx232eiLSzELNGVWKc/09ElolImoh8LyJtbeSsSRWNs0d/54qIERG//zlFZcZZRC5wv+ulIvKBtzPWpEpM121E5EcRWehO26fbyFmTRORNEdkhIkvK6C4i8pzbJmki0tfbGQOKMUb/qvkHhAJrgQ5AHWAR0L1EPzcAE93Ho4CPbef2wjifAES6j68PhnF2+4sGfgbmAkm2c3vhe+4ELARi3edNbOeu5fF9Fbjefdwd2GA7dw2M92CgL7CkjO6nA98AAgwE5tnO7M9/uoV7ZI4G1hhj1hlj8oCPgBEl+hkBvOM+ngycJCLixYw1rcJxNsb8aIw54D6dC/j7/fgq8z0D/Ad4Asj1ZrhaUplxvgZ40RizG8AYs8PLGWtSZcbXAA3cxzFAhhfz1QpjzM/ArnJ6GQG8axxzgYYi0tw76QKPFtwj0xLY7PE83X2t1H6MMQXAHqCxV9LVjsqMs6ercNaQ/VmF4+zuamttjJnuzWC1qDLfc2egs4j8JiJzReQ0r6WreZUZ3/HApSKSDnwN3OSdaFZVdX5X5dD74apaIyKXAknAENtZapOIhAD/Ay63HMXbwnB2Kx+PsxfjZxFJMMZk2wxViy4C3jbG/FdEBgGTRKSnMabIdjDlH3QL98hsAVp7PG/lvlZqPyIShrMrKssr6WpHZcYZERkK3AsMN8Yc8lK22lLROEcDPYHZIrIB51jXND8/caoy33M6MM0Yk2+MWQ+swinA/qgy43sV8AmAMWYOEIFzgf9AVqn5XVWOFtwjMx/oJCLtRaQOzklR00r0Mw0Y4z4+D/jBuGcj+KkKx1lE+gCv4BRbfz6uV6zccTbG7DHGxBlj2hlj2uEctx5ujEm2E7dGVGbanoqzdYuIxOHsYl7nxYw1qTLjuwk4CUBEuuEU3EyvpvS+acBo92zlgcAeY8xW26H8le5SPgLGmAIRuRGYgXOW45vGmKUi8m8g2RgzDXgDZ9fTGpyTE0bZS3zkKjnOTwJRwKfu+WGbjDHDrYU+QpUc54BSyXGeAZwiIsuAQuBOY4xf7r2p5PjeDrwmIrfhnEB1uZ+vPCMiH+KsNMW5x6YfBMIBjDETcY5Vnw6sAQ4AV9hJGhj00o5KKaWUF+guZaWUUsoLtOAqpZRSXqAFVymllPICLbhKKaWUF2jBVUoppbxAC65SSinlBVpwlVJKKS/QgquUHxGR/u59SSNEpL57H9qetnMppSqmF75Qys+IyMM4lxWsB6QbYx6zHEkpVQlacJXyM+61fufj3Hf3GGNMoeVISqlK0F3KSvmfxjjXqo7G2dJVSvkB3cJVys+IyDTgI6A90NwYc6PlSEqpStC7BSnlR0RkNJBvjPlAREKB30XkRGPMD7azKaXKp1u4SimllBfoMVyllFLKC7TgKqWUUl6gBVcppZTyAi24SimllBdowVVKKaW8QAuuUkop5QVacJVSSikv+H/fAuFxNTDSQAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The max absolute difference is: 1.77636e-15\n" - ] - } - ], + "execution_count": 4, + "id": "95529755", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], "source": [ "import autograd.numpy as np\n", "\n", @@ -1073,8 +1160,10 @@ }, { "cell_type": "markdown", - "id": "1ebd2eda", - "metadata": {}, + "id": "55c4b9d9", + "metadata": { + "editable": true + }, "source": [ "## Using autograd\n", "\n", @@ -1087,19 +1176,13 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "bd61cdf6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The gradient of f1 evaluated at a = 1 using autograd is: 3\n", - "The gradient of f1 evaluated at a = 1 by finding the analytic expression is: 3\n" - ] - } - ], + "execution_count": 5, + "id": "93380f99", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], "source": [ "import autograd.numpy as np\n", "from autograd import grad\n", @@ -1122,8 +1205,10 @@ }, { "cell_type": "markdown", - "id": "02a4c4b9", - "metadata": {}, + "id": "11db92cf", + "metadata": { + "editable": true + }, "source": [ "## Autograd with more complicated functions\n", "\n", @@ -1135,8 +1220,11 @@ { "cell_type": "code", "execution_count": 6, - "id": "cc20df7d", - "metadata": {}, + "id": "eba3b5b0", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1176,16 +1264,20 @@ }, { "cell_type": "markdown", - "id": "95b76c23", - "metadata": {}, + "id": "ca4cdea0", + "metadata": { + "editable": true + }, "source": [ "Note that the grad function will not produce the true gradient of the function. The true gradient of a function with two or more variables will produce a vector, where each element is the function differentiated w.r.t a variable." ] }, { "cell_type": "markdown", - "id": "9a7c62d5", - "metadata": {}, + "id": "c2beffc7", + "metadata": { + "editable": true + }, "source": [ "## More complicated functions using the elements of their arguments directly" ] @@ -1193,8 +1285,11 @@ { "cell_type": "code", "execution_count": 7, - "id": "2f836f3c", - "metadata": {}, + "id": "6ec33241", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1218,8 +1313,10 @@ }, { "cell_type": "markdown", - "id": "c3b33087", - "metadata": {}, + "id": "d69be5df", + "metadata": { + "editable": true + }, "source": [ "Note that in this case, when sending an array as input argument, the\n", "output from Autograd is another array. This is the true gradient of\n", @@ -1231,8 +1328,10 @@ }, { "cell_type": "markdown", - "id": "87ad6d05", - "metadata": {}, + "id": "a2fa242b", + "metadata": { + "editable": true + }, "source": [ "## Functions using mathematical functions from Numpy" ] @@ -1240,8 +1339,11 @@ { "cell_type": "code", "execution_count": 8, - "id": "72cd7440", - "metadata": {}, + "id": "0befa3ff", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1265,8 +1367,10 @@ }, { "cell_type": "markdown", - "id": "76ff845c", - "metadata": {}, + "id": "a22bc18d", + "metadata": { + "editable": true + }, "source": [ "## More autograd" ] @@ -1274,8 +1378,11 @@ { "cell_type": "code", "execution_count": 9, - "id": "90719a0e", - "metadata": {}, + "id": "e0acdb7c", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1296,8 +1403,10 @@ }, { "cell_type": "markdown", - "id": "0009d423", - "metadata": {}, + "id": "e788436c", + "metadata": { + "editable": true + }, "source": [ "## And with loops" ] @@ -1305,8 +1414,11 @@ { "cell_type": "code", "execution_count": 10, - "id": "dacfa26f", - "metadata": {}, + "id": "d15a1cd7", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1338,8 +1450,11 @@ { "cell_type": "code", "execution_count": 11, - "id": "81789038", - "metadata": {}, + "id": "25f2c928", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1355,8 +1470,10 @@ }, { "cell_type": "markdown", - "id": "73769c7f", - "metadata": {}, + "id": "1e108150", + "metadata": { + "editable": true + }, "source": [ "## Using recursion" ] @@ -1364,8 +1481,11 @@ { "cell_type": "code", "execution_count": 12, - "id": "8a023f81", - "metadata": {}, + "id": "98686ee1", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1399,16 +1519,20 @@ }, { "cell_type": "markdown", - "id": "88c98aa0", - "metadata": {}, + "id": "95f2045f", + "metadata": { + "editable": true + }, "source": [ "Note that if n is equal to zero or one, Autograd will give an error message. This message appears when the output is independent on input." ] }, { "cell_type": "markdown", - "id": "2cb88a5d", - "metadata": {}, + "id": "3107dae1", + "metadata": { + "editable": true + }, "source": [ "## Unsupported functions\n", "Autograd supports many features. However, there are some functions that is not supported (yet) by Autograd.\n", @@ -1419,8 +1543,11 @@ { "cell_type": "code", "execution_count": 13, - "id": "ed3ac3e3", - "metadata": {}, + "id": "1bbc93e8", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1438,16 +1565,20 @@ }, { "cell_type": "markdown", - "id": "452ee1c7", - "metadata": {}, + "id": "344beeca", + "metadata": { + "editable": true + }, "source": [ "Here, Autograd tells us that an 'ArrayBox' does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible." ] }, { "cell_type": "markdown", - "id": "492c3948", - "metadata": {}, + "id": "d2b53241", + "metadata": { + "editable": true + }, "source": [ "## The syntax a.dot(b) when finding the dot product" ] @@ -1455,8 +1586,11 @@ { "cell_type": "code", "execution_count": 14, - "id": "5c0be95c", - "metadata": {}, + "id": "8839a3a8", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1474,8 +1608,10 @@ }, { "cell_type": "markdown", - "id": "878ebbaa", - "metadata": {}, + "id": "c5096993", + "metadata": { + "editable": true + }, "source": [ "Here we are told that the 'dot' function does not belong to Autograd's\n", "version of a Numpy array. To overcome this, an alternative syntax\n", @@ -1485,8 +1621,11 @@ { "cell_type": "code", "execution_count": 15, - "id": "35744b22", - "metadata": {}, + "id": "9db83bcc", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1507,8 +1646,10 @@ }, { "cell_type": "markdown", - "id": "7155b378", - "metadata": {}, + "id": "df3a3262", + "metadata": { + "editable": true + }, "source": [ "## Recommended to avoid\n", "The documentation recommends to avoid inplace operations such as" @@ -1517,8 +1658,11 @@ { "cell_type": "code", "execution_count": 16, - "id": "263ae33c", - "metadata": {}, + "id": "3de46d8a", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "a += b\n", @@ -1529,8 +1673,10 @@ }, { "cell_type": "markdown", - "id": "21aa6af8", - "metadata": {}, + "id": "407ca258", + "metadata": { + "editable": true + }, "source": [ "## Using Autograd with OLS\n", "\n", @@ -1541,22 +1687,13 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "54f02097", - "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'autograd'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mrandom\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mautograd\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'autograd'" - ] - } - ], + "execution_count": 17, + "id": "a96ae442", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], "source": [ "# Using Autograd to calculate gradients for OLS\n", "from random import random, seed\n", @@ -1611,8 +1748,10 @@ }, { "cell_type": "markdown", - "id": "9e6d50fa", - "metadata": {}, + "id": "b9e24580", + "metadata": { + "editable": true + }, "source": [ "## Including Stochastic Gradient Descent with Autograd\n", "In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using **autograd**." @@ -1620,22 +1759,13 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "95265123", - "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'autograd'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mrandom\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mautograd\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'autograd'" - ] - } - ], + "execution_count": 18, + "id": "4eb94f4f", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], "source": [ "# Using Autograd to calculate gradients using SGD\n", "# OLS example\n", @@ -1702,9 +1832,9 @@ "for epoch in range(n_epochs):\n", "# Can you figure out a better way of setting up the contributions to each batch?\n", " for i in range(m):\n", - " random_index = np.random.randint(m)\n", - " xi = X[random_index*M:random_index*M+M]\n", - " yi = y[random_index*M:random_index*M+M]\n", + " random_index = M*np.random.randint(m)\n", + " xi = X[random_index:random_index+M]\n", + " yi = y[random_index:random_index+M]\n", " gradients = (2.0/M)*training_gradient(yi, xi, theta)\n", " eta = learning_schedule(epoch*m+i)\n", " theta = theta - eta*gradients\n", @@ -1714,8 +1844,10 @@ }, { "cell_type": "markdown", - "id": "f136ad8d", - "metadata": {}, + "id": "f669467a", + "metadata": { + "editable": true + }, "source": [ "## And Logistic Regression" ] @@ -1723,8 +1855,11 @@ { "cell_type": "code", "execution_count": 19, - "id": "c565eefd", - "metadata": {}, + "id": "28babb1b", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import autograd.numpy as np\n", @@ -1764,8 +1899,10 @@ }, { "cell_type": "markdown", - "id": "e8994c9d", - "metadata": {}, + "id": "4958f8c1", + "metadata": { + "editable": true + }, "source": [ "## Videos on Neural Networks\n", "\n", @@ -1776,8 +1913,10 @@ }, { "cell_type": "markdown", - "id": "b60dc921", - "metadata": {}, + "id": "59db6bbd", + "metadata": { + "editable": true + }, "source": [ "## Neural networks\n", "\n", @@ -1792,8 +1931,10 @@ }, { "cell_type": "markdown", - "id": "fd7c1d6b", - "metadata": {}, + "id": "ea10c766", + "metadata": { + "editable": true + }, "source": [ "## Artificial neurons\n", "\n", @@ -1814,8 +1955,10 @@ }, { "cell_type": "markdown", - "id": "d34115ec", - "metadata": {}, + "id": "1f7dace3", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -1830,8 +1973,10 @@ }, { "cell_type": "markdown", - "id": "c5e82cb4", - "metadata": {}, + "id": "423135f6", + "metadata": { + "editable": true + }, "source": [ "Here, the output $y$ of the neuron is the value of its activation function, which have as input\n", "a weighted sum of signals $x_i, \\dots ,x_n$ received by $n$ other neurons.\n", @@ -1868,8 +2013,10 @@ }, { "cell_type": "markdown", - "id": "153d950b", - "metadata": {}, + "id": "2d9ca81c", + "metadata": { + "editable": true + }, "source": [ "## Neural network types\n", "\n", @@ -1895,8 +2042,10 @@ }, { "cell_type": "markdown", - "id": "9ca9a5c9", - "metadata": {}, + "id": "ca5fd1e4", + "metadata": { + "editable": true + }, "source": [ "## Feed-forward neural networks\n", "\n", @@ -1914,8 +2063,10 @@ }, { "cell_type": "markdown", - "id": "dceb31cf", - "metadata": {}, + "id": "2422d4f2", + "metadata": { + "editable": true + }, "source": [ "## Convolutional Neural Network\n", "\n", @@ -1941,8 +2092,10 @@ }, { "cell_type": "markdown", - "id": "04880cf8", - "metadata": {}, + "id": "6841ded4", + "metadata": { + "editable": true + }, "source": [ "## Recurrent neural networks\n", "\n", @@ -1960,8 +2113,10 @@ }, { "cell_type": "markdown", - "id": "a9312de2", - "metadata": {}, + "id": "5e8ecbf3", + "metadata": { + "editable": true + }, "source": [ "## Other types of networks\n", "\n", @@ -1979,8 +2134,10 @@ }, { "cell_type": "markdown", - "id": "d2597fdc", - "metadata": {}, + "id": "94d4fd33", + "metadata": { + "editable": true + }, "source": [ "## Multilayer perceptrons\n", "\n", @@ -1994,8 +2151,10 @@ }, { "cell_type": "markdown", - "id": "7f401b4d", - "metadata": {}, + "id": "7fb176d9", + "metadata": { + "editable": true + }, "source": [ "## Why multilayer perceptrons?\n", "\n", @@ -2013,8 +2172,10 @@ }, { "cell_type": "markdown", - "id": "c8f771bf", - "metadata": {}, + "id": "a6d5a606", + "metadata": { + "editable": true + }, "source": [ "## Illustration of a single perceptropn model and a multi-perceptron model\n", "\n", @@ -2027,8 +2188,10 @@ }, { "cell_type": "markdown", - "id": "f1be67dd", - "metadata": {}, + "id": "e2b42e71", + "metadata": { + "editable": true + }, "source": [ "## Examples of XOR, OR and AND gates\n", "\n", @@ -2042,8 +2205,11 @@ { "cell_type": "code", "execution_count": 20, - "id": "bf659192", - "metadata": {}, + "id": "30bc30e7", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\"\"\"\n", @@ -2080,16 +2246,20 @@ }, { "cell_type": "markdown", - "id": "d96eb58b", - "metadata": {}, + "id": "d2d04274", + "metadata": { + "editable": true + }, "source": [ "What is happening here?" ] }, { "cell_type": "markdown", - "id": "855e5b9d", - "metadata": {}, + "id": "9020b30b", + "metadata": { + "editable": true + }, "source": [ "## Does Logistic Regression do a better Job?" ] @@ -2097,8 +2267,11 @@ { "cell_type": "code", "execution_count": 21, - "id": "9f7fb553", - "metadata": {}, + "id": "5e66d354", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\"\"\"\n", @@ -2154,16 +2327,20 @@ }, { "cell_type": "markdown", - "id": "97a10f63", - "metadata": {}, + "id": "663371a3", + "metadata": { + "editable": true + }, "source": [ "Not exactly impressive, but somewhat better." ] }, { "cell_type": "markdown", - "id": "8bef3dd5", - "metadata": {}, + "id": "2e4b8494", + "metadata": { + "editable": true + }, "source": [ "## Adding Neural Networks" ] @@ -2171,8 +2348,11 @@ { "cell_type": "code", "execution_count": 22, - "id": "0217c385", - "metadata": {}, + "id": "1814b2ea", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\n", @@ -2188,8 +2368,10 @@ }, { "cell_type": "markdown", - "id": "dce7493f", - "metadata": {}, + "id": "d986a868", + "metadata": { + "editable": true + }, "source": [ "## Mathematical model\n", "\n", @@ -2198,8 +2380,10 @@ }, { "cell_type": "markdown", - "id": "45794322", - "metadata": {}, + "id": "709c9dea", + "metadata": { + "editable": true + }, "source": [ "$$\n", "y = f\\left(\\sum_{i=1}^n w_ix_i + b_i\\right) = f(z),\n", @@ -2208,8 +2392,10 @@ }, { "cell_type": "markdown", - "id": "7a9de81a", - "metadata": {}, + "id": "4254b0ab", + "metadata": { + "editable": true + }, "source": [ "This function receives $x_i$ as inputs.\n", "Here the activation $z=(\\sum_{i=1}^n w_ix_i+b_i)$. \n", @@ -2221,8 +2407,10 @@ }, { "cell_type": "markdown", - "id": "fdda1f07", - "metadata": {}, + "id": "8857a349", + "metadata": { + "editable": true + }, "source": [ "## Mathematical model\n", "\n", @@ -2231,8 +2419,10 @@ }, { "cell_type": "markdown", - "id": "9524f638", - "metadata": {}, + "id": "4c150122", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2246,8 +2436,10 @@ }, { "cell_type": "markdown", - "id": "a0e26b0f", - "metadata": {}, + "id": "5cd9f1e4", + "metadata": { + "editable": true + }, "source": [ "Here $b_i$ is the so-called bias which is normally needed in\n", "case of zero activation weights or inputs. How to fix the biases and\n", @@ -2259,8 +2451,10 @@ }, { "cell_type": "markdown", - "id": "087299fd", - "metadata": {}, + "id": "085ba7ef", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2275,8 +2469,10 @@ }, { "cell_type": "markdown", - "id": "b07ec0ac", - "metadata": {}, + "id": "94102dba", + "metadata": { + "editable": true + }, "source": [ "where we assume that all nodes in the same layer have identical\n", "activation functions, hence the notation $f$. In general, we could assume in the more general case that different layers have different activation functions.\n", @@ -2285,8 +2481,10 @@ }, { "cell_type": "markdown", - "id": "9eb05c72", - "metadata": {}, + "id": "f7be1a6a", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2301,8 +2499,10 @@ }, { "cell_type": "markdown", - "id": "0fae4fc2", - "metadata": {}, + "id": "c62cd931", + "metadata": { + "editable": true + }, "source": [ "where $N_l$ is the number of nodes in layer $l$. When the output of\n", "all the nodes in the first hidden layer are computed, the values of\n", @@ -2312,8 +2512,10 @@ }, { "cell_type": "markdown", - "id": "9ecda7bb", - "metadata": {}, + "id": "5ba6e940", + "metadata": { + "editable": true + }, "source": [ "## Mathematical model\n", "\n", @@ -2322,8 +2524,10 @@ }, { "cell_type": "markdown", - "id": "d1023fd9", - "metadata": {}, + "id": "df6992b9", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2338,8 +2542,10 @@ }, { "cell_type": "markdown", - "id": "4982f798", - "metadata": {}, + "id": "04fd00e7", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2354,16 +2560,20 @@ }, { "cell_type": "markdown", - "id": "9fb25411", - "metadata": {}, + "id": "0de91466", + "metadata": { + "editable": true + }, "source": [ "where we have substituted $y_k^1$ with the inputs $x_k$. Finally, the ANN output reads" ] }, { "cell_type": "markdown", - "id": "7607d280", - "metadata": {}, + "id": "9eb9f176", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2378,8 +2588,10 @@ }, { "cell_type": "markdown", - "id": "6356f959", - "metadata": {}, + "id": "f61adc0b", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2395,8 +2607,10 @@ }, { "cell_type": "markdown", - "id": "c5e33d21", - "metadata": {}, + "id": "7c0aad8c", + "metadata": { + "editable": true + }, "source": [ "## Mathematical model\n", "\n", @@ -2406,8 +2620,10 @@ }, { "cell_type": "markdown", - "id": "e86b6000", - "metadata": {}, + "id": "ee2df9a8", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2422,8 +2638,10 @@ }, { "cell_type": "markdown", - "id": "7813cdda", - "metadata": {}, + "id": "a7e1cbb6", + "metadata": { + "editable": true + }, "source": [ "which illustrates a basic property of MLPs: The only independent\n", "variables are the input values $x_n$." @@ -2431,8 +2649,10 @@ }, { "cell_type": "markdown", - "id": "4b6e2c05", - "metadata": {}, + "id": "498e9724", + "metadata": { + "editable": true + }, "source": [ "## Mathematical model\n", "\n", @@ -2448,8 +2668,10 @@ }, { "cell_type": "markdown", - "id": "8d21fc10", - "metadata": {}, + "id": "00817577", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2464,8 +2686,10 @@ }, { "cell_type": "markdown", - "id": "60f584ec", - "metadata": {}, + "id": "324012a8", + "metadata": { + "editable": true + }, "source": [ "where the parameters $c_i$ are weights and biases. By adjusting these\n", "parameters, the activation functions can be shifted up and down or\n", @@ -2475,8 +2699,10 @@ }, { "cell_type": "markdown", - "id": "ae5c91ab", - "metadata": {}, + "id": "f3d107a5", + "metadata": { + "editable": true + }, "source": [ "### Matrix-vector notation\n", "\n", @@ -2493,8 +2719,10 @@ }, { "cell_type": "markdown", - "id": "6a4966cf", - "metadata": {}, + "id": "4e46d94b", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2524,8 +2752,10 @@ }, { "cell_type": "markdown", - "id": "c10d3258", - "metadata": {}, + "id": "60c72663", + "metadata": { + "editable": true + }, "source": [ "### Matrix-vector notation and activation\n", "\n", @@ -2534,8 +2764,10 @@ }, { "cell_type": "markdown", - "id": "a0822fc5", - "metadata": {}, + "id": "49b8e555", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2551,8 +2783,10 @@ }, { "cell_type": "markdown", - "id": "344cc850", - "metadata": {}, + "id": "fb36a61b", + "metadata": { + "editable": true + }, "source": [ "This is not just a convenient and compact notation, but also a useful\n", "and intuitive way to think about MLPs: The output is calculated by a\n", @@ -2563,8 +2797,10 @@ }, { "cell_type": "markdown", - "id": "5cf49f38", - "metadata": {}, + "id": "f6b8fdcc", + "metadata": { + "editable": true + }, "source": [ "### Activation functions\n", "\n", @@ -2584,8 +2820,10 @@ }, { "cell_type": "markdown", - "id": "a8d8a0c4", - "metadata": {}, + "id": "985d1cf2", + "metadata": { + "editable": true + }, "source": [ "### Activation functions, Logistic and Hyperbolic ones\n", "\n", @@ -2601,8 +2839,10 @@ }, { "cell_type": "markdown", - "id": "f4039df2", - "metadata": {}, + "id": "9cf30088", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f(x) = \\frac{1}{1 + e^{-x}},\n", @@ -2611,16 +2851,20 @@ }, { "cell_type": "markdown", - "id": "5bfe16d7", - "metadata": {}, + "id": "2536e875", + "metadata": { + "editable": true + }, "source": [ "and the *hyperbolic tangent* function" ] }, { "cell_type": "markdown", - "id": "9c0053a5", - "metadata": {}, + "id": "2428ac57", + "metadata": { + "editable": true + }, "source": [ "$$\n", "f(x) = \\tanh(x)\n", @@ -2629,8 +2873,10 @@ }, { "cell_type": "markdown", - "id": "ee719a5b", - "metadata": {}, + "id": "0429f882", + "metadata": { + "editable": true + }, "source": [ "### Relevance\n", "\n", @@ -2644,8 +2890,11 @@ { "cell_type": "code", "execution_count": 23, - "id": "ce0a726a", - "metadata": {}, + "id": "4b81f111", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\"\"\"The sigmoid function (or the logistic curve) is a \n", @@ -2723,8 +2972,10 @@ }, { "cell_type": "markdown", - "id": "67c1d4e9", - "metadata": {}, + "id": "5942f6ae", + "metadata": { + "editable": true + }, "source": [ "## The multilayer perceptron (MLP)\n", "\n", @@ -2759,8 +3010,10 @@ }, { "cell_type": "markdown", - "id": "b0f10db3", - "metadata": {}, + "id": "d8c8f241", + "metadata": { + "editable": true + }, "source": [ "## From one to many layers, the universal approximation theorem\n", "\n", @@ -2787,8 +3040,10 @@ }, { "cell_type": "markdown", - "id": "92203fb4", - "metadata": {}, + "id": "fbc4bb7f", + "metadata": { + "editable": true + }, "source": [ "## Deriving the back propagation code for a multilayer perceptron model\n", "\n", @@ -2806,8 +3061,10 @@ }, { "cell_type": "markdown", - "id": "6755430b", - "metadata": {}, + "id": "9007a7e1", + "metadata": { + "editable": true + }, "source": [ "$$\n", "{\\cal C}(\\hat{W}) = \\frac{1}{2}\\sum_{i=1}^n\\left(y_i - t_i\\right)^2,\n", @@ -2816,8 +3073,10 @@ }, { "cell_type": "markdown", - "id": "fb15dd2f", - "metadata": {}, + "id": "9416d184", + "metadata": { + "editable": true + }, "source": [ "where the $t_i$s are our $n$ targets (the values we want to\n", "reproduce), while the outputs of the network after having propagated\n", @@ -2829,8 +3088,10 @@ }, { "cell_type": "markdown", - "id": "50b70e8e", - "metadata": {}, + "id": "be124c7b", + "metadata": { + "editable": true + }, "source": [ "## Definitions\n", "\n", @@ -2844,8 +3105,10 @@ }, { "cell_type": "markdown", - "id": "0914d857", - "metadata": {}, + "id": "f900b0e2", + "metadata": { + "editable": true + }, "source": [ "$$\n", "z_j^l = \\sum_{i=1}^{M_{l-1}}w_{ij}^la_i^{l-1}+b_j^l,\n", @@ -2854,8 +3117,10 @@ }, { "cell_type": "markdown", - "id": "42ee574d", - "metadata": {}, + "id": "ae026bf6", + "metadata": { + "editable": true + }, "source": [ "where $b_k^l$ are the biases from layer $l$. Here $M_{l-1}$\n", "represents the total number of nodes/neurons/units of layer $l-1$. The\n", @@ -2865,8 +3130,10 @@ }, { "cell_type": "markdown", - "id": "74ab9769", - "metadata": {}, + "id": "bd53f785", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\hat{z}^l = \\left(\\hat{W}^l\\right)^T\\hat{a}^{l-1}+\\hat{b}^l.\n", @@ -2875,8 +3142,10 @@ }, { "cell_type": "markdown", - "id": "887bdbbf", - "metadata": {}, + "id": "82ab1706", + "metadata": { + "editable": true + }, "source": [ "With the activation values $\\hat{z}^l$ we can in turn define the\n", "output of layer $l$ as $\\hat{a}^l = f(\\hat{z}^l)$ where $f$ is our\n", @@ -2887,8 +3156,10 @@ }, { "cell_type": "markdown", - "id": "955a6aa0", - "metadata": {}, + "id": "0a64ba41", + "metadata": { + "editable": true + }, "source": [ "$$\n", "a_j^l = f(z_j^l) = \\frac{1}{1+\\exp{-(z_j^l)}}.\n", @@ -2897,8 +3168,10 @@ }, { "cell_type": "markdown", - "id": "cba6e7a3", - "metadata": {}, + "id": "7ef830c1", + "metadata": { + "editable": true + }, "source": [ "## Derivatives and the chain rule\n", "\n", @@ -2907,8 +3180,10 @@ }, { "cell_type": "markdown", - "id": "6974ae9d", - "metadata": {}, + "id": "5e41dbc7", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial z_j^l}{\\partial w_{ij}^l} = a_i^{l-1},\n", @@ -2917,16 +3192,20 @@ }, { "cell_type": "markdown", - "id": "fb9b70f8", - "metadata": {}, + "id": "6e9ee76a", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "e1c0e49d", - "metadata": {}, + "id": "8fc731d6", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial z_j^l}{\\partial a_i^{l-1}} = w_{ji}^l.\n", @@ -2935,16 +3214,20 @@ }, { "cell_type": "markdown", - "id": "9cf72b96", - "metadata": {}, + "id": "c235c716", + "metadata": { + "editable": true + }, "source": [ "With our definition of the activation function we have that (note that this function depends only on $z_j^l$)" ] }, { "cell_type": "markdown", - "id": "1740efe7", - "metadata": {}, + "id": "dd5cd724", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial a_j^l}{\\partial z_j^{l}} = a_j^l(1-a_j^l)=f(z_j^l)(1-f(z_j^l)).\n", @@ -2953,8 +3236,10 @@ }, { "cell_type": "markdown", - "id": "984a0649", - "metadata": {}, + "id": "6c3eba41", + "metadata": { + "editable": true + }, "source": [ "## Derivative of the cost function\n", "\n", @@ -2965,8 +3250,10 @@ }, { "cell_type": "markdown", - "id": "dd62eb2d", - "metadata": {}, + "id": "3bef1f38", + "metadata": { + "editable": true + }, "source": [ "$$\n", "{\\cal C}(\\hat{W^L}) = \\frac{1}{2}\\sum_{i=1}^n\\left(y_i - t_i\\right)^2=\\frac{1}{2}\\sum_{i=1}^n\\left(a_i^L - t_i\\right)^2,\n", @@ -2975,16 +3262,20 @@ }, { "cell_type": "markdown", - "id": "f6b0248e", - "metadata": {}, + "id": "2063e479", + "metadata": { + "editable": true + }, "source": [ "The derivative of this function with respect to the weights is" ] }, { "cell_type": "markdown", - "id": "70335126", - "metadata": {}, + "id": "32081e17", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial{\\cal C}(\\hat{W^L})}{\\partial w_{jk}^L} = \\left(a_j^L - t_j\\right)\\frac{\\partial a_j^L}{\\partial w_{jk}^{L}},\n", @@ -2993,16 +3284,20 @@ }, { "cell_type": "markdown", - "id": "674ed935", - "metadata": {}, + "id": "c4c03ac0", + "metadata": { + "editable": true + }, "source": [ "The last partial derivative can easily be computed and reads (by applying the chain rule)" ] }, { "cell_type": "markdown", - "id": "da979440", - "metadata": {}, + "id": "852f8a23", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial a_j^L}{\\partial w_{jk}^{L}} = \\frac{\\partial a_j^L}{\\partial z_{j}^{L}}\\frac{\\partial z_j^L}{\\partial w_{jk}^{L}}=a_j^L(1-a_j^L)a_k^{L-1},\n", @@ -3011,8 +3306,10 @@ }, { "cell_type": "markdown", - "id": "009e951d", - "metadata": {}, + "id": "77c328d5", + "metadata": { + "editable": true + }, "source": [ "## Bringing it together, first back propagation equation\n", "\n", @@ -3021,8 +3318,10 @@ }, { "cell_type": "markdown", - "id": "1d994773", - "metadata": {}, + "id": "cc4c3eb8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial{\\cal C}(\\hat{W^L})}{\\partial w_{jk}^L} = \\left(a_j^L - t_j\\right)a_j^L(1-a_j^L)a_k^{L-1},\n", @@ -3031,16 +3330,20 @@ }, { "cell_type": "markdown", - "id": "1e2a9586", - "metadata": {}, + "id": "c3185717", + "metadata": { + "editable": true + }, "source": [ "Defining" ] }, { "cell_type": "markdown", - "id": "e8f99c9e", - "metadata": {}, + "id": "62aad72f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^L = a_j^L(1-a_j^L)\\left(a_j^L - t_j\\right) = f'(z_j^L)\\frac{\\partial {\\cal C}}{\\partial (a_j^L)},\n", @@ -3049,16 +3352,20 @@ }, { "cell_type": "markdown", - "id": "a04289ae", - "metadata": {}, + "id": "80363a1c", + "metadata": { + "editable": true + }, "source": [ "and using the Hadamard product of two vectors we can write this as" ] }, { "cell_type": "markdown", - "id": "2ee50711", - "metadata": {}, + "id": "8681d308", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\hat{\\delta}^L = f'(\\hat{z}^L)\\circ\\frac{\\partial {\\cal C}}{\\partial (\\hat{a}^L)}.\n", @@ -3067,8 +3374,10 @@ }, { "cell_type": "markdown", - "id": "6edfc5dc", - "metadata": {}, + "id": "fbc68aef", + "metadata": { + "editable": true + }, "source": [ "This is an important expression. The second term on the right handside\n", "measures how fast the cost function is changing as a function of the $j$th\n", @@ -3089,8 +3398,10 @@ }, { "cell_type": "markdown", - "id": "46ee5244", - "metadata": {}, + "id": "d25efc85", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial {\\cal C}}{\\partial (a_j^L)}\n", @@ -3099,16 +3410,20 @@ }, { "cell_type": "markdown", - "id": "49d84243", - "metadata": {}, + "id": "0a9568a0", + "metadata": { + "editable": true + }, "source": [ "With the definition of $\\delta_j^L$ we have a more compact definition of the derivative of the cost function in terms of the weights, namely" ] }, { "cell_type": "markdown", - "id": "56613bb1", - "metadata": {}, + "id": "effdc44a", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial{\\cal C}(\\hat{W^L})}{\\partial w_{jk}^L} = \\delta_j^La_k^{L-1}.\n", @@ -3117,8 +3432,10 @@ }, { "cell_type": "markdown", - "id": "4e0c595d", - "metadata": {}, + "id": "8cdfa62d", + "metadata": { + "editable": true + }, "source": [ "## Derivatives in terms of $z_j^L$\n", "\n", @@ -3127,8 +3444,10 @@ }, { "cell_type": "markdown", - "id": "18d63f4f", - "metadata": {}, + "id": "842ac6aa", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^L =\\frac{\\partial {\\cal C}}{\\partial z_j^L}= \\frac{\\partial {\\cal C}}{\\partial a_j^L}\\frac{\\partial a_j^L}{\\partial z_j^L},\n", @@ -3137,16 +3456,20 @@ }, { "cell_type": "markdown", - "id": "83539a64", - "metadata": {}, + "id": "f4f40e9d", + "metadata": { + "editable": true + }, "source": [ "which can also be interpreted as the partial derivative of the cost function with respect to the biases $b_j^L$, namely" ] }, { "cell_type": "markdown", - "id": "3fadddb8", - "metadata": {}, + "id": "7c3f7692", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^L = \\frac{\\partial {\\cal C}}{\\partial b_j^L}\\frac{\\partial b_j^L}{\\partial z_j^L}=\\frac{\\partial {\\cal C}}{\\partial b_j^L},\n", @@ -3155,16 +3478,20 @@ }, { "cell_type": "markdown", - "id": "b570ccfb", - "metadata": {}, + "id": "38d73332", + "metadata": { + "editable": true + }, "source": [ "That is, the error $\\delta_j^L$ is exactly equal to the rate of change of the cost function as a function of the bias." ] }, { "cell_type": "markdown", - "id": "299cbb0c", - "metadata": {}, + "id": "32266730", + "metadata": { + "editable": true + }, "source": [ "## Bringing it together\n", "\n", @@ -3175,8 +3502,10 @@ }, { "cell_type": "markdown", - "id": "cd426846", - "metadata": {}, + "id": "90e5e922", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3191,16 +3520,20 @@ }, { "cell_type": "markdown", - "id": "ee9de6fc", - "metadata": {}, + "id": "55a6708f", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "5a3fb6d8", - "metadata": {}, + "id": "4958b0bc", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3215,16 +3548,20 @@ }, { "cell_type": "markdown", - "id": "e4d9983f", - "metadata": {}, + "id": "f7dab092", + "metadata": { + "editable": true + }, "source": [ "and" ] }, { "cell_type": "markdown", - "id": "df4bc05e", - "metadata": {}, + "id": "10787835", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -3239,8 +3576,10 @@ }, { "cell_type": "markdown", - "id": "5bc2cf34", - "metadata": {}, + "id": "a5349b20", + "metadata": { + "editable": true + }, "source": [ "An interesting consequence of the above equations is that when the\n", "activation $a_k^{L-1}$ is small, the gradient term, that is the\n", @@ -3264,8 +3603,10 @@ }, { "cell_type": "markdown", - "id": "50b8b1e2", - "metadata": {}, + "id": "ac5b120e", + "metadata": { + "editable": true + }, "source": [ "## Final back propagating equation\n", "\n", @@ -3274,8 +3615,10 @@ }, { "cell_type": "markdown", - "id": "bbde849a", - "metadata": {}, + "id": "5c822241", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^l =\\frac{\\partial {\\cal C}}{\\partial z_j^l}.\n", @@ -3284,16 +3627,20 @@ }, { "cell_type": "markdown", - "id": "16451372", - "metadata": {}, + "id": "15efeef8", + "metadata": { + "editable": true + }, "source": [ "We want to express this in terms of the equations for layer $l+1$. Using the chain rule and summing over all $k$ entries we have" ] }, { "cell_type": "markdown", - "id": "3918985e", - "metadata": {}, + "id": "f34ad50c", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^l =\\sum_k \\frac{\\partial {\\cal C}}{\\partial z_k^{l+1}}\\frac{\\partial z_k^{l+1}}{\\partial z_j^{l}}=\\sum_k \\delta_k^{l+1}\\frac{\\partial z_k^{l+1}}{\\partial z_j^{l}},\n", @@ -3302,16 +3649,20 @@ }, { "cell_type": "markdown", - "id": "89abd340", - "metadata": {}, + "id": "f3a4c387", + "metadata": { + "editable": true + }, "source": [ "and recalling that" ] }, { "cell_type": "markdown", - "id": "ebb68622", - "metadata": {}, + "id": "0b810b9a", + "metadata": { + "editable": true + }, "source": [ "$$\n", "z_j^{l+1} = \\sum_{i=1}^{M_{l}}w_{ij}^{l+1}a_i^{l}+b_j^{l+1},\n", @@ -3320,16 +3671,20 @@ }, { "cell_type": "markdown", - "id": "ed39bd8f", - "metadata": {}, + "id": "f7a3bf81", + "metadata": { + "editable": true + }, "source": [ "with $M_l$ being the number of nodes in layer $l$, we obtain" ] }, { "cell_type": "markdown", - "id": "86fe1bde", - "metadata": {}, + "id": "d599b0a2", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^l =\\sum_k \\delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l),\n", @@ -3338,8 +3693,10 @@ }, { "cell_type": "markdown", - "id": "4cc0b6fb", - "metadata": {}, + "id": "64cbb773", + "metadata": { + "editable": true + }, "source": [ "This is our final equation.\n", "\n", @@ -3348,8 +3705,10 @@ }, { "cell_type": "markdown", - "id": "9a75f32a", - "metadata": {}, + "id": "e69ed5e5", + "metadata": { + "editable": true + }, "source": [ "## Setting up the Back propagation algorithm\n", "\n", @@ -3369,8 +3728,10 @@ }, { "cell_type": "markdown", - "id": "44bc8455", - "metadata": {}, + "id": "bc52c09f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^L = f'(z_j^L)\\frac{\\partial {\\cal C}}{\\partial (a_j^L)}.\n", @@ -3379,16 +3740,20 @@ }, { "cell_type": "markdown", - "id": "d29e9ac2", - "metadata": {}, + "id": "05e6d24f", + "metadata": { + "editable": true + }, "source": [ "Then we compute the back propagate error for each $l=L-1,L-2,\\dots,2$ as" ] }, { "cell_type": "markdown", - "id": "1224c135", - "metadata": {}, + "id": "a33e6684", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\delta_j^l = \\sum_k \\delta_k^{l+1}w_{kj}^{l+1}f'(z_j^l).\n", @@ -3397,16 +3762,20 @@ }, { "cell_type": "markdown", - "id": "6d66941f", - "metadata": {}, + "id": "0e0090b2", + "metadata": { + "editable": true + }, "source": [ "Finally, we update the weights and the biases using gradient descent for each $l=L-1,L-2,\\dots,2$ and update the weights and biases according to the rules" ] }, { "cell_type": "markdown", - "id": "b6c51464", - "metadata": {}, + "id": "a559b53f", + "metadata": { + "editable": true + }, "source": [ "$$\n", "w_{jk}^l\\leftarrow = w_{jk}^l- \\eta \\delta_j^la_k^{l-1},\n", @@ -3415,8 +3784,10 @@ }, { "cell_type": "markdown", - "id": "9566a11e", - "metadata": {}, + "id": "493fddb7", + "metadata": { + "editable": true + }, "source": [ "$$\n", "b_j^l \\leftarrow b_j^l-\\eta \\frac{\\partial {\\cal C}}{\\partial b_j^l}=b_j^l-\\eta \\delta_j^l,\n", @@ -3425,33 +3796,17 @@ }, { "cell_type": "markdown", - "id": "9d66e99e", - "metadata": {}, + "id": "dfc0b1d9", + "metadata": { + "editable": true + }, "source": [ "The parameter $\\eta$ is the learning parameter discussed in connection with the gradient descent methods.\n", "Here it is convenient to use stochastic gradient descent (see the examples below) with mini-batches with an outer loop that steps through multiple epochs of training." ] } ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/src/week40/week40.do.txt b/doc/src/week40/week40.do.txt index 547f60e21..630545efd 100644 --- a/doc/src/week40/week40.do.txt +++ b/doc/src/week40/week40.do.txt @@ -1039,9 +1039,9 @@ theta = np.random.randn(2,1) for epoch in range(n_epochs): # Can you figure out a better way of setting up the contributions to each batch? for i in range(m): - random_index = np.random.randint(m) - xi = X[random_index*M:random_index*M+M] - yi = y[random_index*M:random_index*M+M] + random_index = M*np.random.randint(m) + xi = X[random_index:random_index+M] + yi = y[random_index:random_index+M] gradients = (2.0/M)*training_gradient(yi, xi, theta) eta = learning_schedule(epoch*m+i) theta = theta - eta*gradients