diff --git a/doc/Articles/cs231n_2017_lecture10.pdf b/doc/Articles/cs231n_2017_lecture10.pdf
new file mode 100644
index 000000000..55943f601
Binary files /dev/null and b/doc/Articles/cs231n_2017_lecture10.pdf differ
diff --git a/doc/Articles/cs231n_2017_lecture5.pdf b/doc/Articles/cs231n_2017_lecture5.pdf
new file mode 100644
index 000000000..9c2dbb8e0
Binary files /dev/null and b/doc/Articles/cs231n_2017_lecture5.pdf differ
diff --git a/doc/pub/week39/html/week39-bs.html b/doc/pub/week39/html/week39-bs.html
index 63c281832..3b9e06844 100644
--- a/doc/pub/week39/html/week39-bs.html
+++ b/doc/pub/week39/html/week39-bs.html
@@ -418,7 +418,7 @@ MathJax.Hub.Config({
-Oct 4, 2022
+Oct 31, 2022
diff --git a/doc/pub/week39/html/week39-reveal.html b/doc/pub/week39/html/week39-reveal.html
index 24943ab03..426497b98 100644
--- a/doc/pub/week39/html/week39-reveal.html
+++ b/doc/pub/week39/html/week39-reveal.html
@@ -184,7 +184,7 @@ MathJax.Hub.Config({
-Oct 4, 2022
+Oct 31, 2022
@@ -3642,11 +3642,11 @@ delta = 1e-8
# Previous value for the outer product of gradients
Previous = Giter
# Accumulated gradient
- Giter +=gradients @ gradients.T
+# Giter +=gradients @ gradients.T
# Scaling with rho the new and the previous results
- Gnew = (rho*Previous+(1-rho)*Giter)
+ Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
- Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+ Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# Hadamard product
update = np.multiply(Ginverse,gradients)
theta -= update
diff --git a/doc/pub/week39/html/week39-solarized.html b/doc/pub/week39/html/week39-solarized.html
index 29b59d761..da9a76f76 100644
--- a/doc/pub/week39/html/week39-solarized.html
+++ b/doc/pub/week39/html/week39-solarized.html
@@ -332,7 +332,7 @@ MathJax.Hub.Config({
-Oct 4, 2022
+Oct 31, 2022
@@ -3575,11 +3575,11 @@ delta = 1e-8
# Previous value for the outer product of gradients
Previous = Giter
# Accumulated gradient
- Giter +=gradients @ gradients.T
+# Giter +=gradients @ gradients.T
# Scaling with rho the new and the previous results
- Gnew = (rho*Previous+(1-rho)*Giter)
+ Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
- Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+ Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# Hadamard product
update = np.multiply(Ginverse,gradients)
theta -= update
diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html
index 14b139823..1f134d2a7 100644
--- a/doc/pub/week39/html/week39.html
+++ b/doc/pub/week39/html/week39.html
@@ -409,7 +409,7 @@ MathJax.Hub.Config({
-Oct 4, 2022
+Oct 31, 2022
@@ -3652,11 +3652,11 @@ delta = 1e-8# Previous value for the outer product of gradients
Previous = Giter
# Accumulated gradient
- Giter +=gradients @ gradients.T
+# Giter +=gradients @ gradients.T
# Scaling with rho the new and the previous results
- Gnew = (rho*Previous+(1-rho)*Giter)
+ Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
- Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+ Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# Hadamard product
update = np.multiply(Ginverse,gradients)
theta -= update
diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
index 270859f3e..cc1d3ae5d 100644
Binary files a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz and b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz differ
diff --git a/doc/pub/week39/ipynb/week39.ipynb b/doc/pub/week39/ipynb/week39.ipynb
index 890cd2f0a..a241169a5 100644
--- a/doc/pub/week39/ipynb/week39.ipynb
+++ b/doc/pub/week39/ipynb/week39.ipynb
@@ -2,8 +2,10 @@
"cells": [
{
"cell_type": "markdown",
- "id": "d7f16da9",
- "metadata": {},
+ "id": "583ac34f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"\n",
@@ -12,21 +14,25 @@
},
{
"cell_type": "markdown",
- "id": "0fae925e",
- "metadata": {},
+ "id": "7f768498",
+ "metadata": {
+ "editable": true
+ },
"source": [
"# Week 39: Optimization and Gradient Methods\n",
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and Facility for Rare Isotope Beams, Michigan State University\n",
"\n",
- "Date: **Oct 4, 2022**\n",
+ "Date: **Oct 31, 2022**\n",
"\n",
"Copyright 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license"
]
},
{
"cell_type": "markdown",
- "id": "966d1a29",
- "metadata": {},
+ "id": "bccd38a9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Plan for week 39\n",
"\n",
@@ -51,8 +57,10 @@
},
{
"cell_type": "markdown",
- "id": "5569e96c",
- "metadata": {},
+ "id": "1415ce8c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Optimization, the central part of any Machine Learning algortithm\n",
"\n",
@@ -70,8 +78,10 @@
},
{
"cell_type": "markdown",
- "id": "39e97ae4",
- "metadata": {},
+ "id": "1682f0f0",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Revisiting our Logistic Regression case\n",
"\n",
@@ -85,8 +95,10 @@
},
{
"cell_type": "markdown",
- "id": "5169dc48",
- "metadata": {},
+ "id": "22720198",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\begin{align*}\n",
@@ -98,16 +110,20 @@
},
{
"cell_type": "markdown",
- "id": "31f7150a",
- "metadata": {},
+ "id": "82b4e195",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where $\\boldsymbol{\\beta}$ are the weights we wish to extract from data, in our case $\\beta_0$ and $\\beta_1$."
]
},
{
"cell_type": "markdown",
- "id": "86d3105e",
- "metadata": {},
+ "id": "2e5ded0f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The equations to solve\n",
"\n",
@@ -120,8 +136,10 @@
},
{
"cell_type": "markdown",
- "id": "7f654478",
- "metadata": {},
+ "id": "4c31ba97",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}} = -\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{p}\\right).\n",
@@ -130,8 +148,10 @@
},
{
"cell_type": "markdown",
- "id": "b7a0f9f5",
- "metadata": {},
+ "id": "75e132d2",
+ "metadata": {
+ "editable": true
+ },
"source": [
"If we in addition define a diagonal matrix $\\boldsymbol{W}$ with elements \n",
"$p(y_i\\vert x_i,\\boldsymbol{\\beta})(1-p(y_i\\vert x_i,\\boldsymbol{\\beta})$, we can obtain a compact expression of the second derivative as"
@@ -139,8 +159,10 @@
},
{
"cell_type": "markdown",
- "id": "f199b655",
- "metadata": {},
+ "id": "a0fd914f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T} = \\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X}.\n",
@@ -149,16 +171,20 @@
},
{
"cell_type": "markdown",
- "id": "c2f05181",
- "metadata": {},
+ "id": "666409d6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"This defines what is called the Hessian matrix."
]
},
{
"cell_type": "markdown",
- "id": "964efe40",
- "metadata": {},
+ "id": "23e12354",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Solving using Newton-Raphson's method\n",
"\n",
@@ -169,8 +195,10 @@
},
{
"cell_type": "markdown",
- "id": "344fe7a6",
- "metadata": {},
+ "id": "e711613c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{\\beta}^{\\mathrm{new}} = \\boldsymbol{\\beta}^{\\mathrm{old}}-\\left(\\frac{\\partial^2 \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}\\partial \\boldsymbol{\\beta}^T}\\right)^{-1}_{\\boldsymbol{\\beta}^{\\mathrm{old}}}\\times \\left(\\frac{\\partial \\mathcal{C}(\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}\\right)_{\\boldsymbol{\\beta}^{\\mathrm{old}}},\n",
@@ -179,16 +207,20 @@
},
{
"cell_type": "markdown",
- "id": "2d901712",
- "metadata": {},
+ "id": "e0da6e71",
+ "metadata": {
+ "editable": true
+ },
"source": [
"or in matrix form as"
]
},
{
"cell_type": "markdown",
- "id": "e9a98c62",
- "metadata": {},
+ "id": "c8d5e4e0",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{\\beta}^{\\mathrm{new}} = \\boldsymbol{\\beta}^{\\mathrm{old}}-\\left(\\boldsymbol{X}^T\\boldsymbol{W}\\boldsymbol{X} \\right)^{-1}\\times \\left(-\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{p}) \\right)_{\\boldsymbol{\\beta}^{\\mathrm{old}}}.\n",
@@ -197,8 +229,10 @@
},
{
"cell_type": "markdown",
- "id": "f3807702",
- "metadata": {},
+ "id": "7c307d5b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"The right-hand side is computed with the old values of $\\beta$. \n",
"\n",
@@ -207,8 +241,10 @@
},
{
"cell_type": "markdown",
- "id": "e9dae57f",
- "metadata": {},
+ "id": "96cac52e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Brief reminder on Newton-Raphson's method\n",
"\n",
@@ -225,8 +261,10 @@
},
{
"cell_type": "markdown",
- "id": "825f5fb4",
- "metadata": {},
+ "id": "f14922a3",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The equations\n",
"\n",
@@ -239,8 +277,10 @@
},
{
"cell_type": "markdown",
- "id": "b9c5e8fa",
- "metadata": {},
+ "id": "83676afb",
+ "metadata": {
+ "editable": true
+ },
"source": [
"\n",
"\n",
@@ -253,8 +293,10 @@
},
{
"cell_type": "markdown",
- "id": "57511f62",
- "metadata": {},
+ "id": "54afff8c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"For small enough values of the function and for well-behaved\n",
"functions, the terms beyond linear are unimportant, hence we obtain"
@@ -262,8 +304,10 @@
},
{
"cell_type": "markdown",
- "id": "7e862186",
- "metadata": {},
+ "id": "ce1efd26",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"f(x)+(s-x)f'(x)\\approx 0,\n",
@@ -272,16 +316,20 @@
},
{
"cell_type": "markdown",
- "id": "8404da03",
- "metadata": {},
+ "id": "587d4556",
+ "metadata": {
+ "editable": true
+ },
"source": [
"yielding"
]
},
{
"cell_type": "markdown",
- "id": "da40084f",
- "metadata": {},
+ "id": "f5b04720",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"s\\approx x-\\frac{f(x)}{f'(x)}.\n",
@@ -290,16 +338,20 @@
},
{
"cell_type": "markdown",
- "id": "a6ef576c",
- "metadata": {},
+ "id": "d084b40b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Having in mind an iterative procedure, it is natural to start iterating with"
]
},
{
"cell_type": "markdown",
- "id": "639e31cf",
- "metadata": {},
+ "id": "4ac0298a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"x_{n+1}=x_n-\\frac{f(x_n)}{f'(x_n)}.\n",
@@ -308,8 +360,10 @@
},
{
"cell_type": "markdown",
- "id": "68676d8e",
- "metadata": {},
+ "id": "20ca5b5b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Simple geometric interpretation\n",
"\n",
@@ -328,8 +382,10 @@
},
{
"cell_type": "markdown",
- "id": "c628108d",
- "metadata": {},
+ "id": "6c9d98dc",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Extending to more than one variable\n",
"\n",
@@ -339,8 +395,10 @@
},
{
"cell_type": "markdown",
- "id": "36224bc2",
- "metadata": {},
+ "id": "b5f4a3b9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\begin{array}{cc} f_1(x_1,x_2) &=0\\\\\n",
@@ -350,16 +408,20 @@
},
{
"cell_type": "markdown",
- "id": "666eb3fb",
- "metadata": {},
+ "id": "7216495f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"which we Taylor expand to obtain"
]
},
{
"cell_type": "markdown",
- "id": "1eb4f2f1",
- "metadata": {},
+ "id": "3e3b0da1",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\begin{array}{cc} 0=f_1(x_1+h_1,x_2+h_2)=&f_1(x_1,x_2)+h_1\n",
@@ -374,16 +436,20 @@
},
{
"cell_type": "markdown",
- "id": "331077c8",
- "metadata": {},
+ "id": "b818c377",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Defining the Jacobian matrix ${\\bf \\boldsymbol{J}}$ we have"
]
},
{
"cell_type": "markdown",
- "id": "e460ff1d",
- "metadata": {},
+ "id": "802fdbb2",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"{\\bf \\boldsymbol{J}}=\\left( \\begin{array}{cc}\n",
@@ -395,16 +461,20 @@
},
{
"cell_type": "markdown",
- "id": "1d8e8d5f",
- "metadata": {},
+ "id": "5d6e301d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"we can rephrase Newton's method as"
]
},
{
"cell_type": "markdown",
- "id": "fc24c557",
- "metadata": {},
+ "id": "5f915b85",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\left(\\begin{array}{c} x_1^{n+1} \\\\ x_2^{n+1} \\end{array} \\right)=\n",
@@ -415,16 +485,20 @@
},
{
"cell_type": "markdown",
- "id": "f176130f",
- "metadata": {},
+ "id": "faf0adb4",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where we have defined"
]
},
{
"cell_type": "markdown",
- "id": "73e6dba7",
- "metadata": {},
+ "id": "9ebe1dcc",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\left(\\begin{array}{c} h_1^{n} \\\\ h_2^{n} \\end{array} \\right)=\n",
@@ -435,8 +509,10 @@
},
{
"cell_type": "markdown",
- "id": "82333090",
- "metadata": {},
+ "id": "83db9607",
+ "metadata": {
+ "editable": true
+ },
"source": [
"We need thus to compute the inverse of the Jacobian matrix and it\n",
"is to understand that difficulties may\n",
@@ -448,8 +524,10 @@
},
{
"cell_type": "markdown",
- "id": "c751397c",
- "metadata": {},
+ "id": "21cf9b71",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Steepest descent\n",
"\n",
@@ -463,8 +541,10 @@
},
{
"cell_type": "markdown",
- "id": "c9964415",
- "metadata": {},
+ "id": "38ecb654",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{x}_{k+1} = \\mathbf{x}_k - \\gamma_k \\nabla F(\\mathbf{x}_k),\n",
@@ -473,8 +553,10 @@
},
{
"cell_type": "markdown",
- "id": "38ac9b9a",
- "metadata": {},
+ "id": "a9ea135f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"with $\\gamma_k > 0$.\n",
"\n",
@@ -485,8 +567,10 @@
},
{
"cell_type": "markdown",
- "id": "0515ce67",
- "metadata": {},
+ "id": "0dd6ca78",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## More on Steepest descent\n",
"\n",
@@ -498,8 +582,10 @@
},
{
"cell_type": "markdown",
- "id": "c1e611c7",
- "metadata": {},
+ "id": "4746eef0",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{x}_{k+1} = \\mathbf{x}_k - \\gamma_k \\nabla F(\\mathbf{x}_k), \\ \\ k \\geq 0.\n",
@@ -508,8 +594,10 @@
},
{
"cell_type": "markdown",
- "id": "850112da",
- "metadata": {},
+ "id": "8142e86d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"The parameter $\\gamma_k$ is often referred to as the step length or\n",
"the learning rate within the context of Machine Learning."
@@ -517,8 +605,10 @@
},
{
"cell_type": "markdown",
- "id": "c4746e1e",
- "metadata": {},
+ "id": "bb6aa138",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The ideal\n",
"\n",
@@ -543,8 +633,10 @@
},
{
"cell_type": "markdown",
- "id": "def061ce",
- "metadata": {},
+ "id": "377a7c58",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The sensitiveness of the gradient descent\n",
"\n",
@@ -563,8 +655,10 @@
},
{
"cell_type": "markdown",
- "id": "e5dafb57",
- "metadata": {},
+ "id": "b1532ef1",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Convex functions\n",
"\n",
@@ -583,8 +677,10 @@
},
{
"cell_type": "markdown",
- "id": "dd243b08",
- "metadata": {},
+ "id": "ebcfcc2a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Convex function\n",
"\n",
@@ -593,8 +689,10 @@
},
{
"cell_type": "markdown",
- "id": "dab08d9a",
- "metadata": {},
+ "id": "568ce124",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conditions on convex functions\n",
"\n",
@@ -628,8 +726,10 @@
},
{
"cell_type": "markdown",
- "id": "35cd06e5",
- "metadata": {},
+ "id": "73988f05",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## More on convex functions\n",
"\n",
@@ -654,8 +754,10 @@
},
{
"cell_type": "markdown",
- "id": "b353eaa0",
- "metadata": {},
+ "id": "32029fd7",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Some simple problems\n",
"\n",
@@ -682,8 +784,10 @@
},
{
"cell_type": "markdown",
- "id": "7e331d81",
- "metadata": {},
+ "id": "37adc473",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Standard steepest descent\n",
"\n",
@@ -700,8 +804,10 @@
},
{
"cell_type": "markdown",
- "id": "60251599",
- "metadata": {},
+ "id": "197f049d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{A}\\boldsymbol{x} = \\boldsymbol{b}.\n",
@@ -710,16 +816,20 @@
},
{
"cell_type": "markdown",
- "id": "090340dd",
- "metadata": {},
+ "id": "6c3d1d0a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"In the iterative process we end up with a problem like"
]
},
{
"cell_type": "markdown",
- "id": "1a58340e",
- "metadata": {},
+ "id": "593a054c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{r}= \\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x},\n",
@@ -728,8 +838,10 @@
},
{
"cell_type": "markdown",
- "id": "76280f71",
- "metadata": {},
+ "id": "59297c03",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where $\\boldsymbol{r}$ is the so-called residual or error in the iterative process.\n",
"\n",
@@ -738,8 +850,10 @@
},
{
"cell_type": "markdown",
- "id": "1c1298d3",
- "metadata": {},
+ "id": "057cdad6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Gradient method\n",
"\n",
@@ -748,8 +862,10 @@
},
{
"cell_type": "markdown",
- "id": "7ec5c931",
- "metadata": {},
+ "id": "6d33a864",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"P(\\boldsymbol{x})=\\frac{1}{2}\\boldsymbol{x}^T\\boldsymbol{A}\\boldsymbol{x} - \\boldsymbol{x}^T\\boldsymbol{b},\n",
@@ -758,8 +874,10 @@
},
{
"cell_type": "markdown",
- "id": "7f53efbc",
- "metadata": {},
+ "id": "34d0fd48",
+ "metadata": {
+ "editable": true
+ },
"source": [
"with the constraint that the matrix $\\boldsymbol{A}$ is positive definite and\n",
"symmetric. This defines also the Hessian and we want it to be positive definite."
@@ -767,8 +885,10 @@
},
{
"cell_type": "markdown",
- "id": "769e008e",
- "metadata": {},
+ "id": "3d42eea4",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Steepest descent method\n",
"\n",
@@ -778,8 +898,10 @@
},
{
"cell_type": "markdown",
- "id": "b7c59723",
- "metadata": {},
+ "id": "c79756c9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{x}_0=0,\n",
@@ -788,16 +910,20 @@
},
{
"cell_type": "markdown",
- "id": "815e3b43",
- "metadata": {},
+ "id": "5c86a7ce",
+ "metadata": {
+ "editable": true
+ },
"source": [
"or consider the system"
]
},
{
"cell_type": "markdown",
- "id": "5c500e07",
- "metadata": {},
+ "id": "286c2a94",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{A}\\boldsymbol{z} = \\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x}_0,\n",
@@ -806,16 +932,20 @@
},
{
"cell_type": "markdown",
- "id": "cf532bbd",
- "metadata": {},
+ "id": "37d208ee",
+ "metadata": {
+ "editable": true
+ },
"source": [
"instead."
]
},
{
"cell_type": "markdown",
- "id": "e4952b7c",
- "metadata": {},
+ "id": "3c8c11c4",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Steepest descent method\n",
"One can show that the solution $\\boldsymbol{x}$ is also the unique minimizer of the quadratic form"
@@ -823,8 +953,10 @@
},
{
"cell_type": "markdown",
- "id": "7cb52a59",
- "metadata": {},
+ "id": "d8c8761a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"f(\\boldsymbol{x}) = \\frac{1}{2}\\boldsymbol{x}^T\\boldsymbol{A}\\boldsymbol{x} - \\boldsymbol{x}^T \\boldsymbol{x} , \\quad \\boldsymbol{x}\\in\\mathbf{R}^n.\n",
@@ -833,8 +965,10 @@
},
{
"cell_type": "markdown",
- "id": "02ce9749",
- "metadata": {},
+ "id": "23703339",
+ "metadata": {
+ "editable": true
+ },
"source": [
"This suggests taking the first basis vector $\\boldsymbol{r}_1$ (see below for definition) \n",
"to be the gradient of $f$ at $\\boldsymbol{x}=\\boldsymbol{x}_0$, \n",
@@ -843,8 +977,10 @@
},
{
"cell_type": "markdown",
- "id": "3d50553f",
- "metadata": {},
+ "id": "73ef4ac9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{A}\\boldsymbol{x}_0-\\boldsymbol{b},\n",
@@ -853,8 +989,10 @@
},
{
"cell_type": "markdown",
- "id": "4c7ce539",
- "metadata": {},
+ "id": "a110886a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"and \n",
"$\\boldsymbol{x}_0=0$ it is equal $-\\boldsymbol{b}$."
@@ -862,8 +1000,10 @@
},
{
"cell_type": "markdown",
- "id": "c542be7e",
- "metadata": {},
+ "id": "da8567f7",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Final expressions\n",
"We can compute the residual iteratively as"
@@ -871,8 +1011,10 @@
},
{
"cell_type": "markdown",
- "id": "add3b7d6",
- "metadata": {},
+ "id": "a21844d3",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{r}_{k+1}=\\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x}_{k+1},\n",
@@ -881,16 +1023,20 @@
},
{
"cell_type": "markdown",
- "id": "a1ea84df",
- "metadata": {},
+ "id": "443f94d6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"which equals"
]
},
{
"cell_type": "markdown",
- "id": "42d21b86",
- "metadata": {},
+ "id": "7c997294",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{b}-\\boldsymbol{A}(\\boldsymbol{x}_k+\\alpha_k\\boldsymbol{r}_k),\n",
@@ -899,16 +1045,20 @@
},
{
"cell_type": "markdown",
- "id": "5f2a72d4",
- "metadata": {},
+ "id": "73e04632",
+ "metadata": {
+ "editable": true
+ },
"source": [
"or"
]
},
{
"cell_type": "markdown",
- "id": "ce139be2",
- "metadata": {},
+ "id": "4b28339c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"(\\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x}_k)-\\alpha_k\\boldsymbol{A}\\boldsymbol{r}_k,\n",
@@ -917,16 +1067,20 @@
},
{
"cell_type": "markdown",
- "id": "e036170a",
- "metadata": {},
+ "id": "c8281e7d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"which gives"
]
},
{
"cell_type": "markdown",
- "id": "c00fa237",
- "metadata": {},
+ "id": "b4c113cb",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\alpha_k = \\frac{\\boldsymbol{r}_k^T\\boldsymbol{r}_k}{\\boldsymbol{r}_k^T\\boldsymbol{A}\\boldsymbol{r}_k}\n",
@@ -935,16 +1089,20 @@
},
{
"cell_type": "markdown",
- "id": "def1e8ba",
- "metadata": {},
+ "id": "f4108458",
+ "metadata": {
+ "editable": true
+ },
"source": [
"leading to the iterative scheme"
]
},
{
"cell_type": "markdown",
- "id": "8fdc4cd1",
- "metadata": {},
+ "id": "037994ec",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{x}_{k+1}=\\boldsymbol{x}_k+\\alpha_k\\boldsymbol{r}_{k},\n",
@@ -953,8 +1111,10 @@
},
{
"cell_type": "markdown",
- "id": "36f4fbe4",
- "metadata": {},
+ "id": "904823ee",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Steepest descent example"
]
@@ -962,8 +1122,11 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "4d5792a9",
- "metadata": {},
+ "id": "a4b6ca0a",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"%matplotlib inline\n",
@@ -992,8 +1155,10 @@
},
{
"cell_type": "markdown",
- "id": "d2d8619e",
- "metadata": {},
+ "id": "c2dbc9b5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"And then as countor plot"
]
@@ -1001,8 +1166,11 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "88f291dc",
- "metadata": {},
+ "id": "c7fb6aa3",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"pt.axis(\"equal\")\n",
@@ -1012,8 +1180,10 @@
},
{
"cell_type": "markdown",
- "id": "6ed5e8ab",
- "metadata": {},
+ "id": "ee9a52f1",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Find guesses"
]
@@ -1021,8 +1191,11 @@
{
"cell_type": "code",
"execution_count": 3,
- "id": "fdef6758",
- "metadata": {},
+ "id": "0ef16313",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"x = guesses[-1]\n",
@@ -1031,8 +1204,10 @@
},
{
"cell_type": "markdown",
- "id": "e2e25272",
- "metadata": {},
+ "id": "1edac416",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Run it!"
]
@@ -1040,8 +1215,11 @@
{
"cell_type": "code",
"execution_count": 4,
- "id": "f6c654cc",
- "metadata": {},
+ "id": "d9d151d3",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"def f1d(alpha):\n",
@@ -1055,8 +1233,10 @@
},
{
"cell_type": "markdown",
- "id": "33fb428f",
- "metadata": {},
+ "id": "754cba6b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"What happened?"
]
@@ -1064,8 +1244,11 @@
{
"cell_type": "code",
"execution_count": 5,
- "id": "b53775a7",
- "metadata": {},
+ "id": "cb686721",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"pt.axis(\"equal\")\n",
@@ -1076,16 +1259,20 @@
},
{
"cell_type": "markdown",
- "id": "e3937787",
- "metadata": {},
+ "id": "4c568d9a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Note that we did only one iteration here. We can easily add more using our previous guesses."
]
},
{
"cell_type": "markdown",
- "id": "ba99c964",
- "metadata": {},
+ "id": "9a20583c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method\n",
"In the CG method we define so-called conjugate directions and two vectors \n",
@@ -1096,8 +1283,10 @@
},
{
"cell_type": "markdown",
- "id": "0df9b94c",
- "metadata": {},
+ "id": "d456864b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{s}^T\\boldsymbol{A}\\boldsymbol{t}= 0.\n",
@@ -1106,8 +1295,10 @@
},
{
"cell_type": "markdown",
- "id": "836b3947",
- "metadata": {},
+ "id": "c9447537",
+ "metadata": {
+ "editable": true
+ },
"source": [
"The philosophy of the CG method is to perform searches in various conjugate directions\n",
"of our vectors $\\boldsymbol{x}_i$ obeying the above criterion, namely"
@@ -1115,8 +1306,10 @@
},
{
"cell_type": "markdown",
- "id": "3c825832",
- "metadata": {},
+ "id": "2561b08f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{x}_i^T\\boldsymbol{A}\\boldsymbol{x}_j= 0.\n",
@@ -1125,8 +1318,10 @@
},
{
"cell_type": "markdown",
- "id": "a35e7e94",
- "metadata": {},
+ "id": "9acfa2b6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Two vectors are conjugate if they are orthogonal with respect to \n",
"this inner product. Being conjugate is a symmetric relation: if $\\boldsymbol{s}$ is conjugate to $\\boldsymbol{t}$, then $\\boldsymbol{t}$ is conjugate to $\\boldsymbol{s}$."
@@ -1134,8 +1329,10 @@
},
{
"cell_type": "markdown",
- "id": "6cfd0d05",
- "metadata": {},
+ "id": "c4a63ef5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method\n",
"An example is given by the eigenvectors of the matrix"
@@ -1143,8 +1340,10 @@
},
{
"cell_type": "markdown",
- "id": "e3932de8",
- "metadata": {},
+ "id": "30998da6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{v}_i^T\\boldsymbol{A}\\boldsymbol{v}_j= \\lambda\\boldsymbol{v}_i^T\\boldsymbol{v}_j,\n",
@@ -1153,16 +1352,20 @@
},
{
"cell_type": "markdown",
- "id": "1cd3eb74",
- "metadata": {},
+ "id": "66d87e8f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"which is zero unless $i=j$."
]
},
{
"cell_type": "markdown",
- "id": "45502872",
- "metadata": {},
+ "id": "97ea59d5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method\n",
"Assume now that we have a symmetric positive-definite matrix $\\boldsymbol{A}$ of size\n",
@@ -1171,8 +1374,10 @@
},
{
"cell_type": "markdown",
- "id": "ced89923",
- "metadata": {},
+ "id": "50b9225f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{x}_{i+1}=\\boldsymbol{x}_{i}+\\alpha_i\\boldsymbol{p}_{i}.\n",
@@ -1181,8 +1386,10 @@
},
{
"cell_type": "markdown",
- "id": "41d51d10",
- "metadata": {},
+ "id": "ca17e40b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"We assume that $\\boldsymbol{p}_{i}$ is a sequence of $n$ mutually conjugate directions. \n",
"Then the $\\boldsymbol{p}_{i}$ form a basis of $R^n$ and we can expand the solution \n",
@@ -1191,8 +1398,10 @@
},
{
"cell_type": "markdown",
- "id": "4f292c21",
- "metadata": {},
+ "id": "a3cb9d18",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{x} = \\sum^{n}_{i=1} \\alpha_i \\boldsymbol{p}_i.\n",
@@ -1201,8 +1410,10 @@
},
{
"cell_type": "markdown",
- "id": "2c7b4ae4",
- "metadata": {},
+ "id": "06371a0a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method\n",
"The coefficients are given by"
@@ -1210,8 +1421,10 @@
},
{
"cell_type": "markdown",
- "id": "7efde24f",
- "metadata": {},
+ "id": "481e895e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{A}\\mathbf{x} = \\sum^{n}_{i=1} \\alpha_i \\mathbf{A} \\mathbf{p}_i = \\mathbf{b}.\n",
@@ -1220,16 +1433,20 @@
},
{
"cell_type": "markdown",
- "id": "9dd325fa",
- "metadata": {},
+ "id": "50c84a7e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Multiplying with $\\boldsymbol{p}_k^T$ from the left gives"
]
},
{
"cell_type": "markdown",
- "id": "fe08d203",
- "metadata": {},
+ "id": "b2ef7b62",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{p}_k^T \\boldsymbol{A}\\boldsymbol{x} = \\sum^{n}_{i=1} \\alpha_i\\boldsymbol{p}_k^T \\boldsymbol{A}\\boldsymbol{p}_i= \\boldsymbol{p}_k^T \\boldsymbol{b},\n",
@@ -1238,16 +1455,20 @@
},
{
"cell_type": "markdown",
- "id": "59f0a8a5",
- "metadata": {},
+ "id": "c46e574a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"and we can define the coefficients $\\alpha_k$ as"
]
},
{
"cell_type": "markdown",
- "id": "66984b8f",
- "metadata": {},
+ "id": "91e5770e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\alpha_k = \\frac{\\boldsymbol{p}_k^T \\boldsymbol{b}}{\\boldsymbol{p}_k^T \\boldsymbol{A} \\boldsymbol{p}_k}\n",
@@ -1256,8 +1477,10 @@
},
{
"cell_type": "markdown",
- "id": "e8a3dc7f",
- "metadata": {},
+ "id": "05200d8c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method and iterations\n",
"\n",
@@ -1274,8 +1497,10 @@
},
{
"cell_type": "markdown",
- "id": "8226379a",
- "metadata": {},
+ "id": "091e6c42",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{x}_0=0,\n",
@@ -1284,16 +1509,20 @@
},
{
"cell_type": "markdown",
- "id": "1f0cd65d",
- "metadata": {},
+ "id": "005a6148",
+ "metadata": {
+ "editable": true
+ },
"source": [
"or consider the system"
]
},
{
"cell_type": "markdown",
- "id": "1b30e5f7",
- "metadata": {},
+ "id": "55cf6c0d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{A}\\boldsymbol{z} = \\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x}_0,\n",
@@ -1302,16 +1531,20 @@
},
{
"cell_type": "markdown",
- "id": "8ecbcd81",
- "metadata": {},
+ "id": "3375fea4",
+ "metadata": {
+ "editable": true
+ },
"source": [
"instead."
]
},
{
"cell_type": "markdown",
- "id": "f749efb2",
- "metadata": {},
+ "id": "366193e9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method\n",
"One can show that the solution $\\boldsymbol{x}$ is also the unique minimizer of the quadratic form"
@@ -1319,8 +1552,10 @@
},
{
"cell_type": "markdown",
- "id": "a07fdbd6",
- "metadata": {},
+ "id": "f698e8c7",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"f(\\boldsymbol{x}) = \\frac{1}{2}\\boldsymbol{x}^T\\boldsymbol{A}\\boldsymbol{x} - \\boldsymbol{x}^T \\boldsymbol{x} , \\quad \\boldsymbol{x}\\in\\mathbf{R}^n.\n",
@@ -1329,8 +1564,10 @@
},
{
"cell_type": "markdown",
- "id": "69202a7e",
- "metadata": {},
+ "id": "96ebe1cb",
+ "metadata": {
+ "editable": true
+ },
"source": [
"This suggests taking the first basis vector $\\boldsymbol{p}_1$ \n",
"to be the gradient of $f$ at $\\boldsymbol{x}=\\boldsymbol{x}_0$, \n",
@@ -1339,8 +1576,10 @@
},
{
"cell_type": "markdown",
- "id": "d571623c",
- "metadata": {},
+ "id": "7b6bef19",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{A}\\boldsymbol{x}_0-\\boldsymbol{b},\n",
@@ -1349,8 +1588,10 @@
},
{
"cell_type": "markdown",
- "id": "3a85d097",
- "metadata": {},
+ "id": "6c3d49d8",
+ "metadata": {
+ "editable": true
+ },
"source": [
"and \n",
"$\\boldsymbol{x}_0=0$ it is equal $-\\boldsymbol{b}$.\n",
@@ -1360,8 +1601,10 @@
},
{
"cell_type": "markdown",
- "id": "26d102d3",
- "metadata": {},
+ "id": "eb6f4e59",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method\n",
"Let $\\boldsymbol{r}_k$ be the residual at the $k$-th step:"
@@ -1369,8 +1612,10 @@
},
{
"cell_type": "markdown",
- "id": "1fd158c3",
- "metadata": {},
+ "id": "73ee5acb",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{r}_k=\\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x}_k.\n",
@@ -1379,8 +1624,10 @@
},
{
"cell_type": "markdown",
- "id": "3069e358",
- "metadata": {},
+ "id": "ea2f0acc",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Note that $\\boldsymbol{r}_k$ is the negative gradient of $f$ at \n",
"$\\boldsymbol{x}=\\boldsymbol{x}_k$, \n",
@@ -1393,8 +1640,10 @@
},
{
"cell_type": "markdown",
- "id": "e8ed2a46",
- "metadata": {},
+ "id": "ea2c088e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{p}_{k+1}=\\boldsymbol{r}_k-\\frac{\\boldsymbol{p}_k^T \\boldsymbol{A}\\boldsymbol{r}_k}{\\boldsymbol{p}_k^T\\boldsymbol{A}\\boldsymbol{p}_k} \\boldsymbol{p}_k.\n",
@@ -1403,8 +1652,10 @@
},
{
"cell_type": "markdown",
- "id": "1cd356ef",
- "metadata": {},
+ "id": "09d0f89a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Conjugate gradient method\n",
"We can also compute the residual iteratively as"
@@ -1412,8 +1663,10 @@
},
{
"cell_type": "markdown",
- "id": "082eb012",
- "metadata": {},
+ "id": "1ea74654",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{r}_{k+1}=\\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x}_{k+1},\n",
@@ -1422,16 +1675,20 @@
},
{
"cell_type": "markdown",
- "id": "af481a33",
- "metadata": {},
+ "id": "d768070a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"which equals"
]
},
{
"cell_type": "markdown",
- "id": "b2ffeef1",
- "metadata": {},
+ "id": "fd0f164b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{b}-\\boldsymbol{A}(\\boldsymbol{x}_k+\\alpha_k\\boldsymbol{p}_k),\n",
@@ -1440,16 +1697,20 @@
},
{
"cell_type": "markdown",
- "id": "5a28c237",
- "metadata": {},
+ "id": "7725e6c6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"or"
]
},
{
"cell_type": "markdown",
- "id": "3cd63ffa",
- "metadata": {},
+ "id": "9d263e6f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"(\\boldsymbol{b}-\\boldsymbol{A}\\boldsymbol{x}_k)-\\alpha_k\\boldsymbol{A}\\boldsymbol{p}_k,\n",
@@ -1458,16 +1719,20 @@
},
{
"cell_type": "markdown",
- "id": "c8a7fea8",
- "metadata": {},
+ "id": "38950b5c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"which gives"
]
},
{
"cell_type": "markdown",
- "id": "a9bf9fc7",
- "metadata": {},
+ "id": "2a1402d1",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{r}_{k+1}=\\boldsymbol{r}_k-\\boldsymbol{A}\\boldsymbol{p}_{k},\n",
@@ -1476,8 +1741,10 @@
},
{
"cell_type": "markdown",
- "id": "c26c72c2",
- "metadata": {},
+ "id": "691bb2be",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Revisiting our first homework\n",
"\n",
@@ -1498,8 +1765,11 @@
{
"cell_type": "code",
"execution_count": 6,
- "id": "d6959fcb",
- "metadata": {},
+ "id": "cf2962e8",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"x = 2*np.random.rand(m,1)\n",
@@ -1508,8 +1778,10 @@
},
{
"cell_type": "markdown",
- "id": "4804832d",
- "metadata": {},
+ "id": "52392a05",
+ "metadata": {
+ "editable": true
+ },
"source": [
"with $x_i \\in [0,1] $ is chosen randomly using a uniform distribution. Additionally we have a stochastic noise chosen according to a normal distribution $\\cal {N}(0,1)$. \n",
"The linear regression model is given by"
@@ -1517,8 +1789,10 @@
},
{
"cell_type": "markdown",
- "id": "73872b84",
- "metadata": {},
+ "id": "240b39a4",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"h_\\beta(x) = \\boldsymbol{y} = \\beta_0 + \\beta_1 x,\n",
@@ -1527,16 +1801,20 @@
},
{
"cell_type": "markdown",
- "id": "86efb796",
- "metadata": {},
+ "id": "6519bfc2",
+ "metadata": {
+ "editable": true
+ },
"source": [
"such that"
]
},
{
"cell_type": "markdown",
- "id": "44d6e70b",
- "metadata": {},
+ "id": "1280663c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{y}_i = \\beta_0 + \\beta_1 x_i.\n",
@@ -1545,8 +1823,10 @@
},
{
"cell_type": "markdown",
- "id": "0104b680",
- "metadata": {},
+ "id": "1cc5d5b6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Gradient descent example\n",
"\n",
@@ -1557,8 +1837,10 @@
},
{
"cell_type": "markdown",
- "id": "35f38516",
- "metadata": {},
+ "id": "47ebfb08",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"X \\equiv \\begin{bmatrix}\n",
@@ -1571,16 +1853,20 @@
},
{
"cell_type": "markdown",
- "id": "ff18ec20",
- "metadata": {},
+ "id": "c3df406e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"The cost/loss/risk function is given by ("
]
},
{
"cell_type": "markdown",
- "id": "81cb8c14",
- "metadata": {},
+ "id": "8b58bdce",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"C(\\beta) = \\frac{1}{n}||X\\beta-\\mathbf{y}||_{2}^{2} = \\frac{1}{n}\\sum_{i=1}^{100}\\left[ (\\beta_0 + \\beta_1 x_i)^2 - 2 y_i (\\beta_0 + \\beta_1 x_i) + y_i^2\\right]\n",
@@ -1589,16 +1875,20 @@
},
{
"cell_type": "markdown",
- "id": "bde98bf5",
- "metadata": {},
+ "id": "f07f333f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"and we want to find $\\beta$ such that $C(\\beta)$ is minimized."
]
},
{
"cell_type": "markdown",
- "id": "caceb4f9",
- "metadata": {},
+ "id": "65a80224",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The derivative of the cost/loss function\n",
"\n",
@@ -1607,8 +1897,10 @@
},
{
"cell_type": "markdown",
- "id": "52c47ed2",
- "metadata": {},
+ "id": "712eb572",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\nabla_{\\beta} C(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n",
@@ -1619,16 +1911,20 @@
},
{
"cell_type": "markdown",
- "id": "e2f3f0ce",
- "metadata": {},
+ "id": "702c4425",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where $X$ is the design matrix defined above."
]
},
{
"cell_type": "markdown",
- "id": "b2cc74e1",
- "metadata": {},
+ "id": "7b7bc005",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The Hessian matrix\n",
"The Hessian matrix of $C(\\beta)$ is given by"
@@ -1636,8 +1932,10 @@
},
{
"cell_type": "markdown",
- "id": "9849d1d0",
- "metadata": {},
+ "id": "17efa9c8",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{H} \\equiv \\begin{bmatrix}\n",
@@ -1649,16 +1947,20 @@
},
{
"cell_type": "markdown",
- "id": "57196c0b",
- "metadata": {},
+ "id": "48539c00",
+ "metadata": {
+ "editable": true
+ },
"source": [
"This result implies that $C(\\beta)$ is a convex function since the matrix $X^T X$ always is positive semi-definite."
]
},
{
"cell_type": "markdown",
- "id": "c7912505",
- "metadata": {},
+ "id": "e309864e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Simple program\n",
"\n",
@@ -1667,8 +1969,10 @@
},
{
"cell_type": "markdown",
- "id": "e840f3d5",
- "metadata": {},
+ "id": "ad6ac23b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\beta_{k+1} = \\beta_k - \\gamma \\nabla_\\beta C(\\beta_k), \\ k=0,1,\\cdots\n",
@@ -1677,8 +1981,10 @@
},
{
"cell_type": "markdown",
- "id": "2ff5df00",
- "metadata": {},
+ "id": "62ec161c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"We can use the expression we computed for the gradient and let use a\n",
"$\\beta_0$ be chosen randomly and let $\\gamma = 0.001$. Stop iterating\n",
@@ -1690,8 +1996,10 @@
},
{
"cell_type": "markdown",
- "id": "8eedd3c9",
- "metadata": {},
+ "id": "3843c09e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Gradient Descent Example\n",
"\n",
@@ -1701,8 +2009,11 @@
{
"cell_type": "code",
"execution_count": 7,
- "id": "816520e2",
- "metadata": {},
+ "id": "69d9b859",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"\n",
@@ -1755,8 +2066,10 @@
},
{
"cell_type": "markdown",
- "id": "44de9f15",
- "metadata": {},
+ "id": "a0b2fd91",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## And a corresponding example using **scikit-learn**"
]
@@ -1764,8 +2077,11 @@
{
"cell_type": "code",
"execution_count": 8,
- "id": "dbe3ac2e",
- "metadata": {},
+ "id": "6d71b4a2",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"# Importing various packages\n",
@@ -1788,8 +2104,10 @@
},
{
"cell_type": "markdown",
- "id": "ab482d19",
- "metadata": {},
+ "id": "1a6dd499",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Gradient descent and Ridge\n",
"\n",
@@ -1798,8 +2116,10 @@
},
{
"cell_type": "markdown",
- "id": "71103e66",
- "metadata": {},
+ "id": "9a56cbf1",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"C_{\\text{ridge}}(\\beta) = \\frac{1}{n}||X\\beta -\\mathbf{y}||^2 + \\lambda ||\\beta||^2, \\ \\lambda \\geq 0.\n",
@@ -1808,16 +2128,20 @@
},
{
"cell_type": "markdown",
- "id": "af6f6a63",
- "metadata": {},
+ "id": "716ca68c",
+ "metadata": {
+ "editable": true
+ },
"source": [
"In order to minimize $C_{\\text{ridge}}(\\beta)$ using GD we adjust the gradient as follows"
]
},
{
"cell_type": "markdown",
- "id": "2a7c3f16",
- "metadata": {},
+ "id": "eb6ec1d6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\nabla_\\beta C_{\\text{ridge}}(\\beta) = \\frac{2}{n}\\begin{bmatrix} \\sum_{i=1}^{100} \\left(\\beta_0+\\beta_1x_i-y_i\\right) \\\\\n",
@@ -1828,16 +2152,20 @@
},
{
"cell_type": "markdown",
- "id": "235d1d45",
- "metadata": {},
+ "id": "7bd11dab",
+ "metadata": {
+ "editable": true
+ },
"source": [
"We can easily extend our program to minimize $C_{\\text{ridge}}(\\beta)$ using gradient descent and compare with the analytical solution given by"
]
},
{
"cell_type": "markdown",
- "id": "a4a9ccbf",
- "metadata": {},
+ "id": "14b44b82",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\beta_{\\text{ridge}} = \\left(X^T X + n\\lambda I_{2 \\times 2} \\right)^{-1} X^T \\mathbf{y}.\n",
@@ -1846,8 +2174,10 @@
},
{
"cell_type": "markdown",
- "id": "45fe29d1",
- "metadata": {},
+ "id": "f32b8bd3",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The Hessian matrix for Ridge Regression\n",
"The Hessian matrix of Ridge Regression for our simple example is given by"
@@ -1855,8 +2185,10 @@
},
{
"cell_type": "markdown",
- "id": "e1ba248c",
- "metadata": {},
+ "id": "a80f3cf0",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{H} \\equiv \\begin{bmatrix}\n",
@@ -1868,8 +2200,10 @@
},
{
"cell_type": "markdown",
- "id": "fe091054",
- "metadata": {},
+ "id": "f195a664",
+ "metadata": {
+ "editable": true
+ },
"source": [
"This implies that the Hessian matrix is positive definite, hence the stationary point is a\n",
"minimum.\n",
@@ -1880,8 +2214,10 @@
},
{
"cell_type": "markdown",
- "id": "a3b76795",
- "metadata": {},
+ "id": "1d8f6e7a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Program example for gradient descent with Ridge Regression"
]
@@ -1889,8 +2225,11 @@
{
"cell_type": "code",
"execution_count": 9,
- "id": "dd7351ce",
- "metadata": {},
+ "id": "94ef2984",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"from random import random, seed\n",
@@ -1947,8 +2286,10 @@
},
{
"cell_type": "markdown",
- "id": "b560c723",
- "metadata": {},
+ "id": "f8ca1f5d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Using gradient descent methods, limitations\n",
"\n",
@@ -1967,8 +2308,10 @@
},
{
"cell_type": "markdown",
- "id": "95148bfd",
- "metadata": {},
+ "id": "213a069d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Improving gradient descent with momentum\n",
"\n",
@@ -1978,8 +2321,11 @@
{
"cell_type": "code",
"execution_count": 10,
- "id": "75cacd0d",
- "metadata": {},
+ "id": "306b8448",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"from numpy import asarray\n",
@@ -2041,8 +2387,10 @@
},
{
"cell_type": "markdown",
- "id": "e66c83c6",
- "metadata": {},
+ "id": "4da5b553",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Same code but now with momentum gradient descent"
]
@@ -2050,8 +2398,11 @@
{
"cell_type": "code",
"execution_count": 11,
- "id": "1ac1ddd3",
- "metadata": {},
+ "id": "dca96393",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"from numpy import asarray\n",
@@ -2121,8 +2472,10 @@
},
{
"cell_type": "markdown",
- "id": "51f93591",
- "metadata": {},
+ "id": "71836de5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Overview video on Stochastic Gradient Descent\n",
"\n",
@@ -2131,8 +2484,10 @@
},
{
"cell_type": "markdown",
- "id": "cd6e4d65",
- "metadata": {},
+ "id": "5f5900d3",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Batches and mini-batches\n",
"\n",
@@ -2150,8 +2505,10 @@
},
{
"cell_type": "markdown",
- "id": "d5fb048a",
- "metadata": {},
+ "id": "e1c69762",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Stochastic Gradient Descent (SGD)\n",
"\n",
@@ -2180,8 +2537,10 @@
},
{
"cell_type": "markdown",
- "id": "5a8d9347",
- "metadata": {},
+ "id": "a24995d4",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Stochastic Gradient Descent\n",
"\n",
@@ -2195,8 +2554,10 @@
},
{
"cell_type": "markdown",
- "id": "3f9921d1",
- "metadata": {},
+ "id": "b4723ad5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"C(\\mathbf{\\beta}) = \\sum_{i=1}^n c_i(\\mathbf{x}_i,\n",
@@ -2206,8 +2567,10 @@
},
{
"cell_type": "markdown",
- "id": "56c30c1b",
- "metadata": {},
+ "id": "091a9fc0",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Computation of gradients\n",
"\n",
@@ -2217,8 +2580,10 @@
},
{
"cell_type": "markdown",
- "id": "259014e1",
- "metadata": {},
+ "id": "d37822e0",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\nabla_\\beta C(\\mathbf{\\beta}) = \\sum_i^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n",
@@ -2228,8 +2593,10 @@
},
{
"cell_type": "markdown",
- "id": "6e1eb39e",
- "metadata": {},
+ "id": "6bea18e4",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Stochasticity/randomness is introduced by only taking the\n",
"gradient on a subset of the data called minibatches. If there are $n$\n",
@@ -2240,8 +2607,10 @@
},
{
"cell_type": "markdown",
- "id": "776e3fa8",
- "metadata": {},
+ "id": "6d24cc6d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## SGD example\n",
"As an example, suppose we have $10$ data points $(\\mathbf{x}_1,\\cdots, \\mathbf{x}_{10})$ \n",
@@ -2260,8 +2629,10 @@
},
{
"cell_type": "markdown",
- "id": "62397583",
- "metadata": {},
+ "id": "d79d4ee2",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\nabla_{\\beta}\n",
@@ -2273,8 +2644,10 @@
},
{
"cell_type": "markdown",
- "id": "e4564c28",
- "metadata": {},
+ "id": "ffdefdaa",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The gradient step\n",
"\n",
@@ -2283,8 +2656,10 @@
},
{
"cell_type": "markdown",
- "id": "aa84ee27",
- "metadata": {},
+ "id": "5683612e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\beta_{j+1} = \\beta_j - \\gamma_j \\sum_{i \\in B_k}^n \\nabla_\\beta c_i(\\mathbf{x}_i,\n",
@@ -2294,8 +2669,10 @@
},
{
"cell_type": "markdown",
- "id": "8f9a8819",
- "metadata": {},
+ "id": "8a44e062",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where $k$ is picked at random with equal\n",
"probability from $[1,n/M]$. An iteration over the number of\n",
@@ -2306,8 +2683,10 @@
},
{
"cell_type": "markdown",
- "id": "013c788c",
- "metadata": {},
+ "id": "ba4deae8",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Simple example code"
]
@@ -2315,8 +2694,11 @@
{
"cell_type": "code",
"execution_count": 12,
- "id": "896f5e91",
- "metadata": {},
+ "id": "3875f457",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import numpy as np \n",
@@ -2337,8 +2719,10 @@
},
{
"cell_type": "markdown",
- "id": "15ad17d3",
- "metadata": {},
+ "id": "b69098df",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Taking the gradient only on a subset of the data has two important\n",
"benefits. First, it introduces randomness which decreases the chance\n",
@@ -2351,8 +2735,10 @@
},
{
"cell_type": "markdown",
- "id": "b8f386fc",
- "metadata": {},
+ "id": "af41a865",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## When do we stop?\n",
"\n",
@@ -2370,8 +2756,10 @@
},
{
"cell_type": "markdown",
- "id": "3cee74f0",
- "metadata": {},
+ "id": "2364163e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Slightly different approach\n",
"\n",
@@ -2388,8 +2776,10 @@
},
{
"cell_type": "markdown",
- "id": "83219966",
- "metadata": {},
+ "id": "c652788e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Time decay rate\n",
"\n",
@@ -2405,8 +2795,11 @@
{
"cell_type": "code",
"execution_count": 13,
- "id": "7365a067",
- "metadata": {},
+ "id": "84ddc049",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import numpy as np \n",
@@ -2437,8 +2830,10 @@
},
{
"cell_type": "markdown",
- "id": "6deb5f70",
- "metadata": {},
+ "id": "0a5a2ab6",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Code with a Number of Minibatches which varies\n",
"\n",
@@ -2448,8 +2843,11 @@
{
"cell_type": "code",
"execution_count": 14,
- "id": "23120227",
- "metadata": {},
+ "id": "fabd4be2",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"# Importing various packages\n",
@@ -2522,8 +2920,10 @@
},
{
"cell_type": "markdown",
- "id": "e7057b70",
- "metadata": {},
+ "id": "ebfd3188",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Replace or not\n",
"\n",
@@ -2535,8 +2935,10 @@
},
{
"cell_type": "markdown",
- "id": "2646f17c",
- "metadata": {},
+ "id": "dfc82804",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Momentum based GD\n",
"\n",
@@ -2548,8 +2950,10 @@
},
{
"cell_type": "markdown",
- "id": "51c44aad",
- "metadata": {},
+ "id": "e744f5ce",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t) \\nonumber\n",
@@ -2558,8 +2962,10 @@
},
{
"cell_type": "markdown",
- "id": "ac56f315",
- "metadata": {},
+ "id": "89eaa299",
+ "metadata": {
+ "editable": true
+ },
"source": [
"\n",
"\n",
@@ -2574,8 +2980,10 @@
},
{
"cell_type": "markdown",
- "id": "bff13a89",
- "metadata": {},
+ "id": "af3e12e1",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where we have introduced a momentum parameter $\\gamma$, with\n",
"$0\\le\\gamma\\le 1$, and for brevity we dropped the explicit notation to\n",
@@ -2591,8 +2999,10 @@
},
{
"cell_type": "markdown",
- "id": "849a49e0",
- "metadata": {},
+ "id": "43e1f77b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\Delta \\boldsymbol{\\theta}_{t+1} = \\gamma \\Delta \\boldsymbol{\\theta}_t -\\ \\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t),\n",
@@ -2601,16 +3011,20 @@
},
{
"cell_type": "markdown",
- "id": "5d73b966",
- "metadata": {},
+ "id": "984f3016",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where we have defined $\\Delta \\boldsymbol{\\theta}_{t}= \\boldsymbol{\\theta}_t-\\boldsymbol{\\theta}_{t-1}$."
]
},
{
"cell_type": "markdown",
- "id": "20b55fba",
- "metadata": {},
+ "id": "09935c80",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## More on momentum based approaches\n",
"\n",
@@ -2623,8 +3037,10 @@
},
{
"cell_type": "markdown",
- "id": "eff1c57c",
- "metadata": {},
+ "id": "2befe8e7",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"m {d^2 \\mathbf{w} \\over dt^2} + \\mu {d \\mathbf{w} \\over dt }= -\\nabla_w E(\\mathbf{w}).\n",
@@ -2633,16 +3049,20 @@
},
{
"cell_type": "markdown",
- "id": "d8c876c3",
- "metadata": {},
+ "id": "96fae894",
+ "metadata": {
+ "editable": true
+ },
"source": [
"We can discretize this equation in the usual way to get"
]
},
{
"cell_type": "markdown",
- "id": "7b18f065",
- "metadata": {},
+ "id": "481533a5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"m { \\mathbf{w}_{t+\\Delta t}-2 \\mathbf{w}_{t} +\\mathbf{w}_{t-\\Delta t} \\over (\\Delta t)^2}+\\mu {\\mathbf{w}_{t+\\Delta t}- \\mathbf{w}_{t} \\over \\Delta t} = -\\nabla_w E(\\mathbf{w}).\n",
@@ -2651,16 +3071,20 @@
},
{
"cell_type": "markdown",
- "id": "1470a14b",
- "metadata": {},
+ "id": "9b228f7a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Rearranging this equation, we can rewrite this as"
]
},
{
"cell_type": "markdown",
- "id": "37ae4db2",
- "metadata": {},
+ "id": "e75378ad",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\Delta \\mathbf{w}_{t +\\Delta t}= - { (\\Delta t)^2 \\over m +\\mu \\Delta t} \\nabla_w E(\\mathbf{w})+ {m \\over m +\\mu \\Delta t} \\Delta \\mathbf{w}_t.\n",
@@ -2669,8 +3093,10 @@
},
{
"cell_type": "markdown",
- "id": "0efa97bb",
- "metadata": {},
+ "id": "f5d59945",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Momentum parameter\n",
"\n",
@@ -2683,8 +3109,10 @@
},
{
"cell_type": "markdown",
- "id": "7d1c7269",
- "metadata": {},
+ "id": "27a2fbd9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\gamma= {m \\over m +\\mu \\Delta t }, \\qquad \\eta = {(\\Delta t)^2 \\over m +\\mu \\Delta t}.\n",
@@ -2693,8 +3121,10 @@
},
{
"cell_type": "markdown",
- "id": "b840b8d2",
- "metadata": {},
+ "id": "96a8277f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Thus, as the name suggests, the momentum parameter is proportional to\n",
"the mass of the particle and effectively provides inertia.\n",
@@ -2724,8 +3154,10 @@
},
{
"cell_type": "markdown",
- "id": "c4e7ed3f",
- "metadata": {},
+ "id": "af3583c8",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{v}_{t}=\\gamma \\mathbf{v}_{t-1}+\\eta_{t}\\nabla_\\theta E(\\boldsymbol{\\theta}_t +\\gamma \\mathbf{v}_{t-1}) \\nonumber\n",
@@ -2734,8 +3166,10 @@
},
{
"cell_type": "markdown",
- "id": "457dd7c0",
- "metadata": {},
+ "id": "5b81c117",
+ "metadata": {
+ "editable": true
+ },
"source": [
"\n",
"\n",
@@ -2750,16 +3184,20 @@
},
{
"cell_type": "markdown",
- "id": "4c569dbf",
- "metadata": {},
+ "id": "7e12d71b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"One of the major advantages of NAG is that it allows for the use of a larger learning rate than GDM for the same choice of $\\gamma$."
]
},
{
"cell_type": "markdown",
- "id": "46546df0",
- "metadata": {},
+ "id": "85e55cf5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Second moment of the gradient\n",
"\n",
@@ -2787,8 +3225,10 @@
},
{
"cell_type": "markdown",
- "id": "e9503dc2",
- "metadata": {},
+ "id": "951e771a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## RMS prop\n",
"\n",
@@ -2800,8 +3240,10 @@
},
{
"cell_type": "markdown",
- "id": "35dcad4a",
- "metadata": {},
+ "id": "b43fb0ea",
+ "metadata": {
+ "editable": true
+ },
"source": [
"\n",
"\n",
@@ -2816,8 +3258,10 @@
},
{
"cell_type": "markdown",
- "id": "51d9de6f",
- "metadata": {},
+ "id": "65ab44a7",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{s}_t =\\beta \\mathbf{s}_{t-1} +(1-\\beta)\\mathbf{g}_t^2 \\nonumber\n",
@@ -2826,8 +3270,10 @@
},
{
"cell_type": "markdown",
- "id": "a3400c7b",
- "metadata": {},
+ "id": "fafcc41a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\mathbf{g}_t \\over \\sqrt{\\mathbf{s}_t +\\epsilon}}, \\nonumber\n",
@@ -2836,8 +3282,10 @@
},
{
"cell_type": "markdown",
- "id": "d8f2d919",
- "metadata": {},
+ "id": "79b58e4e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where $\\beta$ controls the averaging time of the second moment and is\n",
"typically taken to be about $\\beta=0.9$, $\\eta_t$ is a learning rate\n",
@@ -2852,8 +3300,10 @@
},
{
"cell_type": "markdown",
- "id": "03a42326",
- "metadata": {},
+ "id": "121b70e5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## [ADAM optimizer](https://arxiv.org/abs/1412.6980)\n",
"\n",
@@ -2879,8 +3329,10 @@
},
{
"cell_type": "markdown",
- "id": "f34374af",
- "metadata": {},
+ "id": "a1fb8843",
+ "metadata": {
+ "editable": true
+ },
"source": [
"\n",
"\n",
@@ -2895,8 +3347,10 @@
},
{
"cell_type": "markdown",
- "id": "2f2e4dfe",
- "metadata": {},
+ "id": "8d5c39b5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{m}_t = \\beta_1 \\mathbf{m}_{t-1} + (1-\\beta_1) \\mathbf{g}_t \\nonumber\n",
@@ -2905,8 +3359,10 @@
},
{
"cell_type": "markdown",
- "id": "0b327f20",
- "metadata": {},
+ "id": "f09683e8",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\mathbf{s}_t =\\beta_2 \\mathbf{s}_{t-1} +(1-\\beta_2)\\mathbf{g}_t^2 \\nonumber\n",
@@ -2915,8 +3371,10 @@
},
{
"cell_type": "markdown",
- "id": "ea2a09ad",
- "metadata": {},
+ "id": "ff9285bb",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{\\mathbf{m}}_t={\\mathbf{m}_t \\over 1-\\beta_1^t} \\nonumber\n",
@@ -2925,8 +3383,10 @@
},
{
"cell_type": "markdown",
- "id": "51a7e5da",
- "metadata": {},
+ "id": "3c82455d",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{\\mathbf{s}}_t ={\\mathbf{s}_t \\over1-\\beta_2^t} \\nonumber\n",
@@ -2935,8 +3395,10 @@
},
{
"cell_type": "markdown",
- "id": "307c0bf2",
- "metadata": {},
+ "id": "971cdc8f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\boldsymbol{\\theta}_{t+1}=\\boldsymbol{\\theta}_t - \\eta_t { \\boldsymbol{\\mathbf{m}}_t \\over \\sqrt{\\boldsymbol{\\mathbf{s}}_t} +\\epsilon}, \\nonumber\n",
@@ -2945,8 +3407,10 @@
},
{
"cell_type": "markdown",
- "id": "da3580b9",
- "metadata": {},
+ "id": "8b5ce6d5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"\n",
"\n",
@@ -2960,8 +3424,10 @@
},
{
"cell_type": "markdown",
- "id": "a7708896",
- "metadata": {},
+ "id": "8936027f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"where $\\beta_1$ and $\\beta_2$ set the memory lifetime of the first and\n",
"second moment and are typically taken to be $0.9$ and $0.99$\n",
@@ -2977,8 +3443,10 @@
},
{
"cell_type": "markdown",
- "id": "2e6766e0",
- "metadata": {},
+ "id": "e117183b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"\\Delta \\theta_{t+1}= -\\eta_t { \\boldsymbol{m}_t \\over \\sqrt{\\sigma_t^2 + m_t^2 }+\\epsilon}.\n",
@@ -2987,8 +3455,10 @@
},
{
"cell_type": "markdown",
- "id": "05e25f49",
- "metadata": {},
+ "id": "717e5402",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Algorithms and codes for Adagrad, RMSprop and Adam\n",
"\n",
@@ -2999,8 +3469,10 @@
},
{
"cell_type": "markdown",
- "id": "47d2dcc6",
- "metadata": {},
+ "id": "9ca2e669",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Practical tips\n",
"\n",
@@ -3017,8 +3489,10 @@
},
{
"cell_type": "markdown",
- "id": "e76ea138",
- "metadata": {},
+ "id": "5559377a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Automatic differentiation\n",
"\n",
@@ -3053,8 +3527,10 @@
},
{
"cell_type": "markdown",
- "id": "1254c922",
- "metadata": {},
+ "id": "1890cb32",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"f(x) = \\sin\\left(2\\pi x + x^2\\right)\n",
@@ -3063,16 +3539,20 @@
},
{
"cell_type": "markdown",
- "id": "8b4dbb64",
- "metadata": {},
+ "id": "7d40bd17",
+ "metadata": {
+ "editable": true
+ },
"source": [
"which has the following derivative"
]
},
{
"cell_type": "markdown",
- "id": "2fa13506",
- "metadata": {},
+ "id": "77e56134",
+ "metadata": {
+ "editable": true
+ },
"source": [
"$$\n",
"f'(x) = \\cos\\left(2\\pi x + x^2\\right)\\left(2\\pi + 2x\\right)\n",
@@ -3081,8 +3561,10 @@
},
{
"cell_type": "markdown",
- "id": "f3740ead",
- "metadata": {},
+ "id": "fdc7e85e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Using **autograd** we have"
]
@@ -3090,8 +3572,11 @@
{
"cell_type": "code",
"execution_count": 15,
- "id": "933ce02d",
- "metadata": {},
+ "id": "5c77027e",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3132,8 +3617,10 @@
},
{
"cell_type": "markdown",
- "id": "05d483f7",
- "metadata": {},
+ "id": "d400a43a",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Using autograd\n",
"\n",
@@ -3147,8 +3634,11 @@
{
"cell_type": "code",
"execution_count": 16,
- "id": "365e1426",
- "metadata": {},
+ "id": "e142c575",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3172,8 +3662,10 @@
},
{
"cell_type": "markdown",
- "id": "d5c8b704",
- "metadata": {},
+ "id": "4a8eaab5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Autograd with more complicated functions\n",
"\n",
@@ -3185,8 +3677,11 @@
{
"cell_type": "code",
"execution_count": 17,
- "id": "4aed250d",
- "metadata": {},
+ "id": "b41b7db3",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3226,16 +3721,20 @@
},
{
"cell_type": "markdown",
- "id": "ec420d54",
- "metadata": {},
+ "id": "d548f0cb",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Note that the grad function will not produce the true gradient of the function. The true gradient of a function with two or more variables will produce a vector, where each element is the function differentiated w.r.t a variable."
]
},
{
"cell_type": "markdown",
- "id": "c6582d10",
- "metadata": {},
+ "id": "96676ad1",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## More complicated functions using the elements of their arguments directly"
]
@@ -3243,8 +3742,11 @@
{
"cell_type": "code",
"execution_count": 18,
- "id": "a855790b",
- "metadata": {},
+ "id": "0aeeac8c",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3268,8 +3770,10 @@
},
{
"cell_type": "markdown",
- "id": "dbc4859b",
- "metadata": {},
+ "id": "d4523d9b",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Note that in this case, when sending an array as input argument, the\n",
"output from Autograd is another array. This is the true gradient of\n",
@@ -3281,8 +3785,10 @@
},
{
"cell_type": "markdown",
- "id": "78222bf3",
- "metadata": {},
+ "id": "65e8cd48",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Functions using mathematical functions from Numpy"
]
@@ -3290,8 +3796,11 @@
{
"cell_type": "code",
"execution_count": 19,
- "id": "e6173737",
- "metadata": {},
+ "id": "c45b4b03",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3315,8 +3824,10 @@
},
{
"cell_type": "markdown",
- "id": "d3a872cc",
- "metadata": {},
+ "id": "47e146d0",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## More autograd"
]
@@ -3324,8 +3835,11 @@
{
"cell_type": "code",
"execution_count": 20,
- "id": "5c36b35c",
- "metadata": {},
+ "id": "7cf9ba71",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3346,8 +3860,10 @@
},
{
"cell_type": "markdown",
- "id": "3fdbc6a1",
- "metadata": {},
+ "id": "304cb623",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## And with loops"
]
@@ -3355,8 +3871,11 @@
{
"cell_type": "code",
"execution_count": 21,
- "id": "35ea422c",
- "metadata": {},
+ "id": "aaf5c76c",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3388,8 +3907,11 @@
{
"cell_type": "code",
"execution_count": 22,
- "id": "2a340b57",
- "metadata": {},
+ "id": "d8fac3e8",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3405,8 +3927,10 @@
},
{
"cell_type": "markdown",
- "id": "384969d1",
- "metadata": {},
+ "id": "af8a7233",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Using recursion"
]
@@ -3414,8 +3938,11 @@
{
"cell_type": "code",
"execution_count": 23,
- "id": "68514367",
- "metadata": {},
+ "id": "35a7b10b",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3449,16 +3976,20 @@
},
{
"cell_type": "markdown",
- "id": "e8c6dade",
- "metadata": {},
+ "id": "90d3920f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Note that if n is equal to zero or one, Autograd will give an error message. This message appears when the output is independent on input."
]
},
{
"cell_type": "markdown",
- "id": "5610fec2",
- "metadata": {},
+ "id": "128a60a9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Unsupported functions\n",
"Autograd supports many features. However, there are some functions that is not supported (yet) by Autograd.\n",
@@ -3469,8 +4000,11 @@
{
"cell_type": "code",
"execution_count": 24,
- "id": "b84fe928",
- "metadata": {},
+ "id": "fa821d4c",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3488,16 +4022,20 @@
},
{
"cell_type": "markdown",
- "id": "ac0bf243",
- "metadata": {},
+ "id": "4601ca85",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Here, Autograd tells us that an 'ArrayBox' does not support item assignment. The item assignment is done when the program tries to assign x[2] to the value 3. However, Autograd has implemented the computation of the derivative such that this assignment is not possible."
]
},
{
"cell_type": "markdown",
- "id": "fea32d24",
- "metadata": {},
+ "id": "646813ea",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## The syntax a.dot(b) when finding the dot product"
]
@@ -3505,8 +4043,11 @@
{
"cell_type": "code",
"execution_count": 25,
- "id": "922c7208",
- "metadata": {},
+ "id": "a83e6d33",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3524,8 +4065,10 @@
},
{
"cell_type": "markdown",
- "id": "dc7c262d",
- "metadata": {},
+ "id": "66408265",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Here we are told that the 'dot' function does not belong to Autograd's\n",
"version of a Numpy array. To overcome this, an alternative syntax\n",
@@ -3535,8 +4078,11 @@
{
"cell_type": "code",
"execution_count": 26,
- "id": "1636ee3a",
- "metadata": {},
+ "id": "5394dda5",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -3557,8 +4103,10 @@
},
{
"cell_type": "markdown",
- "id": "d62d7c1b",
- "metadata": {},
+ "id": "5668b5a9",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Recommended to avoid\n",
"The documentation recommends to avoid inplace operations such as"
@@ -3567,8 +4115,11 @@
{
"cell_type": "code",
"execution_count": 27,
- "id": "b1632217",
- "metadata": {},
+ "id": "ab83c77e",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"a += b\n",
@@ -3579,8 +4130,10 @@
},
{
"cell_type": "markdown",
- "id": "517b4f0f",
- "metadata": {},
+ "id": "971987ed",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Using Autograd with OLS\n",
"\n",
@@ -3592,8 +4145,11 @@
{
"cell_type": "code",
"execution_count": 28,
- "id": "9441c838",
- "metadata": {},
+ "id": "74c2088a",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"# Using Autograd to calculate gradients for OLS\n",
@@ -3649,235 +4205,23 @@
},
{
"cell_type": "markdown",
- "id": "a38f1162",
- "metadata": {},
+ "id": "c42fc6e5",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Same code but now with momentum gradient descent"
]
},
{
"cell_type": "code",
- "execution_count": 2,
- "id": "09b63595",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Own inversion\n",
- "[[4.]\n",
- " [3.]]\n",
- "Eigenvalues of Hessian Matrix:[0.25827439 4.23187426]\n",
- "0 [-14.70105464] [-16.45425295]\n",
- "1 [-0.08725637] [0.07708186]\n",
- "2 [-0.08193105] [0.0723775]\n",
- "3 [-0.07693074] [0.06796024]\n",
- "4 [-0.0722356] [0.06381258]\n",
- "5 [-0.06782701] [0.05991805]\n",
- "6 [-0.06368748] [0.05626121]\n",
- "7 [-0.05980058] [0.05282755]\n",
- "8 [-0.05615091] [0.04960344]\n",
- "9 [-0.05272398] [0.04657611]\n",
- "10 [-0.0495062] [0.04373353]\n",
- "11 [-0.0464848] [0.04106444]\n",
- "12 [-0.0436478] [0.03855825]\n",
- "13 [-0.04098394] [0.03620501]\n",
- "14 [-0.03848266] [0.03399539]\n",
- "15 [-0.03613403] [0.03192063]\n",
- "16 [-0.03392875] [0.02997249]\n",
- "17 [-0.03185805] [0.02814325]\n",
- "18 [-0.02991373] [0.02642565]\n",
- "19 [-0.02808807] [0.02481287]\n",
- "20 [-0.02637384] [0.02329852]\n",
- "21 [-0.02476422] [0.0218766]\n",
- "22 [-0.02325285] [0.02054145]\n",
- "23 [-0.02183371] [0.01928779]\n",
- "24 [-0.02050118] [0.01811064]\n",
- "25 [-0.01924998] [0.01700534]\n",
- "26 [-0.01807514] [0.01596749]\n",
- "27 [-0.016972] [0.01499298]\n",
- "28 [-0.01593619] [0.01407795]\n",
- "29 [-0.01496359] [0.01321876]\n",
- "30 [-0.01405035] [0.01241201]\n",
- "31 [-0.01319285] [0.0116545]\n",
- "32 [-0.01238768] [0.01094321]\n",
- "33 [-0.01163165] [0.01027534]\n",
- "34 [-0.01092176] [0.00964823]\n",
- "35 [-0.0102552] [0.00905939]\n",
- "36 [-0.00962931] [0.00850649]\n",
- "37 [-0.00904163] [0.00798733]\n",
- "38 [-0.00848981] [0.00749986]\n",
- "39 [-0.00797167] [0.00704214]\n",
- "40 [-0.00748516] [0.00661235]\n",
- "41 [-0.00702833] [0.00620879]\n",
- "42 [-0.00659939] [0.00582987]\n",
- "43 [-0.00619662] [0.00547407]\n",
- "44 [-0.00581844] [0.00513998]\n",
- "45 [-0.00546333] [0.00482628]\n",
- "46 [-0.0051299] [0.00453173]\n",
- "47 [-0.00481682] [0.00425516]\n",
- "48 [-0.00452285] [0.00399546]\n",
- "49 [-0.00424681] [0.00375162]\n",
- "50 [-0.00398763] [0.00352265]\n",
- "51 [-0.00374426] [0.00330766]\n",
- "52 [-0.00351575] [0.00310579]\n",
- "53 [-0.00330118] [0.00291624]\n",
- "54 [-0.0030997] [0.00273826]\n",
- "55 [-0.00291053] [0.00257115]\n",
- "56 [-0.0027329] [0.00241423]\n",
- "57 [-0.0025661] [0.00226688]\n",
- "58 [-0.00240949] [0.00212853]\n",
- "59 [-0.00226244] [0.00199863]\n",
- "60 [-0.00212436] [0.00187665]\n",
- "61 [-0.00199471] [0.00176212]\n",
- "62 [-0.00187297] [0.00165457]\n",
- "63 [-0.00175866] [0.00155359]\n",
- "64 [-0.00165133] [0.00145878]\n",
- "65 [-0.00155055] [0.00136975]\n",
- "66 [-0.00145592] [0.00128615]\n",
- "67 [-0.00136706] [0.00120766]\n",
- "68 [-0.00128363] [0.00113395]\n",
- "69 [-0.00120529] [0.00106475]\n",
- "70 [-0.00113173] [0.00099976]\n",
- "71 [-0.00106266] [0.00093875]\n",
- "72 [-0.0009978] [0.00088145]\n",
- "73 [-0.00093691] [0.00082766]\n",
- "74 [-0.00087973] [0.00077715]\n",
- "75 [-0.00082604] [0.00072972]\n",
- "76 [-0.00077562] [0.00068518]\n",
- "77 [-0.00072829] [0.00064336]\n",
- "78 [-0.00068384] [0.0006041]\n",
- "79 [-0.0006421] [0.00056723]\n",
- "80 [-0.00060291] [0.00053261]\n",
- "81 [-0.00056612] [0.00050011]\n",
- "82 [-0.00053157] [0.00046958]\n",
- "83 [-0.00049913] [0.00044093]\n",
- "84 [-0.00046866] [0.00041402]\n",
- "85 [-0.00044006] [0.00038875]\n",
- "86 [-0.0004132] [0.00036502]\n",
- "87 [-0.00038799] [0.00034274]\n",
- "88 [-0.00036431] [0.00032183]\n",
- "89 [-0.00034207] [0.00030219]\n",
- "90 [-0.0003212] [0.00028374]\n",
- "91 [-0.00030159] [0.00026643]\n",
- "92 [-0.00028319] [0.00025017]\n",
- "93 [-0.0002659] [0.0002349]\n",
- "94 [-0.00024968] [0.00022056]\n",
- "95 [-0.00023444] [0.0002071]\n",
- "96 [-0.00022013] [0.00019446]\n",
- "97 [-0.00020669] [0.00018259]\n",
- "98 [-0.00019408] [0.00017145]\n",
- "99 [-0.00018224] [0.00016099]\n",
- "theta from own gd\n",
- "[[3.99933748]\n",
- " [3.00058527]]\n",
- "0 [-0.00017111] [0.00015116]\n",
- "1 [-0.00016067] [0.00014194]\n",
- "2 [-0.00014773] [0.00013051]\n",
- "3 [-0.00013483] [0.00011911]\n",
- "4 [-0.00012274] [0.00010842]\n",
- "5 [-0.00011162] [9.86002251e-05]\n",
- "6 [-0.00010147] [8.96355442e-05]\n",
- "7 [-9.22300939e-05] [8.1475617e-05]\n",
- "8 [-8.38301216e-05] [7.40551223e-05]\n",
- "9 [-7.61939161e-05] [6.73093355e-05]\n",
- "10 [-6.9252884e-05] [6.11776614e-05]\n",
- "11 [-6.29440201e-05] [5.56044417e-05]\n",
- "12 [-5.72098413e-05] [5.0538896e-05]\n",
- "13 [-5.19980291e-05] [4.59348064e-05]\n",
- "14 [-4.72610077e-05] [4.17501447e-05]\n",
- "15 [-4.29555273e-05] [3.79467043e-05]\n",
- "16 [-3.90422759e-05] [3.4489757e-05]\n",
- "17 [-3.54855218e-05] [3.13477376e-05]\n",
- "18 [-3.22527882e-05] [2.84919565e-05]\n",
- "19 [-2.93145568e-05] [2.58963371e-05]\n",
- "20 [-2.66439985e-05] [2.35371789e-05]\n",
- "21 [-2.42167283e-05] [2.13929402e-05]\n",
- "22 [-2.20105826e-05] [1.94440417e-05]\n",
- "23 [-2.00054169e-05] [1.76726881e-05]\n",
- "24 [-1.81829219e-05] [1.60627049e-05]\n",
- "25 [-1.65264564e-05] [1.45993913e-05]\n",
- "26 [-1.5020895e-05] [1.32693857e-05]\n",
- "27 [-1.36524903e-05] [1.20605437e-05]\n",
- "28 [-1.24087475e-05] [1.09618272e-05]\n",
- "29 [-1.12783097e-05] [9.96320398e-06]\n",
- "30 [-1.02508548e-05] [9.05555536e-06]\n",
- "31 [-9.31700118e-06] [8.2305936e-06]\n",
- "32 [-8.46822168e-06] [7.48078591e-06]\n",
- "33 [-7.69676606e-06] [6.79928576e-06]\n",
- "34 [-6.99559011e-06] [6.17987033e-06]\n",
- "35 [-6.35829133e-06] [5.61688369e-06]\n",
- "36 [-5.77905052e-06] [5.10518517e-06]\n",
- "37 [-5.25257859e-06] [4.64010242e-06]\n",
- "38 [-4.77406829e-06] [4.21738875e-06]\n",
- "39 [-4.33915032e-06] [3.83318432e-06]\n",
- "40 [-3.9438534e-06] [3.48398095e-06]\n",
- "41 [-3.58456807e-06] [3.16659002e-06]\n",
- "42 [-3.25801365e-06] [2.87811344e-06]\n",
- "43 [-2.96120837e-06] [2.61591709e-06]\n",
- "44 [-2.69144207e-06] [2.37760685e-06]\n",
- "45 [-2.4462515e-06] [2.16100669e-06]\n",
- "46 [-2.2233978e-06] [1.96413882e-06]\n",
- "47 [-2.0208461e-06] [1.78520563e-06]\n",
- "48 [-1.83674687e-06] [1.62257327e-06]\n",
- "49 [-1.6694191e-06] [1.47475672e-06]\n",
- "50 [-1.51733489e-06] [1.34040628e-06]\n",
- "51 [-1.37910557e-06] [1.21829517e-06]\n",
- "52 [-1.25346895e-06] [1.10730839e-06]\n",
- "53 [-1.13927784e-06] [1.00643251e-06]\n",
- "54 [-1.03548954e-06] [9.14746433e-07]\n",
- "55 [-9.41156369e-07] [8.31412966e-07]\n",
- "56 [-8.55416954e-07] [7.55671183e-07]\n",
- "57 [-7.77488407e-07] [6.86829482e-07]\n",
- "58 [-7.06659155e-07] [6.24259266e-07]\n",
- "59 [-6.42282454e-07] [5.67389201e-07]\n",
- "60 [-5.83770475e-07] [5.15700003e-07]\n",
- "61 [-5.30588942e-07] [4.68719694e-07]\n",
- "62 [-4.8225225e-07] [4.26019295e-07]\n",
- "63 [-4.38319034e-07] [3.87208907e-07]\n",
- "64 [-3.98388138e-07] [3.51934146e-07]\n",
- "65 [-3.62094949e-07] [3.1987292e-07]\n",
- "66 [-3.29108073e-07] [2.90732473e-07]\n",
- "67 [-2.99126303e-07] [2.64246723e-07]\n",
- "68 [-2.71875874e-07] [2.40173828e-07]\n",
- "69 [-2.47107963e-07] [2.18293973e-07]\n",
- "70 [-2.24596409e-07] [1.98407376e-07]\n",
- "71 [-2.04135659e-07] [1.8033245e-07]\n",
- "72 [-1.85538885e-07] [1.63904151e-07]\n",
- "73 [-1.68636279e-07] [1.48972471e-07]\n",
- "74 [-1.53273502e-07] [1.35401067e-07]\n",
- "75 [-1.39310275e-07] [1.2306602e-07]\n",
- "76 [-1.26619098e-07] [1.11854697e-07]\n",
- "77 [-1.15084089e-07] [1.01664725e-07]\n",
- "78 [-1.0459992e-07] [9.24030614e-08]\n",
- "79 [-9.50708595e-08] [8.39851359e-08]\n",
- "80 [-8.64098971e-08] [7.63340839e-08]\n",
- "81 [-7.85379493e-08] [6.93800429e-08]\n",
- "82 [-7.1383136e-08] [6.30595163e-08]\n",
- "83 [-6.48801277e-08] [5.7314789e-08]\n",
- "84 [-5.89695428e-08] [5.20934079e-08]\n",
- "85 [-5.35974134e-08] [4.73476943e-08]\n",
- "86 [-4.87146848e-08] [4.3034316e-08]\n",
- "87 [-4.42767728e-08] [3.91138874e-08]\n",
- "88 [-4.02431562e-08] [3.55506087e-08]\n",
- "89 [-3.65770019e-08] [3.23119455e-08]\n",
- "90 [-3.32448344e-08] [2.93683246e-08]\n",
- "91 [-3.02162277e-08] [2.66928679e-08]\n",
- "92 [-2.74635268e-08] [2.4261146e-08]\n",
- "93 [-2.49615966e-08] [2.20509549e-08]\n",
- "94 [-2.26875948e-08] [2.00421096e-08]\n",
- "95 [-2.06207528e-08] [1.82162724e-08]\n",
- "96 [-1.87422006e-08] [1.65567687e-08]\n",
- "97 [-1.70347846e-08] [1.5048446e-08]\n",
- "98 [-1.54829141e-08] [1.36775317e-08]\n",
- "99 [-1.40724195e-08] [1.24315077e-08]\n",
- "theta from own gd wth momentum\n",
- "[[3.99999995]\n",
- " [3.00000004]]\n"
- ]
- }
- ],
+ "execution_count": 29,
+ "id": "dd515b08",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
"source": [
"# Using Autograd to calculate gradients for OLS\n",
"from random import random, seed\n",
@@ -3905,7 +4249,7 @@
"\n",
"theta = np.random.randn(2,1)\n",
"eta = 1.0/np.max(EigValues)\n",
- "Niterations = 100\n",
+ "Niterations = 30\n",
"\n",
"# define the gradient\n",
"training_gradient = grad(CostOLS)\n",
@@ -3936,37 +4280,23 @@
},
{
"cell_type": "markdown",
- "id": "108d652d",
- "metadata": {},
+ "id": "5e386451",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## But noen of these can compete with Newton's method"
]
},
{
"cell_type": "code",
- "execution_count": 1,
- "id": "31886c3a",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Own inversion\n",
- "[[4.08031249]\n",
- " [2.84228384]]\n",
- "Eigenvalues of Hessian Matrix:[0.28785076 4.70181637]\n",
- "0 [-9.75100318] [-11.34125693]\n",
- "1 [6.68649164e-15] [6.88970488e-15]\n",
- "2 [5.17814958e-16] [6.60680672e-16]\n",
- "3 [5.17814958e-16] [6.60680672e-16]\n",
- "4 [5.17814958e-16] [6.60680672e-16]\n",
- "beta from own Newton code\n",
- "[[4.08031249]\n",
- " [2.84228384]]\n"
- ]
- }
- ],
+ "execution_count": 30,
+ "id": "027bf4c7",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
"source": [
"# Using Newton's method\n",
"from random import random, seed\n",
@@ -4010,8 +4340,10 @@
},
{
"cell_type": "markdown",
- "id": "f6f8c822",
- "metadata": {},
+ "id": "c1f45ccb",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Including Stochastic Gradient Descent with Autograd\n",
"In this code we include the stochastic gradient descent approach discussed above. Note here that we specify which argument we are taking the derivative with respect to when using **autograd**."
@@ -4019,43 +4351,13 @@
},
{
"cell_type": "code",
- "execution_count": 2,
- "id": "ca99ff51",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Own inversion\n",
- "[[4.]\n",
- " [3.]]\n",
- "Eigenvalues of Hessian Matrix:[0.27490791 3.90226737]\n",
- "theta from own gd\n",
- "[[4.]\n",
- " [3.]]\n"
- ]
- },
- {
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkMAAAHFCAYAAADxOP3DAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA9A0lEQVR4nO3de1xUdeL/8fcBBNQUBRFBUKmfm7ctTS3N+7fUTNwML2WF5nbb1Mywi65tSWWubpm2ZmZb2mpeKvGSbRdXRWuzstQtu5i2VmiS10AxUYfz+2NiZGQGZpBhLuf1fDzm0c6Zz5zzOdBZ3n2uhmmapgAAACwqzN8VAAAA8CfCEAAAsDTCEAAAsDTCEAAAsDTCEAAAsDTCEAAAsDTCEAAAsDTCEAAAsDTCEAAAsDTCEBCAFixYIMMwHK+IiAglJibqxhtv1K5du/xWr8mTJ8swDL9dPxh8//33MgxDTz31lL+rAsBDEf6uAAD35s+frxYtWujkyZP6z3/+oylTpmjDhg365ptvVL9+fX9XDwBCAmEICGBt2rRRhw4dJEk9e/aUzWbTo48+qpUrV2rkyJF+rh386fTp045WQwDnh24yIIiUBKOff/7ZcezkyZMaP3682rZtq5iYGMXGxqpz585atWpVme8bhqExY8Zo4cKFatmypWrVqqVLL71Ua9asKVP2rbfeUtu2bRUVFaXU1FS33T4nT57UxIkTlZqaqsjISDVu3FijR4/WL7/84lSuWbNmSktL05o1a9SuXTvVrFlTLVu2dFx7wYIFatmypWrXrq3LL79cn376aYU/j5LuxA0bNujuu+9WgwYNFBcXp/T0dP30009l7n3y5MllztGsWTPdeuutZc65fv163XHHHYqLi1PdunU1fPhwFRYWKi8vT0OHDlW9evWUmJio+++/X6dPny5z3uLiYk2ZMkVNmjRRdHS0OnTooHXr1pUpt2vXLt10001q2LChoqKi1LJlSz333HNOZXJycmQYhhYuXKjx48ercePGioqK0u7du3XixAndf//9Sk1NVXR0tGJjY9WhQwctWbKkwp8fADv+kwIIInv27JEk/e53v3McKyoq0pEjR3T//fercePGOnXqlP79738rPT1d8+fP1/Dhw53O8dZbb2nLli167LHHdMEFF2j69Om6/vrrtXPnTl144YWSpHXr1um6665T586dtXTpUtlsNk2fPt0phEmSaZoaOHCg1q1bp4kTJ6pbt276/PPP9eijj2rz5s3avHmzoqKiHOX/+9//auLEiZo0aZJiYmKUlZWl9PR0TZw4UevWrdOTTz4pwzD00EMPKS0tTXv27FHNmjUr/Lncfvvt6t+/vxYvXqzc3Fw98MADuuWWW7R+/fpK/6xvv/12paena+nSpdq2bZv+/Oc/68yZM9q5c6fS09N155136t///remTZumpKQkZWZmOn1/9uzZatq0qWbOnKni4mJNnz5d/fr108aNG9W5c2dJ0ldffaUrr7xSTZo00dNPP61GjRrp3Xff1dixY3Xo0CE9+uijTuecOHGiOnfurLlz5yosLEwNGzZUZmamFi5cqCeeeELt2rVTYWGhduzYocOHD1f63gHLMQEEnPnz55uSzI8++sg8ffq0eezYMfOdd94xGzVqZHbv3t08ffq02++eOXPGPH36tHnbbbeZ7dq1c/pMkpmQkGAWFBQ4juXl5ZlhYWHm1KlTHceuuOIKMykpyfz1118dxwoKCszY2Fiz9P9tvPPOO6Ykc/r06U7XWbZsmSnJnDdvnuNY06ZNzZo1a5p79+51HNu+fbspyUxMTDQLCwsdx1euXGlKMlevXu3Rz2nUqFFOx6dPn25KMvfv3+90748++miZczRt2tQcMWJEmXPec889TuUGDhxoSjJnzJjhdLxt27bmZZdd5ni/Z88eU5Lbn9/VV1/tONa3b18zOTnZzM/PdzrnmDFjzOjoaPPIkSOmaZrmhg0bTElm9+7dy9S/TZs25sCBA8scB+A5usmAANapUyfVqFFDderU0TXXXKP69etr1apVZcaJvP766+rSpYsuuOACRUREqEaNGnrppZf09ddflzlnr169VKdOHcf7hIQENWzYUD/88IMkqbCwUFu2bFF6erqio6Md5erUqaMBAwY4nauk5aV0N5MkDRkyRLVr1y7TLdS2bVs1btzY8b5ly5aS7OOhatWqVeZ4SZ0q8oc//MHp/SWXXOLV911JS0tzel9Sp/79+5c57uo67n5+mzZtks1m08mTJ7Vu3Tpdf/31qlWrls6cOeN4XXvttTp58qQ++ugjp3MOGjSozHUuv/xyvf3225owYYJycnL066+/VvqeAasiDAEB7J///Ke2bNmi9evX66677tLXX3+tYcOGOZXJzs7W0KFD1bhxYy1atEibN2/Wli1b9Mc//lEnT54sc864uLgyx6Kiohx/RI8ePari4mI1atSoTLlzjx0+fFgRERGKj493Om4Yhho1alSmqyY2NtbpfWRkZLnHXdXflXPvqaRr7nyCgTd1dVVPdz+/U6dO6fjx4zp8+LDOnDmjv//976pRo4bT69prr5UkHTp0yOn7iYmJZc757LPP6qGHHtLKlSvVq1cvxcbGauDAgX5dggEINowZAgJYy5YtHYOme/XqJZvNpn/84x964403NHjwYEnSokWLlJqaqmXLljmtAVRUVFSpa9avX1+GYSgvL6/MZ+cei4uL05kzZ3Tw4EGnQGSapvLy8tSxY8dK1cEXoqKiXP5MfDW2xt3PLzIyUhdccIFq1Kih8PBwZWRkaPTo0S7PkZqa6vTe1RpPtWvXVlZWlrKysvTzzz87WokGDBigb775pmpuBghxtAwBQWT69OmqX7++HnnkERUXF0uy/4GMjIx0+kOZl5fncjaZJ0pmc2VnZzu1eBw7dkxvvvmmU9mrrrpKkj2QlbZ8+XIVFhY6Pg8EzZo10+eff+50bP369Tp+/LhPrufu59etWzeFh4erVq1a6tWrl7Zt26ZLLrlEHTp0KPNy1YpXnoSEBN16660aNmyYdu7cqRMnTlT1bQEhiZYhIIjUr19fEydO1IMPPqjFixfrlltuUVpamrKzszVq1CgNHjxYubm5evzxx5WYmFjprpLHH39c11xzjXr37q3x48fLZrNp2rRpql27to4cOeIo17t3b/Xt21cPPfSQCgoK1KVLF8dssnbt2ikjI6Oqbv28ZWRk6C9/+YseeeQR9ejRQ1999ZVmz56tmJgYn1wvPDxcvXv3VmZmpoqLizVt2jQVFBQoKyvLUWbWrFnq2rWrunXrprvvvlvNmjXTsWPHtHv3br355psezYa74oorlJaWpksuuUT169fX119/rYULF6pz585O47AAuEcYAoLMPffco9mzZ+uxxx7TsGHDNHLkSB04cEBz587Vyy+/rAsvvFATJkzQ3r17nf7weqN3795auXKlHn74Yd1www1q1KiRRo0apV9//dXpnIZhaOXKlZo8ebLmz5+vKVOmqEGDBsrIyNCTTz7pNK3e3x544AEVFBRowYIFeuqpp3T55Zfrtdde03XXXeeT640ZM0YnT57U2LFjdeDAAbVu3VpvvfWWunTp4ijTqlUrbd26VY8//rgefvhhHThwQPXq1VPz5s0d44Yq8n//939avXq1nnnmGZ04cUKNGzfW8OHDNWnSJJ/cFxCKDNM0TX9XAgAAwF8YMwQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACzNEusMFRcX66efflKdOnVcLmcPAAACj2maOnbsmJKSkhQW5rv2G0uEoZ9++kkpKSn+rgYAAKiE3NxcJScn++z8lghDderUkWT/YdatW9fPtQEAAJ4oKChQSkqK4++4r1giDJV0jdWtW5cwBABAkPH1EBcGUAMAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEvzexjatGmTBgwYoKSkJBmGoZUrV7ote9ddd8kwDM2cObPa6gcAAEKb38NQYWGhLr30Us2ePbvccitXrtTHH3+spKSkaqoZAACwggh/V6Bfv37q169fuWX27dunMWPG6N1331X//v2rqWYAAMAK/N4yVJHi4mJlZGTogQceUOvWrf1dHQAAEGL83jJUkWnTpikiIkJjx471+DtFRUUqKipyvC8oKPBF1QAAQAgI6Jahzz77TLNmzdKCBQtkGIbH35s6dapiYmIcr5SUFB/WEgAABLOADkPvv/++Dhw4oCZNmigiIkIRERH64YcfNH78eDVr1szt9yZOnKj8/HzHKzc3t/oqDQAAgkpAd5NlZGTo6quvdjrWt29fZWRkaOTIkW6/FxUVpaioKF9XDwAAhAC/h6Hjx49r9+7djvd79uzR9u3bFRsbqyZNmiguLs6pfI0aNdSoUSNdfPHF1V1VAAAQgvwehj799FP16tXL8T4zM1OSNGLECC1YsMBPtQIAAFbh9zDUs2dPmabpcfnvv//ed5UBAACWE9ADqAEAAHyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACyNMAQAACzN72Fo06ZNGjBggJKSkmQYhlauXOn47PTp03rooYf0+9//XrVr11ZSUpKGDx+un376yX8VBgAAIcXvYaiwsFCXXnqpZs+eXeazEydOaOvWrfrLX/6irVu3Kjs7W99++63+8Ic/+KGmAAAgFBmmaZr+rkQJwzC0YsUKDRw40G2ZLVu26PLLL9cPP/ygJk2aeHTegoICxcTEKD8/X3Xr1q2i2gIAAF+qrr/fET47s4/k5+fLMAzVq1fPbZmioiIVFRU53hcUFFRDzQAAQDDyezeZN06ePKkJEybopptuKjchTp06VTExMY5XSkpKNdYSAAAEk6AJQ6dPn9aNN96o4uJizZkzp9yyEydOVH5+vuOVm5tbTbUEAADBJii6yU6fPq2hQ4dqz549Wr9+fYX9hlFRUYqKiqqm2gEAgGAW8GGoJAjt2rVLGzZsUFxcnL+rBAAAQojfw9Dx48e1e/dux/s9e/Zo+/btio2NVVJSkgYPHqytW7dqzZo1stlsysvLkyTFxsYqMjLSX9UGAAAhwu9T63NyctSrV68yx0eMGKHJkycrNTXV5fc2bNignj17enQNptYDABB8LDO1vmfPniovjwXQMkgAACAEBc1sMgAAAF8gDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEvzexjatGmTBgwYoKSkJBmGoZUrVzp9bpqmJk+erKSkJNWsWVM9e/bUl19+6Z/KAgCAkOP3MFRYWKhLL71Us2fPdvn59OnTNWPGDM2ePVtbtmxRo0aN1Lt3bx07dqyaawoAAEJRhL8r0K9fP/Xr18/lZ6ZpaubMmZo0aZLS09MlSa+88ooSEhK0ePFi3XXXXdVZVQAAEIL83jJUnj179igvL099+vRxHIuKilKPHj304Ycfuv1eUVGRCgoKnF4AACA4HM87rpUTP9aYy9z/ra9Kfm8ZKk9eXp4kKSEhwel4QkKCfvjhB7ffmzp1qrKysnxaNwAAUHV++M9erZn1nd5cX1sbDv9ep3SFpOppzAjoMFTCMAyn96ZpljlW2sSJE5WZmel4X1BQoJSUFJ/VDwAAeMd2yqaP53+lNa8c1ptbG2tHUXNJyY7PUyN+VJ8WO/TCDt/XJaDDUKNGjSTZW4gSExMdxw8cOFCmtai0qKgoRUVF+bx+AADAc/k/5uu9mV/pzVU2/WtPSx02f+/4LEw2dam7Q2ldjmrAqBS1uPZCHTteTy/E+L5eAR2GUlNT1ahRI61du1bt2rWTJJ06dUobN27UtGnT/Fw7AAAszGaT3n9f2r9fSkyUunWTwsPLFNu97getmf293txYR5uO/l5n1NnxWYzy1a/Jl0rrb6rf+FaKvejS6rwDB7+HoePHj2v37t2O93v27NH27dsVGxurJk2aaNy4cXryySfVvHlzNW/eXE8++aRq1aqlm266yY+1BgDAIlyFnlWrpHvvlfbuPVsuOVmaNUunrxmgD1/8Um8u/EVrvmiinaculNTUUeziyP8p7fc/akBGPV15R2vVqHVl9d/TOQzTNE1/ViAnJ0e9evUqc3zEiBFasGCBTNNUVlaWXnjhBR09elRXXHGFnnvuObVp08bjaxQUFCgmJkb5+fmqW7duVVYfAIDQlZ1dNvTExUmHD5cpWhImhuufWqQMx/EInVb3+l8orXuB0kY3U/PezTy+fHX9/fZ7GKoOhCEAALyUnS0NHiy5iAmmJFfTmIplaK+S1UGf6poLv9WA68LVZ1wrxTSp3MCf6vr77fduMgAA4EeuusEke4uQm/YSd/O5w2SqiXK1/70vFN77Kt/U1wcIQwAAWJWrbrDkZOmOO5yPeSn80IEqqFz1IQwBAGBFbrrBzL37pEcfddv645FSy+EEg4DejgMAAPiAzea2G8yQqUoPJjYMKSXlbFdbkKBlCACAYObhej+lHZ6/SnHldIOVtJS4GyjtUsnOEDNnVnj9QEMYAgAgWLkb8zNrlpSe7jhUfKZY25bu1JqXftabnzRU8xNFWuLB6Q3JHnJKtyCVvD93in1ysj0IlbpusCAMAQAQjNxNfd+3Txo8WCdfWqT3dl+kNdlFWvPt77S/uKWklpKkC+ThAOesLOnFF8uGrZkzpeuu87pFKlCxzhAAAMHGZpOaNXM746tY0j4lq5m+V7HsAaW2jqtP4g6l9T2ta8f+PzX6w+X24OQqBhiGPfTs2WN/76fQwzpDAADAtfffL3fqe5ikFO3VkLBsxbduqLQbaqvnPb9XVN1OZwvNmmVvWXLVDSY5j/3p2bOq7yCgEIYAAPAnLwdAF+wt0NdPbdQVHpx6yT9Py7i5h+sP09OlN95wPeYoSMf+VBZhCACA6lYSgFatkl59VTp48OxnLgZA/y/nR635+x6tyblAOUd+ryvVQzkeXMZonFR+gfT0kBr7U1mEIQAAqkN5Aai0fftkDh6sr//4lF754jKt+W+yvir6f5KaOIrsj0jRL0a8Yk4fkuFqVaCSMT+erPcTHh7y3WAVIQwBAOArngag0kxTpgxd8NJMPaU9Kla4wnVG3ep9obSu+Uob1UQX97tIyp5rH/MjD8b8oFyEIQAAfMHVGkAeKtnw9LGEObpoaHv1va+V6qe2cy7EmJ8qw9R6AACqmrs1gLy1eLE0bFj5ZSqxAnWwYGo9AACBqKLwUc6+X17zZMNTxvycN8IQAACesNmkKVPsM72OHDl7/JzZX+bGTTIq0TXmxJsB0DhvhCEAANw5dUqaM0d6911p40bp11/Llvlt9tf2G/+ql3dcIePLL/Xs+VyTAdDVjjAEAIArDz4ozZhhbxEqz2+zv+KWzNYcjVc3V1PdvcEA6GpHGAIA4NxxQGvWSE8/7fHXS2Z/TUuZrd/d3FHFrzRWWN5Pno8bio+Xbr7ZvgBiCA2ADhaEIQCAtdhsUk6O/SXZg8dLL1VqCvy57p/WUBp2pdTxWdf7fpVGAAoYhCEAgHVkZ0t33ikdPuyb85fM/nK3BhABKCCxzhAAwBqys6VBg3x3/pQUac+estPsQ3QNoOrAOkMAAHjLXfiw2aSxY313XcNwPfuLNYCCAmEIABDcPNkBPjZW2rfPN9ePi5PmzWP2VxAjDAEAglcF+3+Ze/dJgwZpYditGl7V146NtV970iS6voIcYQgAELjOnfnVs6f9FR7u0f5fhkwVy1Df4reqpj5hYdK110rjxzP+J4QQhgAAgaP0mJ9du8puffHEE/Zuqblzpfvu82gdnzCZStBB2erHKvzokQrLSzq7CvSSJfa6fPeddNFF0qhRUmRkJW4MgYwwBAAIDBV0eZUwDx+WhgyR4eXpw0cMtw9y9gSrQFsKYQgA4H8edHmVMKTKbXhRsraPq3WGSsb/NG/OFHgLIgwBAPzLZrMHES+WvfOqVaj0DvDh4fZQ5G4cEiyJMAQAqHrlDXw+1/vvV8lWGC652gE+PFy66ir7CxBhCABQ1bKzpTvucD3w+Zz1eA58eVBfZ+Wox/lcr7z9vxj7Aw+E+bsCAIAQUrLlxREXs7YOH5Y5aJC+f2C2pvTOUec6X6hRmzg9mtOzctdKTpZee01q3Nj5eHy8NG6ctGGDfXsMghAqwN5kAIDKOXfriyuvtP/TVRD6jSkpV8lK1fcqlr3bqn30Dq0r/j/VPXVIhjdDo5cvtwcd9v8KWexNBgAIDDabtG6dtHChdPy41LWrvVXm/vudx/o0aFBuEJLsA5+baK8erDdPqf1b69p7myu5Yxspe659NpnK6fIqcW53G/t/4TzRMgQAcM1mk6ZMkaZOlU6erLC4KS9meS1eLA0b5nzM1TpDycnSbbfZ6yIx88tiaBkCAPhPdrbr9XjK4dV098TEssfS0+3T3unyQjUL+AHUZ86c0cMPP6zU1FTVrFlTF154oR577DEVFxf7u2oAEJpKFkD0Igh5pUEDe8hxpaTLa9gwWoBQbQK+ZWjatGmaO3euXnnlFbVu3VqffvqpRo4cqZiYGN17773+rh4ABJfSg40bNrQfO3DgbCuM5PUCiF6bM4eQg4AS8GFo8+bNuu6669S/f39JUrNmzbRkyRJ9+umnfq4ZAASZivb+Sk62rw/kqwUQJemBB6QhQ3x3fqASAr6brGvXrlq3bp2+/fZbSdJ///tfffDBB7r22mv9XDMACCIlXV/lBB1z716Zjz56ftcxDPtsL1dr/7z+ujR9+vmdH/CBgG8Zeuihh5Sfn68WLVooPDxcNptNU6ZM0bBzZyGUUlRUpKKiIsf7goKC6qgqAAQmD/f+MiQVy8uB0E4n+O2b8+YxEBpBJeDD0LJly7Ro0SItXrxYrVu31vbt2zVu3DglJSVpxIgRLr8zdepUZWVlVXNNAaAaebPQoBd7f51Xd8G5W1+w9g+CRMCvM5SSkqIJEyZo9OjRjmNPPPGEFi1apG+++cbld1y1DKWkpLDOEIDQ4G49nlmznLaeMItNffvuHu2eNF/9tz3h/XXK2/MrLEy67z4pLY3WH/gM6wz95sSJEwoLc/5vlfDw8HKn1kdFRSkqKsrXVQOA6lcy9ufckLJvnzR4sE4vXKr385rrzVcLtGZHU+0+faF66Cr1l5dhKCtLevFF58BVq5bUpo00dKh0zz1SZOT53w8QAAI+DA0YMEBTpkxRkyZN1Lp1a23btk0zZszQH//4R39XDQCqV3ljf0xTxZJ+viVTvfWDY9+vGjql6Pq1dOxkvC741YO9vwzD3so0aZL9xbgfWEDAd5MdO3ZMf/nLX7RixQodOHBASUlJGjZsmB555BFFevhfJWzHASAk5ORIvXpVWGygVqj+/4vTgOtrqPe41qqTVOdsi5LkvuurZAD0G2+w0zsCQnX9/Q74MFQVCEMAgl1RQZG+uf0pXfr6wxWWLV64SGG33Fz2g4rWGUpJcR4ADfgZY4YAIJR4M/vrN3mfH9C/ntmpNe/W0Hv726iDuijHg0uFJTd2/cG5e3+5WoGabjBYEGEIAHzNi9lf25ft1Jp/5OnNj+O1pbC1pIaOz3cbzXUkPF71zxx0vRZQyXgfd/t+SWf3/gLg4FUYys3NVUpKiq/qAgChp4LZX0Uvv6q1/7tIa944qTU7m2tfcQtJLRzFOtT6SmmXH1DabQlqd+PFCls91/XYn5LxPjNn0roDeMmrMUO1a9dWZmamJkyYoNq1a/uyXlWKMUMAfKa87i+bTWrWzO0YnWJJ+5SsZvreMfurlgrVu9EOpfU5pf73/U6JbRPKftFVSxPjfRCCAnIA9Ycffqj77rtPubm5mjJlikaOHOmzilUlwhCAKlUSgFatkhYtkg4dOvtZ6e4vD2d/DQ17TfGtGiptSC31Gvt7RdeL9rwOTHtHCAvIMFTin//8pyZNmqQGDRromWeeUc8A738mDAE4LyXBY98+6d//toego0ddl/2tu+rXFxdqx8rv1HFNxRufmotelXHzTVVZYyAkVNff70ptQzN8+HB9++23GjBggPr376/rr79eu3fvruq6AYD/ZWfbu7p69ZJuuUVasMB9EJLsix+apg7dPkEPreni0SWMxklVUlUAlVPpPflM01SfPn105513avXq1WrTpo3Gjx+vY8eOVWX9AMB/SgY/e7jJaYkwSSnaq0TjZx2tEe9+zWfDsI/1KW/2FwCf82o22dy5c7VlyxZt2bJFX3/9tcLDw3XJJZdo9OjRatu2rV599VW1atVKK1asUIcOHXxVZwCoGhUNfna39YWHFi2UjJrM/gICnVdjhlJSUtSpUyfHq0OHDmU2RH3yySe1ePFi7dixo8orW1mMGQLgUBKAVqyQXnlFys8/+1klBj+Xa8MG+5o+zP4CKiWgB1CX5+eff1ZSUpJsNltVnva8EIYASKp4OwpJMgydeXWpvn37O7Va+OfKXadk8cM9e5xbmpj9BXglaLfjaNiwodavX1/VpwUAz7kKHqtWuV788BzFpqm8mzI1Rq/ovP6f7NzuL1Z+BgJWlYchwzDUo0ePqj4tAHjG3dYXv/7q0fifMEnJ2qcY5etweLxibW62vnCH7i8g6LA3GYDgVHrtn4MHpfh46bvvpMmTy4YeL2eDSdLyf/6qsNpzPWpNUt260h//aN8Ele4vIOgQhgAEF5tNmjLFPtD5yBGfXSYspbG9W+uNN9yPM4qNtX82aRIBCAhiVT6AOhAxgBoIYqXH/+zaJT37rHT4sG+vGR9vv965g59Lt0I1bkwrEOBjQTuAGgDOW3l7f1WH555j8DNgIYQhAIGhdAB69VV7C4w/PPCANGSIf64NwC8IQwD8z5P1f86XYdjH+ERH27u7zhUfL82Zc3a1aACWQRgCUL3OXQPo0CFp6NDz2vaiQiVbX8ybZ5/xxfgfAKUQhgBUD3ezwMLDfRuEJPs6Q6XX/mH8D4BSCEMAqo67WVcHD0p33+16Fpivtu7JypKaN2frCwAVIgwBOH/VtPaPR1gBGoCXCEMAzk92tnTnnb5f+8ed2Fj7OKCrrmLsD4BKIQwBqJjNJq1bJy1cKB0/LnXtKt1zj7RmjWfbVVSl5GTpjjvoAgNQZViBGkD5srOl4cOlwkLn42FhUq1a9nDka/Hx0s03s/cXYDGsQA3A/7KzpUGDXH9WXFy1QSg83HkwNQEIQDUhDAFwzWaTMjKq73pLlpzdE4zuLwDViDAEwLV166QTJ3x/nbg4+2KIzP4C4Cdh/q4AgMBjFps6OmWOby8SF2dfC+jnnwlCAPyKliEglJ279UU5XU8nfzmpnNk7tOa1Qq356iI9bTPkZrSQ91JSpBkzpAYN6AYDEHAIQ0AoKR1+du2ydz+V3pQ0Odm+MOJvLTF5nx/QWzN2as17NbR2fxsVqoOj6EfqpEFaWfE1DcMebn766eyx2FjpD3+Qrr6atX8ABDym1gOhwMMVoE3DkEzp9TaT9dSeQdpS2Nrp86Sw/Ur73S6lDY7W/93VXLWbNrDPGitPZqY0fbrHLVAA4Knq+vtNGAKCVUkr0KpV0ssvSwUFHn2tWIb2Klmp2qNihatDra804IoDSrstQe2GtZARZpwt/OCD0t/+5v5kHTtKn3xynjcCAK6xzhAA97KzpXvvlfbu9fqrYTLVRLla3fc5XfbXG5TYtpWkVq4LT59u/+eMGc5rABmGNG6c/TgABDlahoBA5G73927d7C1BVbEFxuLF0rBhnpU9dUqaM0f67jvpooukUaOkyMjzuz4AVICWIcCqymv1SU6Wfv21avYCS0z0vGxkpL0lCABCEGEI8CebTcrJsb8k+6DjrCy3xc29e2W4/dRDhmEPVd26ne+ZACAkEIaA6lYSgJ5/XnrrLenkSY+/WiVBSJJmzmS2FwD8JihWoN63b59uueUWxcXFqVatWmrbtq0+++wzf1cLqFhJ8FmyxP7PN96QEhLs6+8sX+5VEKoSycn2OrDiMwA4BHzL0NGjR9WlSxf16tVLb7/9tho2bKjvvvtO9erV83fVAPc8XPfH5+LipLFjpebNWf8HANwI+DA0bdo0paSkaP78+Y5jzZo181+FgIpkZ0t33ikdPuy/OpSEoEmTCD8AUIGA7yZbvXq1OnTooCFDhqhhw4Zq166dXnzxxXK/U1RUpIKCAqcXUC2ys+3T3n0ZhAzDHnYaN3Y+Hh9vn/G1YYN989NHHiEIAYAHAn6doejoaElSZmamhgwZok8++UTjxo3TCy+8oOHDh7v8zuTJk5XlYkYO6wzBp2w2qVmzSi2E6BXDsI/7ue46tsAAENLYjuM3kZGR6tChgz788EPHsbFjx2rLli3avHmzy+8UFRWpqKjI8b6goEApKSmEIfhWTo7Uq5dvr5GSYp8JxgBoABbAoou/SUxMVKtWzlsFtGzZUsuXL3f7naioKEVFRfm6aghlpXd/r6DV5cCXB/WvZ3bq6PJNuq8q65CVZV/t+dwVqGn9AYAqFfBhqEuXLtq5c6fTsW+//VZNmzb1U40Q8lytAJ2cbJ8Zlp4us9jU5298qzUv7tebmxvok8JWMtVVPXSmasJQXJw0bx6tPwBQTQI+DN1333268sor9eSTT2ro0KH65JNPNG/ePM2bN8/fVUMwc9fyUzIA+pzeY3PfPmnQIL2Y8rie+Gmkcm0XS7rY8fllNb9Wzw42ndqRoBq/HJDhbe9zdLSUlib96U9Sz560/gBANQr4MUOStGbNGk2cOFG7du1SamqqMjMzdccdd3j8fTZqhRN3LT8zZkiZmW4HQBfL0F4lK1V7FKUiXZ3whdKuLlL/cc3VuEPi2XMPHmz/364erbg4+8rTsbFnt+Do2ZMABAAuMIC6ChGGLMpV64+7Hd8NQ6ZperTdxeYRz6vtjBGqGVvTdQFXYYt1fwDAawygBs6Hq0DSuLF9+wtX+d80Zcqzvb86942R3AUhyT7Wh2nvABA0CEMIPW7G/WjfvnK/5vEKpImJFZcJD7d3fQEAAh5hCMHJ3QBom83eIuSL3l/DsI8t6tat6s8NAPAbwhCCR0kAWrVKWrRIOnTo7GclU99jY6tmBWjDcA5Uxm8daDNn0t0FACGGMITAdeqUNGeO9N13UmGh9N577ru69u2TOWiwtl00SJdV9nolLT8zZkj33Vd2thkrPwNASCIMITDdf7/0zDNScbFn5U1Tpgw1/m5j5a5XuuUnPV26/noGQAOARRCG4D/uxv0MHGjvCvNSmEwl6KBO1IhRzdMFMuRi3JBh2LvSatYsv+WHAdAAYBmEIVSv0uN+Xn3Vvu9WieRkaejQSgWh0mqNHmkfPyQ3437mzWPqOwDAgUUX4VulW3927pRmz5YOH3Zd9txBy5W1YYN05EjZdYbY8R0AggqLLiL4ZWfbV12uYH0fh/MNQqWnvoeH0/oDAPAIYQi+kZ0tDRpU/dctPfWdcT8AAA8QhlA57gY/l3x2553VWx+6wAAAlUQYgvdef10aNcr1oofp6fbd2N2NC6pK8fHSzTfbu8PoAgMAVBJhCBUr3Qq0apW0bFnZMnv3SoMHy3ztdeUt+rc82L2rcrp0kUaPZgwQAKDKEIbg7Nzur0OHyq7G7IZpmvppyFjN1616uDLXLpn6fv/90pIlzteMiZFeeEG64YbKnBkAALcIQzjLVfeXFwxJjfWTCnRB5a5feuHDqVOZCQYAqBaEIasraQl6+mlpzZoqOeUTLzSS/hzn+bihtDRp/HjnwMNMMABANSEMWYWr2V+rVpVdmLAKRP4u1b7Kc0VT6+Pj7RuxDh5cpdcHAMAbhKFQVhKAVq6UFiyQ8vPPfhbnRcuNN+Ljz7bwLF9eNmwxAwwAEGAIQ6EqO7v8Vh9fTX1/7rmzASc9nVWgAQABjzAUirKz7V1P1b3t3AMPSEOGOB9j7A8AIMARhoKVuxWgbTZ7i1B1BiHG/gAAghhhKJiUBKBVq6RFi1yvAB0bW+UDostITpbuuENq3pyuLwBA0CMMBbrSAejVV6WDB12X27dP5uDB+rrNELWq6jqMH2+f/s64HwBACCIMBbKKBkGXZpoyZSjuiw1Vd326vwAAFkAYChTnjgE6eNC+9YQXY3/CZCpBB3U8Ika1zxTIkAffPXeKPVPfAQAWQxgKBK5agMLDKz0I+oIxI+3jhyr6+gMPsO0FAMDyCEP+5m4avM1W+XOWtOq462KLj7evB1QyDZ6p7wAACyMM+VNVT4M3DPtMr5LWnZIFD/fts3e7xcdLjRvT+gMAQCmEoap06pR9wPF330kXXWTfAT4y0m3xQy+tUoOqngY/cyabnQIA4AXCUFV58EFpxgzn7q3775cyM6Xp0yVJxWeKtXXxN1rz8gG9+UmCfvdrkZZU1fVTUuxBKD29qs4IAIAlEIYq49wWoNxcexA6l80m829/0+5NP2na0bv01u7fKa+4lfTbSkB1tf/86sHMLwAAzhthyFuuWoAqkPrxUr2il3VGkbpAx9Qn6Uul9T2ta+9tJaUl28f0uBs3VLLFRgkCEAAAVYow5I0HH5T+9jevvmJIipBNSxMzVXfsSHUf1UZRdTudLTBrln02mWE4ByLDsP9z6VKpQQOmvgMA4COEIU+dOuW6K8xDgwYZ0oT2ZT9IT5feeKPsNPjkZMYAAQBQDQhDnpoz5/zW/rnoIvefpaefnQZPCxAAANXKemHo3G0vygkdZ06e0Ycvfqk3/3lUl362VbdU9prh4fZp9hWVYRo8AADVzlphaMIE6fXXpUOHzh5LTraP2/mtO+ronl/0zoyv9Oabpt7+sbV+MS+VJN2r7bpFCyt33czMctcbAgAA/hPm7wp4a+rUqTIMQ+PGjfP+y88/7xyEJJn79skcNFir2j+mHvW2K/7CC3TT7Cu15Icu+sWsp1jjiG5J/UBdxrSVGebljys83L7/12/rDAEAgMATVC1DW7Zs0bx583TJJZdU2TkN01SxDLXb+g99oEkqVrhaR+1SWtt9GjAiVp1ua63wyK72wjXHlz+b7L77pCZNPF6BGgAA+F/QhKHjx4/r5ptv1osvvqgnnniiSs8dJlNNlKvXu/9d7R4fpNTuzSU1L1uwpIXn3HWGwsOdVpoGAADBI2i6yUaPHq3+/fvr6quvrrBsUVGRCgoKnF6eSP9TglK7p5RfaPp06cQJ6ZlnpDFj7P88cYIgBABAkAqKlqGlS5dq69at2rJli0flp06dqqysLO8vlJjoWbnISKkyY5YAAEDACfiWodzcXN17771atGiRoqOjPfrOxIkTlZ+f73jl5uaW/wXDsG902q1bFdQYAAAEE8M03W2KFRhWrlyp66+/XuGl1gKy2WwyDENhYWEqKipy+syVgoICxcTEKF9SXVcFDMO+CjSrPQMAEDAcf7/z81W3rsu/4FUi4LvJrrrqKn3xxRdOx0aOHKkWLVrooYceqjAIVSglhW0vAACwsIAPQ3Xq1FGbNm2cjtWuXVtxcXFljnuMnd8BAMBvAj4MVal//MO+/g8BCAAA/CbgxwxVherqcwQAAFWnuv5+B/xsMgAAAF8iDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsjDAEAAEsL+DA0depUdezYUXXq1FHDhg01cOBA7dy509/VAgAAISLgw9DGjRs1evRoffTRR1q7dq3OnDmjPn36qLCw0N9VAwAAIcAwTdP0dyW8cfDgQTVs2FAbN25U9+7dPfpOQUGBYmJilJ+fr7p16/q4hgAAoCpU19/vCJ+d2Ufy8/MlSbGxsW7LFBUVqaioyPG+oKDA5/UCAADBKeC7yUozTVOZmZnq2rWr2rRp47bc1KlTFRMT43ilpKRUYy0BAEAwCapustGjR+utt97SBx98oOTkZLflXLUMpaSk0E0GAEAQoZvsHPfcc49Wr16tTZs2lRuEJCkqKkpRUVHVVDMAABDMAj4Mmaape+65RytWrFBOTo5SU1P9XSUAABBCAj4MjR49WosXL9aqVatUp04d5eXlSZJiYmJUs2ZNP9cOAAAEu4AfM2QYhsvj8+fP16233urROZhaDwBA8GHM0G8CPKsBAIAgF1RT6wEAAKoaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFgaYQgAAFha0IShOXPmKDU1VdHR0Wrfvr3ef/99f1cJAACEgKAIQ8uWLdO4ceM0adIkbdu2Td26dVO/fv30448/+rtqAAAgyBmmaZr+rkRFrrjiCl122WV6/vnnHcdatmypgQMHaurUqRV+v6CgQDExMcrPz1fdunV9WVUAAFBFquvvd8C3DJ06dUqfffaZ+vTp43S8T58++vDDD/1UKwAAECoi/F2Bihw6dEg2m00JCQlOxxMSEpSXl+fyO0VFRSoqKnK8z8/Pl2RPmAAAIDiU/N32dSdWwIehEoZhOL03TbPMsRJTp05VVlZWmeMpKSk+qRsAAPCdw4cPKyYmxmfnD/gw1KBBA4WHh5dpBTpw4ECZ1qISEydOVGZmpuP9L7/8oqZNm+rHH3/06Q8z0BQUFCglJUW5ubmWGivFfXPfVsB9c99WkJ+fryZNmig2Ntan1wn4MBQZGan27dtr7dq1uv766x3H165dq+uuu87ld6KiohQVFVXmeExMjKX+JSpRt25d7ttCuG9r4b6txar3HRbm2yHOAR+GJCkzM1MZGRnq0KGDOnfurHnz5unHH3/Un/70J39XDQAABLmgCEM33HCDDh8+rMcee0z79+9XmzZt9K9//UtNmzb1d9UAAECQC4owJEmjRo3SqFGjKvXdqKgoPfrooy67zkIZ9819WwH3zX1bAfft2/sOikUXAQAAfCXgF10EAADwJcIQAACwNMIQAACwNMIQAACwtKAMQ3PmzFFqaqqio6PVvn17vf/+++WW37hxo9q3b6/o6GhdeOGFmjt3bpkyy5cvV6tWrRQVFaVWrVppxYoVvqp+pXlz39nZ2erdu7fi4+NVt25dde7cWe+++65TmQULFsgwjDKvkydP+vpWvOLNfefk5Li8p2+++capXKj9vm+99VaX9926dWtHmWD4fW/atEkDBgxQUlKSDMPQypUrK/xOKDzf3t53qDzf3t53qDzf3t53qDzfU6dOVceOHVWnTh01bNhQAwcO1M6dOyv8XnU840EXhpYtW6Zx48Zp0qRJ2rZtm7p166Z+/frpxx9/dFl+z549uvbaa9WtWzdt27ZNf/7znzV27FgtX77cUWbz5s264YYblJGRof/+97/KyMjQ0KFD9fHHH1fXbVXI2/vetGmTevfurX/961/67LPP1KtXLw0YMEDbtm1zKle3bl3t37/f6RUdHV0dt+QRb++7xM6dO53uqXnz5o7PQvH3PWvWLKf7zc3NVWxsrIYMGeJULtB/34WFhbr00ks1e/Zsj8qHyvPt7X2HyvPt7X2XCPbn29v7DpXne+PGjRo9erQ++ugjrV27VmfOnFGfPn1UWFjo9jvV9oybQebyyy83//SnPzkda9GihTlhwgSX5R988EGzRYsWTsfuuusus1OnTo73Q4cONa+55hqnMn379jVvvPHGKqr1+fP2vl1p1aqVmZWV5Xg/f/58MyYmpqqq6BPe3veGDRtMSebRo0fdntMKv+8VK1aYhmGY33//veNYMPy+S5NkrlixotwyofJ8l+bJfbsSjM93aZ7cd6g836VV5vcdCs+3aZrmgQMHTEnmxo0b3Zaprmc8qFqGTp06pc8++0x9+vRxOt6nTx99+OGHLr+zefPmMuX79u2rTz/9VKdPny63jLtzVrfK3Pe5iouLdezYsTKb3R0/flxNmzZVcnKy0tLSyvyXpT+dz323a9dOiYmJuuqqq7Rhwwanz6zw+37ppZd09dVXl1mlPZB/35URCs93VQjG5/t8BPPzXRVC5fnOz8+XpHI3Ya2uZzyowtChQ4dks9nK7FafkJBQZlf7Enl5eS7LnzlzRocOHSq3jLtzVrfK3Pe5nn76aRUWFmro0KGOYy1atNCCBQu0evVqLVmyRNHR0erSpYt27dpVpfWvrMrcd2JioubNm6fly5crOztbF198sa666ipt2rTJUSbUf9/79+/X22+/rdtvv93peKD/visjFJ7vqhCMz3dlhMLzfb5C5fk2TVOZmZnq2rWr2rRp47ZcdT3jQbMdR2mGYTi9N02zzLGKyp973Ntz+kNl67hkyRJNnjxZq1atUsOGDR3HO3XqpE6dOjned+nSRZdddpn+/ve/69lnn626ip8nb+774osv1sUXX+x437lzZ+Xm5uqpp55S9+7dK3VOf6lsHRcsWKB69epp4MCBTseD5fftrVB5visr2J9vb4TS811ZofJ8jxkzRp9//rk++OCDCstWxzMeVC1DDRo0UHh4eJm0d+DAgTKpsESjRo1clo+IiFBcXFy5Zdyds7pV5r5LLFu2TLfddptee+01XX311eWWDQsLU8eOHQPmvyTO575L69Spk9M9hfLv2zRNvfzyy8rIyFBkZGS5ZQPt910ZofB8n49gfr6rSrA93+cjVJ7ve+65R6tXr9aGDRuUnJxcbtnqesaDKgxFRkaqffv2Wrt2rdPxtWvX6sorr3T5nc6dO5cp/95776lDhw6qUaNGuWXcnbO6Vea+Jft/Md56661avHix+vfvX+F1TNPU9u3blZiYeN51rgqVve9zbdu2zemeQvX3Ldlna+zevVu33XZbhdcJtN93ZYTC811Zwf58V5Vge77PR7A/36ZpasyYMcrOztb69euVmppa4Xeq7Rn3eKh1gFi6dKlZo0YN86WXXjK/+uorc9y4cWbt2rUdo+onTJhgZmRkOMr/73//M2vVqmXed9995ldffWW+9NJLZo0aNcw33njDUeY///mPGR4ebv71r381v/76a/Ovf/2rGRERYX700UfVfn/ueHvfixcvNiMiIsznnnvO3L9/v+P1yy+/OMpMnjzZfOedd8zvvvvO3LZtmzly5EgzIiLC/Pjjj6v9/tzx9r6feeYZc8WKFea3335r7tixw5wwYYIpyVy+fLmjTCj+vkvccsst5hVXXOHynMHw+z527Ji5bds2c9u2baYkc8aMGea2bdvMH374wTTN0H2+vb3vUHm+vb3vUHm+vb3vEsH+fN99991mTEyMmZOT4/Tv7YkTJxxl/PWMB10YMk3TfO6558ymTZuakZGR5mWXXeY0LW/EiBFmjx49nMrn5OSY7dq1MyMjI81mzZqZzz//fJlzvv766+bFF19s1qhRw2zRooXTwxUovLnvHj16mJLKvEaMGOEoM27cOLNJkyZmZGSkGR8fb/bp08f88MMPq/GOPOPNfU+bNs286KKLzOjoaLN+/fpm165dzbfeeqvMOUPt922apvnLL7+YNWvWNOfNm+fyfMHw+y6ZOu3u39tQfb69ve9Qeb69ve9Qeb4r8+95KDzfru5Zkjl//nxHGX8948ZvFQQAALCkoBozBAAAUNUIQwAAwNIIQwAAwNIIQwAAwNIIQwAAwNIIQwAAwNIIQwAAwNIIQwAAwNIIQwAAwNIIQwAAwNIIQwCC0pIlSxQdHa19+/Y5jt1+++265JJLlJ+f78eaAQg27E0GICiZpqm2bduqW7dumj17trKysvSPf/xDH330kRo3buzv6gEIIhH+rgAAVIZhGJoyZYoGDx6spKQkzZo1S++//z5BCIDXaBkCENQuu+wyffnll3rvvffUo0cPf1cHQBBizBCAoPXuu+/qm2++kc1mU0JCgr+rAyBI0TIEICht3bpVPXv21HPPPaelS5eqVq1aev311/1dLQBBiDFDAILO999/r/79+2vChAnKyMhQq1at1LFjR3322Wdq3769v6sHIMjQMgQgqBw5ckRdunRR9+7d9cILLziOX3fddSoqKtI777zjx9oBCEaEIQAAYGkMoAYAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJZGGAIAAJb2/wEHxx8L5Bq8/AAAAABJRU5ErkJggg==\n",
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "theta from own sdg\n",
- "[[3.99475389]\n",
- " [3.00496962]]\n"
- ]
- }
- ],
+ "execution_count": 31,
+ "id": "f2c2d0dd",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
"source": [
"# Using Autograd to calculate gradients using SGD\n",
"# OLS example\n",
@@ -4071,7 +4373,7 @@
"\n",
"n = 100\n",
"x = 2*np.random.rand(n,1)\n",
- "y = 4+3*x#+np.random.randn(n,1)\n",
+ "y = 4+3*x+np.random.randn(n,1)\n",
"\n",
"X = np.c_[np.ones((n,1)), x]\n",
"XT_X = X.T @ X\n",
@@ -4134,8 +4436,10 @@
},
{
"cell_type": "markdown",
- "id": "d9ff15bb",
- "metadata": {},
+ "id": "27cfcd23",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Same code but now with momentum gradient descent"
]
@@ -4143,8 +4447,11 @@
{
"cell_type": "code",
"execution_count": 32,
- "id": "c62525ac",
- "metadata": {},
+ "id": "8d2b23d8",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"# Using Autograd to calculate gradients using SGD\n",
@@ -4218,33 +4525,23 @@
},
{
"cell_type": "markdown",
- "id": "bb7ee229",
- "metadata": {},
+ "id": "58a8f732",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Similar (second order function now) problem but now with AdaGrad"
]
},
{
"cell_type": "code",
- "execution_count": 3,
- "id": "67ef0175",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Own inversion\n",
- "[[2.]\n",
- " [3.]\n",
- " [4.]]\n",
- "theta from own AdaGrad\n",
- "[[2.]\n",
- " [3.]\n",
- " [4.]]\n"
- ]
- }
- ],
+ "execution_count": 33,
+ "id": "6781ed86",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
"source": [
"# Using Autograd to calculate gradients using AdaGrad and Stochastic Gradient descent\n",
"# OLS example\n",
@@ -4303,16 +4600,20 @@
},
{
"cell_type": "markdown",
- "id": "2f8a1fab",
- "metadata": {},
+ "id": "358c2c6e",
+ "metadata": {
+ "editable": true
+ },
"source": [
"Running this code we note an almost perfect agreement with the results from matrix inversion."
]
},
{
"cell_type": "markdown",
- "id": "5875fd60",
- "metadata": {},
+ "id": "ef808ab2",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## RMSprop for adaptive learning rate with Stochastic Gradient Descent"
]
@@ -4320,8 +4621,11 @@
{
"cell_type": "code",
"execution_count": 34,
- "id": "eb223dd3",
- "metadata": {},
+ "id": "4e115811",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"# Using Autograd to calculate gradients using RMSprop and Stochastic Gradient descent\n",
@@ -4372,11 +4676,11 @@
"\t# Previous value for the outer product of gradients\n",
" Previous = Giter\n",
"\t# Accumulated gradient\n",
- " Giter +=gradients @ gradients.T\n",
+ "# Giter +=gradients @ gradients.T\n",
"\t# Scaling with rho the new and the previous results\n",
- " Gnew = (rho*Previous+(1-rho)*Giter)\n",
+ " Giter = (rho*Giter+(1-rho)*gradients*gradients)\n",
"\t# Taking the diagonal only and inverting\n",
- " Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]\n",
+ " Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]\n",
"\t# Hadamard product\n",
" update = np.multiply(Ginverse,gradients)\n",
" theta -= update\n",
@@ -4386,8 +4690,10 @@
},
{
"cell_type": "markdown",
- "id": "a46bd7bd",
- "metadata": {},
+ "id": "33e4f0c7",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## And Logistic Regression"
]
@@ -4395,8 +4701,11 @@
{
"cell_type": "code",
"execution_count": 35,
- "id": "cbbe2787",
- "metadata": {},
+ "id": "5d85c167",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import autograd.numpy as np\n",
@@ -4436,8 +4745,10 @@
},
{
"cell_type": "markdown",
- "id": "dbf7aa1d",
- "metadata": {},
+ "id": "d9c27c54",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Introducing [JAX](https://jax.readthedocs.io/en/latest/)\n",
"\n",
@@ -4453,8 +4764,11 @@
{
"cell_type": "code",
"execution_count": 36,
- "id": "7e253afa",
- "metadata": {},
+ "id": "0c5ee173",
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
"outputs": [],
"source": [
"import jax.numpy as jnp\n",
@@ -4470,8 +4784,10 @@
},
{
"cell_type": "markdown",
- "id": "684b49a8",
- "metadata": {},
+ "id": "650b565f",
+ "metadata": {
+ "editable": true
+ },
"source": [
"## Weekend challenge\n",
"\n",
@@ -4483,25 +4799,7 @@
]
}
],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.9.14"
- }
- },
+ "metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
diff --git a/doc/src/week39/week39.do.txt b/doc/src/week39/week39.do.txt
index 6224f0adc..f53b4c92e 100644
--- a/doc/src/week39/week39.do.txt
+++ b/doc/src/week39/week39.do.txt
@@ -2553,11 +2553,11 @@ for epoch in range(n_epochs):
# Previous value for the outer product of gradients
Previous = Giter
# Accumulated gradient
- Giter +=gradients @ gradients.T
+# Giter +=gradients @ gradients.T
# Scaling with rho the new and the previous results
- Gnew = (rho*Previous+(1-rho)*Giter)
+ Giter = (rho*Giter+(1-rho)*gradients*gradients)
# Taking the diagonal only and inverting
- Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Gnew)))]
+ Ginverse = np.c_[eta/(delta+np.sqrt(np.diagonal(Giter)))]
# Hadamard product
update = np.multiply(Ginverse,gradients)
theta -= update