From 8cb6536df6fa70a736a726154b2e1a298e2b7f7b Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Thu, 24 Nov 2022 15:18:49 +0100 Subject: [PATCH] update --- doc/pub/week47/html/._week47-bs000.html | 2 +- doc/pub/week47/html/._week47-bs001.html | 3 + doc/pub/week47/html/week47-bs.html | 2 +- doc/pub/week47/html/week47-reveal.html | 7 +- doc/pub/week47/html/week47-solarized.html | 5 +- doc/pub/week47/html/week47.html | 5 +- doc/pub/week47/ipynb/ipynb-week47-src.tar.gz | Bin 823750 -> 823750 bytes doc/pub/week47/ipynb/week47.ipynb | 1336 +++++++++++------- doc/src/week47/week47.do.txt | 1 + 9 files changed, 883 insertions(+), 478 deletions(-) diff --git a/doc/pub/week47/html/._week47-bs000.html b/doc/pub/week47/html/._week47-bs000.html index 9a32e7092..6bd878513 100644 --- a/doc/pub/week47/html/._week47-bs000.html +++ b/doc/pub/week47/html/._week47-bs000.html @@ -429,7 +429,7 @@ MathJax.Hub.Config({
-

Nov 22, 2022

+

Nov 24, 2022


diff --git a/doc/pub/week47/html/._week47-bs001.html b/doc/pub/week47/html/._week47-bs001.html index b45089314..6d0ec4c16 100644 --- a/doc/pub/week47/html/._week47-bs001.html +++ b/doc/pub/week47/html/._week47-bs001.html @@ -415,6 +415,9 @@ MathJax.Hub.Config({
diff --git a/doc/pub/week47/html/week47-bs.html b/doc/pub/week47/html/week47-bs.html index 9a32e7092..6bd878513 100644 --- a/doc/pub/week47/html/week47-bs.html +++ b/doc/pub/week47/html/week47-bs.html @@ -429,7 +429,7 @@ MathJax.Hub.Config({
-

Nov 22, 2022

+

Nov 24, 2022


diff --git a/doc/pub/week47/html/week47-reveal.html b/doc/pub/week47/html/week47-reveal.html index d990df214..f569c3359 100644 --- a/doc/pub/week47/html/week47-reveal.html +++ b/doc/pub/week47/html/week47-reveal.html @@ -184,7 +184,7 @@ MathJax.Hub.Config({
-

Nov 22, 2022

+

Nov 24, 2022


@@ -199,6 +199,11 @@ MathJax.Hub.Config({

diff --git a/doc/pub/week47/html/week47-solarized.html b/doc/pub/week47/html/week47-solarized.html index 00e2bd442..c0f7075b6 100644 --- a/doc/pub/week47/html/week47-solarized.html +++ b/doc/pub/week47/html/week47-solarized.html @@ -333,7 +333,7 @@ MathJax.Hub.Config({

-

Nov 22, 2022

+

Nov 24, 2022


@@ -342,6 +342,9 @@ MathJax.Hub.Config({
diff --git a/doc/pub/week47/html/week47.html b/doc/pub/week47/html/week47.html index 76585db59..1731d29e7 100644 --- a/doc/pub/week47/html/week47.html +++ b/doc/pub/week47/html/week47.html @@ -410,7 +410,7 @@ MathJax.Hub.Config({
-

Nov 22, 2022

+

Nov 24, 2022


@@ -419,6 +419,9 @@ MathJax.Hub.Config({
  • Thursday: Dimensionality reduction and unsupervised learning: Principal Component analysis (PCA) and clustering
    • +
  • Friday: PCA and clustering and Summary of Course
diff --git a/doc/pub/week47/ipynb/ipynb-week47-src.tar.gz b/doc/pub/week47/ipynb/ipynb-week47-src.tar.gz index 80de7002c77040ad30b92576521194c01ac9f99a..1f1ddaafcfe99ef60381d1475cdef86d06f21c53 100644 GIT binary patch delta 54 zcmX@M*yz||BX;?24hGxW`bPFvcE(nArdD?5R(6(FcGgyQwpMnaAV(`ZXDd5bD?9gA IcAotc0i=Zv4*&oF delta 54 zcmX@M*yz||BX;?24hE&Nnnw0kcE(nArdD?5R(6(FcGgyQwpMnaAV(`ZXDd5bD?9gA IcAotc0iIh8>Hq)$ diff --git a/doc/pub/week47/ipynb/week47.ipynb b/doc/pub/week47/ipynb/week47.ipynb index 5989e5399..5c90e5720 100644 --- a/doc/pub/week47/ipynb/week47.ipynb +++ b/doc/pub/week47/ipynb/week47.ipynb @@ -2,8 +2,10 @@ "cells": [ { "cell_type": "markdown", - "id": "a83498c9", - "metadata": {}, + "id": "1eb5bea9", + "metadata": { + "editable": true + }, "source": [ "\n", @@ -12,26 +14,32 @@ }, { "cell_type": "markdown", - "id": "d56f0819", - "metadata": {}, + "id": "7e2b5d28", + "metadata": { + "editable": true + }, "source": [ "# Week 47: Unsupervised learning (PCA and Clustering) and Summary of Course\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", "\n", - "Date: **Nov 22, 2022**\n", + "Date: **Nov 24, 2022**\n", "\n", "Copyright 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license" ] }, { "cell_type": "markdown", - "id": "12eb854d", - "metadata": {}, + "id": "3d0a3fd6", + "metadata": { + "editable": true + }, "source": [ "## Overview of week 47\n", "\n", "* **Thursday**: Dimensionality reduction and unsupervised learning: Principal Component analysis (PCA) and clustering\n", "\n", + " * [Video of lecture](https://youtu.be/VJIsEQM2lCI)\n", + "\n", "* **Friday**: PCA and clustering and Summary of Course\n", "\n", "1. We recommend highly the video on PCA by [Brunton and Kutz](http://www.databookuw.com/page-2/page-4/), see in particular the video of section 1.5. Repeating about the singular value discussion is also very useful as we will use this material as background.\n", @@ -49,8 +57,10 @@ }, { "cell_type": "markdown", - "id": "8824691f", - "metadata": {}, + "id": "f8c766e3", + "metadata": { + "editable": true + }, "source": [ "## Basic ideas of the Principal Component Analysis (PCA)\n", "\n", @@ -74,8 +84,10 @@ }, { "cell_type": "markdown", - "id": "905a4a02", - "metadata": {}, + "id": "9c37d5ab", + "metadata": { + "editable": true + }, "source": [ "## Introducing the Covariance and Correlation functions\n", "\n", @@ -88,8 +100,10 @@ }, { "cell_type": "markdown", - "id": "4783f702", - "metadata": {}, + "id": "4b6fa50c", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -100,16 +114,20 @@ }, { "cell_type": "markdown", - "id": "162f14c5", - "metadata": {}, + "id": "fdcdd0c1", + "metadata": { + "editable": true + }, "source": [ "where for example" ] }, { "cell_type": "markdown", - "id": "27f9fe87", - "metadata": {}, + "id": "8a225893", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] =\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})(y_i- \\overline{y}).\n", @@ -118,16 +136,20 @@ }, { "cell_type": "markdown", - "id": "9cd3e4de", - "metadata": {}, + "id": "a7348be2", + "metadata": { + "editable": true + }, "source": [ "With this definition and recalling that the variance is defined as" ] }, { "cell_type": "markdown", - "id": "116e2869", - "metadata": {}, + "id": "64b8a051", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathrm{var}[\\boldsymbol{x}]=\\frac{1}{n} \\sum_{i=0}^{n-1}(x_i- \\overline{x})^2,\n", @@ -136,16 +158,20 @@ }, { "cell_type": "markdown", - "id": "732f9a40", - "metadata": {}, + "id": "aa0751a7", + "metadata": { + "editable": true + }, "source": [ "we can rewrite the covariance matrix as" ] }, { "cell_type": "markdown", - "id": "57dbc1a5", - "metadata": {}, + "id": "41abf50e", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}] & \\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -156,8 +182,10 @@ }, { "cell_type": "markdown", - "id": "583861a5", - "metadata": {}, + "id": "0ea17959", + "metadata": { + "editable": true + }, "source": [ "## More on the covariance\n", "The covariance takes values between zero and infinity and may thus\n", @@ -169,8 +197,10 @@ }, { "cell_type": "markdown", - "id": "aed6523a", - "metadata": {}, + "id": "f35829ff", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]=\\frac{\\mathrm{cov}[\\boldsymbol{x},\\boldsymbol{y}]}{\\sqrt{\\mathrm{var}[\\boldsymbol{x}] \\mathrm{var}[\\boldsymbol{y}]}}.\n", @@ -179,8 +209,10 @@ }, { "cell_type": "markdown", - "id": "29c685ab", - "metadata": {}, + "id": "a1882301", + "metadata": { + "editable": true + }, "source": [ "The correlation function is then given by values $\\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}]\n", "\\in [-1,1]$. This avoids eventual problems with too large values. We\n", @@ -190,8 +222,10 @@ }, { "cell_type": "markdown", - "id": "9fe3f5f7", - "metadata": {}, + "id": "0f89ccab", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{K}[\\boldsymbol{x},\\boldsymbol{y}] = \\begin{bmatrix} 1 & \\mathrm{corr}[\\boldsymbol{x},\\boldsymbol{y}] \\\\\n", @@ -202,16 +236,20 @@ }, { "cell_type": "markdown", - "id": "219eba8c", - "metadata": {}, + "id": "e72e2111", + "metadata": { + "editable": true + }, "source": [ "In the above example this is the function we constructed using **pandas**." ] }, { "cell_type": "markdown", - "id": "1f1a61e3", - "metadata": {}, + "id": "8ed82b4e", + "metadata": { + "editable": true + }, "source": [ "## Reminding ourselves about Linear Regression\n", "In our derivation of the various regression algorithms like **Ordinary Least Squares** or **Ridge regression**\n", @@ -220,8 +258,10 @@ }, { "cell_type": "markdown", - "id": "b0d9acdb", - "metadata": {}, + "id": "d845fb26", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -237,8 +277,10 @@ }, { "cell_type": "markdown", - "id": "dd640d7e", - "metadata": {}, + "id": "d920b16a", + "metadata": { + "editable": true + }, "source": [ "with $\\boldsymbol{X}\\in {\\mathbb{R}}^{n\\times p}$, with the predictors/features $p$ refering to the column numbers and the\n", "entries $n$ being the row elements.\n", @@ -247,8 +289,10 @@ }, { "cell_type": "markdown", - "id": "307231e1", - "metadata": {}, + "id": "ea9ecda9", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix} \\boldsymbol{x}_0 & \\boldsymbol{x}_1 & \\boldsymbol{x}_2 & \\dots & \\dots & \\boldsymbol{x}_{p-1}\\end{bmatrix},\n", @@ -257,16 +301,20 @@ }, { "cell_type": "markdown", - "id": "b165621f", - "metadata": {}, + "id": "f82b9880", + "metadata": { + "editable": true + }, "source": [ "with a given vector" ] }, { "cell_type": "markdown", - "id": "569eb876", - "metadata": {}, + "id": "bd19fedb", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{x}_i^T = \\begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \\dots & \\dots x_{n-1,i}\\end{bmatrix}.\n", @@ -275,8 +323,10 @@ }, { "cell_type": "markdown", - "id": "8f53560e", - "metadata": {}, + "id": "e44bfb1b", + "metadata": { + "editable": true + }, "source": [ "## Simple Example\n", "With these definitions, we can now rewrite our $2\\times 2$\n", @@ -287,8 +337,10 @@ }, { "cell_type": "markdown", - "id": "6011f58c", - "metadata": {}, + "id": "354e6773", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\begin{bmatrix}\n", @@ -304,8 +356,10 @@ }, { "cell_type": "markdown", - "id": "766c7d25", - "metadata": {}, + "id": "48f55c1e", + "metadata": { + "editable": true + }, "source": [ "## The Correlation Matrix\n", "\n", @@ -314,8 +368,10 @@ }, { "cell_type": "markdown", - "id": "f47cfea6", - "metadata": {}, + "id": "c4d7e097", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{K}[\\boldsymbol{x}] = \\begin{bmatrix}\n", @@ -331,8 +387,10 @@ }, { "cell_type": "markdown", - "id": "1776b677", - "metadata": {}, + "id": "41206f50", + "metadata": { + "editable": true + }, "source": [ "## Numpy Functionality\n", "\n", @@ -345,8 +403,10 @@ }, { "cell_type": "markdown", - "id": "80e8b835", - "metadata": {}, + "id": "fbd32bf9", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{W}^T = \\begin{bmatrix} x_0 & y_0 \\\\\n", @@ -361,8 +421,10 @@ }, { "cell_type": "markdown", - "id": "4385fe3d", - "metadata": {}, + "id": "2a4097ef", + "metadata": { + "editable": true + }, "source": [ "which in turn is converted into into the $2\\times 2$ covariance matrix\n", "$\\boldsymbol{C}$ via the Numpy function **np.cov()**. We note that we can also calculate\n", @@ -374,8 +436,11 @@ { "cell_type": "code", "execution_count": 1, - "id": "43385082", - "metadata": {}, + "id": "3378d3d8", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# Importing various packages\n", @@ -392,8 +457,10 @@ }, { "cell_type": "markdown", - "id": "eb9ebeb2", - "metadata": {}, + "id": "0606a2bc", + "metadata": { + "editable": true + }, "source": [ "## Correlation Matrix again\n", "\n", @@ -407,8 +474,11 @@ { "cell_type": "code", "execution_count": 2, - "id": "4b1df3a0", - "metadata": {}, + "id": "b0363975", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np\n", @@ -436,8 +506,10 @@ }, { "cell_type": "markdown", - "id": "8c8e3bc7", - "metadata": {}, + "id": "ac2a399c", + "metadata": { + "editable": true + }, "source": [ "We see that the matrix elements along the diagonal are one as they\n", "should be and that the matrix is symmetric. Furthermore, diagonalizing\n", @@ -448,8 +520,10 @@ }, { "cell_type": "markdown", - "id": "28dd243b", - "metadata": {}, + "id": "e2486df1", + "metadata": { + "editable": true + }, "source": [ "## Using Pandas\n", "\n", @@ -459,8 +533,11 @@ { "cell_type": "code", "execution_count": 3, - "id": "5840dacd", - "metadata": {}, + "id": "a3880fe4", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np\n", @@ -480,8 +557,10 @@ }, { "cell_type": "markdown", - "id": "425d4eab", - "metadata": {}, + "id": "87105092", + "metadata": { + "editable": true + }, "source": [ "## And then the Franke Function\n", "\n", @@ -491,8 +570,11 @@ { "cell_type": "code", "execution_count": 4, - "id": "c80e7c58", - "metadata": {}, + "id": "e7408c3a", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# Common imports\n", @@ -542,8 +624,10 @@ }, { "cell_type": "markdown", - "id": "b161b5f6", - "metadata": {}, + "id": "bc3c4fc5", + "metadata": { + "editable": true + }, "source": [ "We note here that the covariance is zero for the first rows and\n", "columns since all matrix elements in the design matrix were set to one\n", @@ -555,8 +639,10 @@ }, { "cell_type": "markdown", - "id": "cd30963b", - "metadata": {}, + "id": "d024409a", + "metadata": { + "editable": true + }, "source": [ "## Links with the Design Matrix\n", "\n", @@ -565,8 +651,10 @@ }, { "cell_type": "markdown", - "id": "00d7863a", - "metadata": {}, + "id": "2f87d2f2", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}= \\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}].\n", @@ -575,16 +663,20 @@ }, { "cell_type": "markdown", - "id": "65099d0c", - "metadata": {}, + "id": "b813f1ad", + "metadata": { + "editable": true + }, "source": [ "To see this let us simply look at a design matrix $\\boldsymbol{X}\\in {\\mathbb{R}}^{2\\times 2}$" ] }, { "cell_type": "markdown", - "id": "606c1b67", - "metadata": {}, + "id": "26e3e0d8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -598,8 +690,10 @@ }, { "cell_type": "markdown", - "id": "bb8a9d26", - "metadata": {}, + "id": "e3802c47", + "metadata": { + "editable": true + }, "source": [ "## Computing the Expectation Values\n", "\n", @@ -608,8 +702,10 @@ }, { "cell_type": "markdown", - "id": "2886e7a9", - "metadata": {}, + "id": "d9bf9642", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}=\\begin{bmatrix}\n", @@ -621,16 +717,20 @@ }, { "cell_type": "markdown", - "id": "11ee22ef", - "metadata": {}, + "id": "c4ec3e2d", + "metadata": { + "editable": true + }, "source": [ "which is just" ] }, { "cell_type": "markdown", - "id": "81d72aa6", - "metadata": {}, + "id": "96bf08f7", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]=\\begin{bmatrix} \\mathrm{var}[\\boldsymbol{x}_0] & \\mathrm{cov}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] \\\\\n", @@ -641,8 +741,10 @@ }, { "cell_type": "markdown", - "id": "329fbab3", - "metadata": {}, + "id": "14511ecd", + "metadata": { + "editable": true + }, "source": [ "where we wrote $$\\boldsymbol{C}[\\boldsymbol{x}_0,\\boldsymbol{x}_1] = \\boldsymbol{C}[\\boldsymbol{x}]$$ to indicate that this the covariance of the vectors $\\boldsymbol{x}$ of the design/feature matrix $\\boldsymbol{X}$.\n", "\n", @@ -651,8 +753,10 @@ }, { "cell_type": "markdown", - "id": "a5b0eee8", - "metadata": {}, + "id": "a267f0ce", + "metadata": { + "editable": true + }, "source": [ "## Towards the PCA theorem\n", "\n", @@ -661,8 +765,10 @@ }, { "cell_type": "markdown", - "id": "8e77dcd3", - "metadata": {}, + "id": "7a6a6bb9", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}] = \\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X}= \\mathbb{E}[\\boldsymbol{X}^T\\boldsymbol{X}].\n", @@ -671,8 +777,10 @@ }, { "cell_type": "markdown", - "id": "16eb1305", - "metadata": {}, + "id": "a9b54353", + "metadata": { + "editable": true + }, "source": [ "Let us now assume that we can perform a series of orthogonal transformations where we employ some orthogonal matrices $\\boldsymbol{S}$.\n", "These matrices are defined as $\\boldsymbol{S}\\in {\\mathbb{R}}^{p\\times p}$ and obey the orthogonality requirements $\\boldsymbol{S}\\boldsymbol{S}^T=\\boldsymbol{S}^T\\boldsymbol{S}=\\boldsymbol{I}$. The matrix can be written out in terms of the column vectors $\\boldsymbol{s}_i$ as $\\boldsymbol{S}=[\\boldsymbol{s}_0,\\boldsymbol{s}_1,\\dots,\\boldsymbol{s}_{p-1}]$ and $\\boldsymbol{s}_i \\in {\\mathbb{R}}^{p}$.\n", @@ -684,8 +792,10 @@ }, { "cell_type": "markdown", - "id": "6eededfc", - "metadata": {}, + "id": "20beb610", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{y}] = \\mathbb{E}[\\boldsymbol{S}^T\\boldsymbol{X}^T\\boldsymbol{X}T\\boldsymbol{S}]=\\boldsymbol{S}^T\\boldsymbol{C}[\\boldsymbol{x}]\\boldsymbol{S},\n", @@ -694,16 +804,20 @@ }, { "cell_type": "markdown", - "id": "2b13cd39", - "metadata": {}, + "id": "263042f1", + "metadata": { + "editable": true + }, "source": [ "since the matrix $\\boldsymbol{S}$ is not a data dependent matrix. Multiplying with $\\boldsymbol{S}$ from the left we have" ] }, { "cell_type": "markdown", - "id": "079510b9", - "metadata": {}, + "id": "4733e534", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{S}\\boldsymbol{C}[\\boldsymbol{y}] = \\boldsymbol{C}[\\boldsymbol{x}]\\boldsymbol{S},\n", @@ -712,16 +826,20 @@ }, { "cell_type": "markdown", - "id": "3fb5f232", - "metadata": {}, + "id": "7cd0a8d3", + "metadata": { + "editable": true + }, "source": [ "and since $\\boldsymbol{C}[\\boldsymbol{y}]$ is diagonal we have for a given eigenvalue $i$ of the covariance matrix that" ] }, { "cell_type": "markdown", - "id": "f83004d5", - "metadata": {}, + "id": "b75971de", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{S}_i\\lambda_i = \\boldsymbol{C}[\\boldsymbol{x}]\\boldsymbol{S}_i.\n", @@ -730,8 +848,10 @@ }, { "cell_type": "markdown", - "id": "f301a0b5", - "metadata": {}, + "id": "b6fe0bc4", + "metadata": { + "editable": true + }, "source": [ "## More on the PCA Theorem\n", "\n", @@ -766,8 +886,10 @@ }, { "cell_type": "markdown", - "id": "21f8a7ed", - "metadata": {}, + "id": "d370d658", + "metadata": { + "editable": true + }, "source": [ "## A kind of Bird's view on PCA\n", "\n", @@ -798,8 +920,10 @@ }, { "cell_type": "markdown", - "id": "e48271a8", - "metadata": {}, + "id": "80863908", + "metadata": { + "editable": true + }, "source": [ "## Writing our own PCA code\n", "\n", @@ -809,8 +933,10 @@ }, { "cell_type": "markdown", - "id": "321d6031", - "metadata": {}, + "id": "4576c767", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mu = (-1,2) \\qquad \\Sigma = \\begin{bmatrix} 4 & 2 \\\\\n", @@ -821,8 +947,10 @@ }, { "cell_type": "markdown", - "id": "def29dc6", - "metadata": {}, + "id": "dc4ec2de", + "metadata": { + "editable": true + }, "source": [ "Note that the mean refers to each column of data. \n", "We will generate $n = 10000$ points $X = \\{ x_1, \\ldots, x_N \\}$ from\n", @@ -831,8 +959,10 @@ }, { "cell_type": "markdown", - "id": "40064229", - "metadata": {}, + "id": "0af7320d", + "metadata": { + "editable": true + }, "source": [ "## Implementing it\n", "The following Python code aids in setting up the data and writing out the design matrix.\n", @@ -841,9 +971,12 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "6c97695a", - "metadata": {}, + "execution_count": 5, + "id": "efed5557", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "%matplotlib inline\n", @@ -854,22 +987,26 @@ "from IPython.display import display\n", "n = 10000\n", "mean = (-1, 2)\n", - "cov = [[20, 0.2], [0.2, 0.2]]\n", + "cov = [[4, 2], [2, 2]]\n", "X = np.random.multivariate_normal(mean, cov, n)" ] }, { "cell_type": "markdown", - "id": "2785547a", - "metadata": {}, + "id": "5b02320f", + "metadata": { + "editable": true + }, "source": [ "Now we are going to implement the PCA algorithm. We will break it down into various substeps." ] }, { "cell_type": "markdown", - "id": "498a2380", - "metadata": {}, + "id": "41731f97", + "metadata": { + "editable": true + }, "source": [ "## First Step\n", "\n", @@ -878,8 +1015,10 @@ }, { "cell_type": "markdown", - "id": "dd44ee8c", - "metadata": {}, + "id": "6ecb030d", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\mu_n = \\frac{1}{n} \\sum_{i=1}^n x_i\n", @@ -888,16 +1027,20 @@ }, { "cell_type": "markdown", - "id": "1f02d9a1", - "metadata": {}, + "id": "edb7d948", + "metadata": { + "editable": true + }, "source": [ "and the mean-centered data $\\bar{X} = \\{ \\bar{x}_1, \\ldots, \\bar{x}_n \\}$ takes the form" ] }, { "cell_type": "markdown", - "id": "86add3a3", - "metadata": {}, + "id": "ef817c3a", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\bar{x}_i = x_i - \\mu_n.\n", @@ -906,8 +1049,10 @@ }, { "cell_type": "markdown", - "id": "e83136dc", - "metadata": {}, + "id": "9d5d6092", + "metadata": { + "editable": true + }, "source": [ "When you are done with these steps, print out $\\mu_n$ to verify it is\n", "close to $\\mu$ and plot your mean centered data to verify it is\n", @@ -917,9 +1062,12 @@ }, { "cell_type": "code", - "execution_count": 11, - "id": "211651ae", - "metadata": {}, + "execution_count": 6, + "id": "63f962f0", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "df = pd.DataFrame(X)\n", @@ -931,8 +1079,10 @@ }, { "cell_type": "markdown", - "id": "87f4dcc5", - "metadata": {}, + "id": "ef6f36fb", + "metadata": { + "editable": true + }, "source": [ "## Scaling\n", "Alternatively, we could use the functions we discussed\n", @@ -948,8 +1098,10 @@ }, { "cell_type": "markdown", - "id": "89a3192e", - "metadata": {}, + "id": "ffeda9ec", + "metadata": { + "editable": true + }, "source": [ "## Centered Data\n", "\n", @@ -958,8 +1110,10 @@ }, { "cell_type": "markdown", - "id": "e7ffd716", - "metadata": {}, + "id": "da40df7e", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\Sigma_n = \\frac{1}{n-1} \\sum_{i=1}^n \\bar{x}_i^T \\bar{x}_i = \\frac{1}{n-1} \\sum_{i=1}^n (x_i - \\mu_n)^T (x_i - \\mu_n)\n", @@ -968,8 +1122,10 @@ }, { "cell_type": "markdown", - "id": "665b60d5", - "metadata": {}, + "id": "4d9253e3", + "metadata": { + "editable": true + }, "source": [ "where the data points $x_i \\in \\mathbb{R}^p$ (here in this example $p = 2$) are column vectors and $x^T$ is the transpose of $x$.\n", "We can write our own code or simply use either the functionaly of **numpy** or that of **pandas**, as follows" @@ -977,22 +1133,13 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "472fab3c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " 0 1\n", - "0 19.993702 0.184113\n", - "1 0.184113 0.198691\n", - "[[19.99370195 0.18411281]\n", - " [ 0.18411281 0.19869054]]\n" - ] - } - ], + "execution_count": 7, + "id": "5fad3fd7", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], "source": [ "print(df.cov())\n", "print(np.cov(X_centered.T))" @@ -1000,8 +1147,10 @@ }, { "cell_type": "markdown", - "id": "a15ca582", - "metadata": {}, + "id": "22efa6a2", + "metadata": { + "editable": true + }, "source": [ "Note that the way we define the covariance matrix here has a factor $n-1$ instead of $n$. This is included in the **cov()** function by **numpy** and **pandas**. \n", "Our own code here is not very elegant and asks for obvious improvements. It is tailored to this specific $2\\times 2$ covariance matrix." @@ -1009,30 +1158,13 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "0a380846", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Centered covariance using own code\n", - "[[19.99370195 0.18411281]\n", - " [ 0.18411281 0.19869054]]\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAGdCAYAAADT1TPdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2RElEQVR4nO3dfXBc5WH3/d9KQisZW7JeiGUh+UXGbgumgMG1pITy4tqJb8ahCaG4nmGcZxqTFBwHDEOgToNxYlxCAy6l1ANhgPxBIPfDnbQZYIpiHAiRRYxjprlJpn4NkgHhR5KtdYy9wtJ5/pCuo2vPnl2t5F3tJfv7mdHYu3v27HX25Zzfud5OxPM8TwAAAI4qyHcBAAAA0iGsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcVpTvApyugYEBffDBB5oyZYoikUi+iwMAADLgeZ6OHTum2tpaFRSkrzuZ8GHlgw8+UH19fb6LAQAAxqCjo0N1dXVpl5nwYWXKlCmSBje2rKwsz6UBAACZiMViqq+v94/j6Uz4sGKafsrKyggrAABMMJl04aCDLQAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAAp+U0rLzxxhtavny5amtrFYlE9NOf/jThcc/ztGHDBtXW1qq0tFRXX3213n333VwWCQAATDA5DSvHjx/XJZdcosceeyz08e9973t6+OGH9dhjj2nnzp2qqanRkiVLdOzYsVwWCwAATCBFuVz5smXLtGzZstDHPM/Tli1btH79en3xi1+UJD377LOaNm2annvuOX31q1/NZdEAAMAEkbc+KwcPHlRnZ6eWLl3q3xeNRnXVVVeptbU1X8UCAACOyWnNSjqdnZ2SpGnTpiXcP23aNL333nspnxePxxWPx/3bsVgsNwUEAABOyPtooEgkknDb87yk+2ybN29WeXm5/1dfX5/rIgIAgDzKW1ipqamRNFzDYhw+fDiptsV27733qre31//r6OjIaTkBAEB+5S2szJ49WzU1NWppafHv6+vr0+uvv67m5uaUz4tGoyorK0v4AwAAZ66c9ln54x//qH379vm3Dx48qHfeeUeVlZWaMWOGbr/9dj3wwAOaO3eu5s6dqwceeECTJk3SypUrc1ksAAAwgeQ0rLz99tu65ppr/Nvr1q2TJK1atUrPPPOM7r77bp04cUK33nqrjhw5okWLFunVV1/VlClTclksAAAwgUQ8z/PyXYjTEYvFVF5ert7eXpqEAACYIEZz/M77aCAAAIB0CCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOC0vIeVDRs2KBKJJPzV1NTku1gAAMARRfkugCRddNFF+vnPf+7fLiwszGNpAACAS5wIK0VFRdSmAACAUHlvBpKkvXv3qra2VrNnz9aKFSt04MCBlMvG43HFYrGEPwAAcObKe1hZtGiRfvjDH+q//uu/9OSTT6qzs1PNzc3q7u4OXX7z5s0qLy/3/+rr68e5xAAAYDxFPM/z8l0I2/HjxzVnzhzdfffdWrduXdLj8Xhc8Xjcvx2LxVRfX6/e3l6VlZWNZ1EBAMAYxWIxlZeXZ3T8dqLPiu3cc8/VxRdfrL1794Y+Ho1GFY1Gx7lUAAAgX/LeDBQUj8f1+9//XtOnT893UQAAgAPyHlbuuusuvf766zp48KDeeustfelLX1IsFtOqVavyXTQAAOCAvDcDHTp0SH/7t3+rrq4unXfeeWpsbFRbW5tmzpyZ76IBAAAH5D2sPP/88/kuAgAAcFjem4EAAADSIawAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAHOQo+07NGj2/aGPvbotr16pGXPOJcoM/ks90R9z4AzAWEFGEGmB6l8HMzG+pqFBRE9HPLcR7ft1cMte1RYEMl6WbPhdMs92vfLXj742mZ5198z4ExQlO8CAK4zBylJ6h/wVFgQ0drFc/2D1Lol8yRJO//Qo9b93ZKktYvn+s8PLicNHgTNeoIe3bZX/QOe7rCWz6RsI72mzSxrP9d+Tli5Ugnblkesg7e9LY+07NHOP/Ro4azKpO3LZLtHKnf/gKdHt+1N+b6+dbBbbQd61HagW8+tbkx47OGWPWqeU6VHWvb4ZQh7fx9u2aO2A91q3d+t5jlVat3fPer3bDyM9TuWre8mkE2EFZxRcrGjtQ9S5uBkDlbmIPXotr3+wSuTADDWkJGubCO9ZrrnPvbaPvX1D4zpoGsCgL1O+77Ghkp/WRPo/u/7vQmfQ6qwYHukZY/eOtit5jnVWrdkXkK56ytK9eOdHVJEOnTkhH78dof+5op6SYNhybxu85wqlZUUqXV/t1Y+2abnVjcmvHbr/m41NlT5r1dYEPFfy2yf+ewjUlJQcelAP5rvmF3u4PNMuc39mX43gWwirOCMkq0QEGQf2AsjEf/AFxYOzO10ASAYMuwDanD5VAe54IHRfs26ilK17u/S2sVzk5YLrq8gIvX1D6i4sGBMtQPNc6rVdqAn4X0viAw3iZj/m0AnSbGTp/SZB1/Tm9+81n+/BsvcrQHPS3qNT//TNh07eUqxk6fUdqBH65bMU3Fhgfr6ByRJHUdO+MvWV5Sq48gJvzxlJUWKnTyl4sKI//omsMy65yX/OYNBpdIv6/+7q0PvHz2p5jlVfmDZ8vM9Ghgqnjf03tnv2Yu/OaRDR06odX+XmudUJ7znD7fsUWNDZcL7n61wE1xPMGi17u/S87c0JZRjpPA8UWqQcHYgrMBpo92ZZxoCRmqO+NW+Ln36guqE1127eK62/HyP+j3PP6uet/4VP5CY9d6xZJ4fGooLC1I2TaxdPFc/frvDD0D9nucHIEla8cQO/e6DmGInTyWFrBVP7NBv2o+q79RAQtn6+gcU0WDtwqEjJ7TiiR0qGApX5gBqgtXKJ9v8g7c58Jtypjr42Wfa9vvW2FDpBxaz7Yb9PknDYeLQkROac+/L/nabsrx/5ERC7cqj2/bq/aMnJUnRogLFTw34n2+YjiMnVBiR+odCRezkKUlSX/9wCJp/frn/euY5zXOq1NgwXDPWPKdK7x896dcESfKDijHgSSufbPNrHg4NhabBpqaeof93+69VEIn4tThSckgw373Cgoj/nbWlCjDB9Zjb5nXaDvQkfAbNc6oTnh9WQ2fKbcJ5toOKS7VQcF/E80JOYyaQWCym8vJy9fb2qqysLN/FQZalatJI19RhgojZ0ZqDoanyDzYLpOq7ENxBm8cMs+7iwgKtufYC/3kDnqe2Az1+AAhbl13G4PrsdRjNc6q0cFal3w8keLA2tQe2uopS/+BpMwcw89qNDZUqiETU3vOxDh054d9u3d+tspIiTSkpUiQyeCA2YaF5TpXe6z7u13YURKRzCgcfs18/osRaD7+Wo6jAD1q24sKI+vo9TSkp0sXnl+u3h47qWLxfjQ2VeutAj3K5s2psqPRDniQVFxVInpcQcoLM9ph/o0UFqp5c7Icrm/k8zDaeP7VENy2cIWnw8wyuq66iVG9+81r/+cHvvDnY9w94eutgt/+ZrVsyT637u/ygW1ZSpD/GT/lBq3lOlfoHPD+MB4OoHZ4jGqxBKoxEtH/z/0rapkxDRVgwsX9r9kmDXdP2N1fUJwVmSWrd36VFs6sSnkO4mXhGc/wmrJwGzgyS5aJaO7iTNjUCqc70Uu1wTWgI/tvYUJlQRR5szjFntyYgmIONJH/dUmJgMIHDHPzMgaq+olRfXFCn//ObQ+o4ciIpZISFjlSPmeCQSllJkU5+0p/2YFtXUar/71jcX49dwxGJSK7sHUba1mwa63ab5xUWRLRwVkVC2Ey37nVL5ulfhn4bQcHvfGNDpRbNrkoIrfZnZmqtRmKec/7UEj+I1k0t1YyqSQkB2lZXUaobFtT5v0tTpuCJQFiASHVyEbaO4DYFf4/B92asHcNziWNDZkZz/KYZ6DTkqn+Eq1L9AO0mlWx1zkvVhv5Iyx4/eATLYe8E7Crs/qGjQ7BK29xnV5EHd3iNDZVJtRgl5xSqr38wNNiHFxMkokUFSTv8Q0dOqLAgoo4jJ/Qv1lDY2MlT/tmxvY4wwcfipwbSHljTrUuS31xke+tgjx+KXAkqksYtqEhjD2jmef0DXmhQSbXusCataFGBbrtmsLbOfrwgEtGLuw7p0NHBcGH64ZgwnElQkaT2no8lKaEG6NDREzp0NPXzDx054Yfs//12hzqGwrfprGz/roL7wFQdwe2ThuBv0Kwj7P0J6y820kgwVzs3IzOEldOQzeGfE0GqH6DdnGGaVOzOeebgF2wG+T+/OaTaqaV64atN/rrs4GN3DmyeU51Qi9He87E/34UdhprnVOkzD76mQ0N9EMyoDfM8U9NiymiYtvxf7etK+BzrKkr9ZYoLCxQ9pyChqj5MqgNr8My5f2CwLGFn1Jk6nUAR9tT+AW/EkIPci58aSJrzxfRjKS4c/AbbwSKsuS+dQ0O1esfimYdSu9bG1Ar+8pvXJtSOSIP9qUwn6GCfL2nwd/cvP9+b0DxrgkphJOKHjv4BL6EvlGF+16a/U1jNS74Dwtl2bBgPNANlgfkSmj4KY/0yToSqw1RNJXaTyuDokO6E2oWykiLNP7/cb5u+8sHX/DOzG6+o9w/Y5szNtGMH+3XYwcMwZ5V21XFdRanqKkpH7OcQ1gxzYW2Z2g70hAYSuxYEOFukaoYz+7xg/6hgXzDDTKTnSUnNs6YW1ISiYPOreb09m5b5QSXYlyZdU24+AkK2jg1nKvqs5IE5MzA/prEYa2fS8Qg4YX1I7A6kpnYjrMOoze7caZ+prbOabsz9dvgw7OVSMZ1Cg50c09WGABi7sL4y6TqUm6AS7FQc7Fwc7Kdiv5bZv5gTpGBHePvx528Zrr0d7xO/bBwbzlSjOX4z3X4WPLptr/9lNMM/x2Lt4rl+04c9pXe6M4Pxmjbdfp21i+f622qG8JqgUmjNr2E3uRiDQ2gHA8n5VhPLY6/t86uSUwUVSUPNQumDSuzkKXXG4qH3A8i+YFApKynSwy17tPLJNj26ba/+/fX9/u+2eU6VmoZOWsxvMnbylCLW7UNHTujKB1/zg0q0aPhQZWpkv/FXc1VWUuQ3E5l94NrFcxNOlOxh2qPdL57uJTTMscHMZRS2rlxdiuNMQ5+V05Sq2lHSmKr7Rjur6Hi1jdqv8+O3O/yg4mnwTOe51Y0J83YYqartOoY6AzY2VOo37x31+4w0NgyOogkLJHYTUFitjTS8sws21RBUgPFjOo6bkxNbsHbFMP83Jyp2AAo2QdmT/tmTALbu71JBYN/QdqA7o1GEYU6no2zYaEJ7/id7RJe9Hlea/F1DWDkNYaEgLDyM1trFcxMmFRtpHWEBx57i3LDnKQhOehYcZhj2Y7EnipKGg0rH0FlQupEIqdq8Dx05kTCBmD2RVZC9Y7PbtgG4JZOO46keNRPwZaKuolS/HGouat3flTAKy24ynn3PS/7+ajRN5mM9GUy1jAksYUOzg89DIsJKwGj6gPQPeKFfWHN7rB0xw5qVMgksJqgURiL+TKV2ecxU4NLgj9ZuCzaTUpkOrsGk37q/S9LgTJh2rcaNV9T7wxiNsE6wqUbImPLYzTTpmnmMwkhy1TMAN5xOR8jYyVOqryjVB0dPhtae2mZUTpI0vI8LDhl/bnWjH1SkwX1GcH8aHHYdPAaEjWJKFVTsifrCljGjm94KGdru6jBsVxBWAkZT7ZfJ1WFHa6zNSibgmCDRdqA74QJsUuLwRtMvxH48NjQbqX32Yb/+lJIi/4zABKmHW/Yk9EsJNs9kMsHWaPqTmJlP08xzBmCCMyciYSc+ttb93VrxxA4tml3l9wcxzzHhwn6+fZK2dvFcv0bYnrcp7BiwdvFcP6gURsJPZu3nphskYZq97Zmvg03+Lg3DdgVhJSCf4+PH2qyUaoZXSUmBxa5+TDUk2Jx92K8rScdOnkpo8zXzKXjWOoY7tVXp/77fm1EIGWneElvYFO0AzjzFRQVaMGNqygn2DPs6TJLUZF2uwj5xihYV+J1z7QkmB2ccHmw2T3UByJVPtvkDBvq9xFoPuyYmuL+W5DdPmYBh15pLw8O/7SZ/5mlJ5kRYefzxx/XQQw/pww8/1EUXXaQtW7boyiuvzFt5RtvJNVvG0qwU9gVeOKtSHT0fq3V/d0J1Y3Fhgd+BNbgDMPMe1AWuWGvY84uYDmu/+yCWtA4z30Lr/m7VTS3NeNIpOsACsJUUFYwYVMK07u9OGD1kxE8N+IFFsiaKHPD04m8OqbAgEnoBSLsZydSm2yEiWBMTFljsTrZhtSdhTf75Og65Ku9Dl1944QXdfvvtWr9+vXbv3q0rr7xSy5YtU3t7e17LZQ/PzaSTazbcMcKon7Bmp7CAY6Z1t2s67Gabgkj4sD1P4f1A6itKE4JS6/5uzbrnpYSAMTSppnpPfOLf937vCaembAcwcZzOCUzwgpph9xv1QydYJjjY1w6za5zNfrZ/wPOb0E24MDUxK59sSxqGXFw4eJhNVyuy5toLkqatkPJzHHJV3sPKww8/rL/7u7/TV77yFf3Zn/2ZtmzZovr6ev37v/97XsuVrblTcu0OqzOWsXbxXDXPqUpoq71iVoX//1QdWOutH7Utk46spg/JMWsHQ1ABkC1lJaNvCCiMRPTmN69NOcLQjCg0+z4zj5MJLGG7MDMke3CZLj3Sssff57bu79Zjr+1LqjFp3d+VctSP6VCbap6tiXAcGg95bQbq6+vTrl27dM899yTcv3TpUrW2toY+Jx6PKx4fnvArFouFLnc6sj13Sq4FqyFNXxJbcMbYMOkeY5gwgHwaS01Lv+cNT80fuFRGRMPXOAru2+zmc7vDriS/ZmV4krtq/6KMdm128iVJhieny6TJf6Idh3Itr2Glq6tL/f39mjZtWsL906ZNU2dnZ+hzNm/erPvvvz9nZQrrA2JX+0nJvbODc5YE15frYWZ222bbge6EoGJ+aOaHGZRqptggggqAfEs1X1OQfa0iEx6Cff68ofXZVz43oxnNc+yJL01fvuA+0/QnSTdIwL56e6omfSPYmTY4jDnY6fZsGcac92YgSYoE+lB4npd0n3Hvvfeqt7fX/+vo6MhqWVL1ATHVfvYX3p66eTymvE/HVCEGp7z3NFgVav9MGxsGr2jc2FCZtDwAuCqToFJcGNEhq2knqD7Qh8UElfqK0qR5XcyFFs3wZmlwgIFRGIn4o3rMqEbTtGP6sJhaFRM8MmEfh4KXOrEDTC6PL6d7qYFsy2vNSnV1tQoLC5NqUQ4fPpxU22JEo1FFo9GclSksodqJtrFh8As7UhVdPoaZBSeGMz+8AS+x6tO+qFdweB8AuCaT+ZqMvv7Us1sHL4IoyQ8qqWqPGxuq1NhQlXBxxOLCAl0xqyLpyvKmZsWe7duewyVT9nEoX8OYT+dSA7mQ17BSXFysyy+/XC0tLfrCF77g39/S0qLrr78+jyVLNtIwsmwOMxvrlZSDE8M1z6nS2384kjCdvSfpygdf0y+/ea3+99vZrZUCgFwYzfmU3eTdbM25Yu43wcR0Wi0rKdL5IWGlMBLRwtkV/oE5OCGmqZU286/ETp7yuwuYY0DznCp/DpfTkY9hzK7N9RLxvPyeVr/wwgu6+eabtXXrVjU1NemJJ57Qk08+qXfffVczZ84c8fmjucR0Nox0ue/RXg48LJgMd8ga/KLfsWRewiXWw2ZHtCce6h/wEi7HbjNtvsF/AcAFwY6wY9U8p0rPrW6UNLxfNn1QgjXipo+LCR/mX1N7EJzYzZ5DxYQXE2bs2pdMjgGjMdrjSzaY98hsZzaDymiO33nvs3LTTTdpy5Yt2rhxoy699FK98cYbevnllzMKKuNtpGFkYxlmFtbfxR4Gt/MPg51lTfgIVimaL5L5IZnOW6a5Shr80ZofmQkm9r/Btt3ioUlTUg33C0O3FwCnK5OgEjbhWxi7yd4OKvY+1OxrTR8XUyNt/rUHLQTnWTHMHClmxKUJKtkeapyvYcyuzPXixAy2t956q2699dZ8FyOt0fZRyXSYWaqqNnusv0nT5rbpaGVeo7GhUs1zEq+gbO4viAxOJR02q6I0WNV5fqAdt6/fU1lJkZ5b3ZgwdX86dHsBMFZmX5XJviZ+amDEawZJw6MjzToXWa9h70PtaR2ShxtXJZ0kmgEX5gTQNMvbfVSeW92Y1aHG+RzGPJYL6+aCE2HFdSNds8d8SUd7TR8jXXukXe1n/wC2/HyPBjyFVsm17u9SY0NlQkdaKbEXu9E/1J5rfpSmk1js5Ck9um2v3zk3eIHCMCPtQGhyAhBm58Ejo+roH7w4Ye+JTxKGDZvwYV8jLezirGbUTf+AF3ql5f4BT40NiaNAw0aM2ieYpo/KaI4B6Yz1mnHZ4NJcL4SVNNJd7tt0dF23ZJ5+ta8rNDSku6ZPkD2Sx1S1pUq0Zrkwj27bm9C2apgaEnvGRrtDmV2TY/41X8p1S+bpB788kLAzCJufxcznkmpr6SMDnFnG+ns2+wnzfPsCqLveO5LxXCp/c0W9v59KtW8LNp3b++WR5jsJk+llT4KvNVa5XHc6+QxJYfLewfZ05bKDbaqez7noER3sxGTCQKpqSfuqncFlguUKBhXz+GcefM2fOMkIu3KzqVWxd0ymzTaTaltmvwXOXCP9vlMFmrATGztwnF9R6k9waZ8cTSkp0sXnl2vhrMqE4bV2s8xjr+3zR+mMNAkbwo11VOpojOb4Tc1KGuM1dCu4ThMUgh3BUrWJjjSczXQIMz9u87h9ViLJ/2FLg1dubmyoSpi35WtXzVFhQSShHbiuolR1FaV6/8iJlDussLMdABNPtKhA1ZOLNbPq3MErq1eUqnZqqeorJ6U8cbGDih1c7KBiTr7MvuLGoX2TPfrGXJDw0JETamwY3Dc+Yi1jDp6PbtubMKJnPCblPBONpdYplwgrI8j1+Paw8GPaPMM6gqVqE03XU/uFrzYl3ResqTE/7JVPtum51Y3+j95uhrKDjtkxzUizk7KZoFJcGFFff+aVeekmgwo7M8uk4x2AYammibenrDfipwZ008IZSVO9r3hih6Tk5mH7JCVdM81zqxv9a5oN9jVJblpPCCNDTR/BA6pLfSyQXYSVDIT1J0lnNNVnYe2R5jH7R5mq3dIw0z5n0lM7GFTMaCJTY7LyyTZ/xka7RibYKU1KHl0kJe7k7J1XRNKnykqSdoDppAoqqa5ptKihMm0tT5hszesA5JMd7E1oLysp0vH4KaU7PwgLKoWRwSnrze/MrK+uojTp4G/6ydlN19LgvsH+HZoRhx1HTvj7iLqKUv+k7PlbmlKGi1T/t7nWxwLZRVjJwGiHbo1mmuJMq9pSzVg7lrMI+6qh9o5leAKk7oSOtqbK1V7O3DajkszOzOzk7MmUpOHHzQ5w4axKPf2rg0k7ymD7drCmxK6ZGZzbYHCyJtPZrnV/l38lVXOtDvs1gveboBYWuoCJxA72Zj6RYKC3f1/B35Z9u98bnII+Vb+5sNExZuSM2TfYTcgLZ1foraFLexQXFuiGBXX+CZ19UnY6nUbz1REV44OwMoKxBILx6Osy1rMIU+tj+qTYy5i5WRobKnVoaNhf2PrNj35waPNwe7M9c6PZ5rBZH81kTXaIMGda8VPDnYuDwWWw9mh4J9rYUJVwNmY305mhiPbrStKFtWUqiAzWpHz6guqEx9ONbLA7NCMzhRGlPaPH6THfV3taARM4Ghsq1dHzsb9sXUVpUpNt8KMxAcfUqpqL94XtX+xJJ9Nd+sP8bgqGLqYa1qScKlyMlmt9LJBdeZ/B1mWpAoG5oma6GQTt5eatfyXrnXLTnUWku7qnqfVJ1UzVdqBHv3nvqF87ErZ+025stmnPpmX+rI+mOcpcbdSe9dG+GqkJRuY+03lOGgwzzXOqFD81eN0OSQnXOzI7UftsLDjD4h2BgFRcWOBvX2NDld+Pxx72mG645JprL0iYtTLVFV3PNo0NlSnfi9qp7r9HhTnse2mv23yPR1JWUhT6foYV0wT7NddeoMaGwX5sJhA8f0uTBoZqR9YtmacbFtT5cyatWzJP9RWlSessKynyw0xjQ2VCE40x0v5FUui+wdTU7tm0LKP9JxBEzUoap1utONq+LqMx1rOIkWp9MilvWG1T2Dwt9lmZXV7TNGRPWjfYNFXt/9/uZBzsN2NfM8mUJ2waantIo72ddt+bsJkzgxPgmWm37UDVYbXnBwU7EK5bMk8/frtDh46cSNlhOKwWwtyXyYR8mQqWzRxEg81xmdSK1FeUqmCoD4KteU6V2ns+Trg/H/PrpOo4atQPjWT59cGelM0hBRFpwBscLvtxvN//HIqLCtQXsj3+NWaG+kHZkyxK6TuYm2UvrC3T0ROf6NjQc4Kj/8x7WTe11L/2janFlOT/Bn51z2J/3XYtpxlZaHd2Nf+a77r9mwnu59LtX0baN9hN6PQjwWgQVtI43WpFV6YpDko1wklSRuW1Q1zYzsm0XZuzp+A6gjPrSuG9+sMu2mjKbV/3I6yZzgSLVM1krfu7/HKYkQxS8sgESRrwhif/M/1izP1Sctt/74lPJA0eoC+bMdWvRTJXfQ0LLNOnmgupDYeEfk9J/XvMQTBdp+CwcBOct8KYf365/1kZdRWlevOb1+pPvvVKaP8hc1DtOHJCR4e21e6jZOb0sV/bBJeRAkQq0aICeVJoQDCPB8NQ7OSptEPmzf32O2VfyG5RQ6W/Lea9M7+N8yYX+0HNvuaMCRVhFxM1zZhh74MdFhbOGu5HVVdRGjr6z75IX7AvXOv+rqQgEPzNBqdAMN+pYJ+U0e6vgid4wdfNRt8UnJ2YFC5HUh1E83Fp7VTsqfzXXHvBmMqbq4mDMlmv2bmmmrTP3qGnK5cJJqnWE3bpguCIKnNGbZ+p/vKb1/rLmo7A9sEz2I/AlNfMs2PWaV7DPHc0o5fs17BrguxJ/ezRYO3dH+uGy+sSmgvtDtl2WcwU56net7KSIn3lygatXTzXf4+DfX/MNpWVFGn++eVJNVX23BqGvaw98qy4MKLoOYW6qLYsKZSVlRTp//n0bH+77OcFR8Olu+JusN9a8CrnwffCTLxoB0g7JJh1m/vtq6xn8rsay8SV4zHZF5AJJoXLs4kwhC5Y6zPW8uaqU1sm630kxc54pLO24PKLZlclXAhypPUEg0rwIGYOwGbOGkkJQzulxBofcyA31eR25+dHt+1NuJyDPbOwGQ116MiJ0OHg9RWl+uKCOt2xZF7CRIPmWif2wa55TnXSmfpIsygHL6AZ9r6Z/z9/S5Mfjgsi0l/MHm5+szt1BmtC6ipKE0ZrhdUomOVuGNrW4GMRSf+94bMJZRzu7F3pd9IONmGabQ0Lsraw1zTvl+n71T/g6cXfHPInNDPLDNd0dCU0bWb6uxpLUzUdUTEREVZywPUhdMEwZTd52FwpbyrZ2umOdj2phmra75c50zYHaLP8gOclHeCHD5ZdSWe15oBmLJxVmbQOuwngxzs7VFc5fIA3I6JMU4J9Be5gmUezfSbwjDQizgiG4+Y51f567G2wR3HZtQ72LKX2a4T1f7KZJhq7OTL4vJGaMoPXlrHnGQqWx9wXdsJih0N7+eBnPBoED5wtaAY6y4zn9Y7OdnYz255Ny3LyGhOhSn+kJtFsb8Nom2Bz8R5OhM8FyLfRHL8JK2cZdqLjI3hhyrM1BI53OCaMAxMHfVaQEtXGucf1SYaNd5Oo602wAMaGmhUgizizB4DMULMC5Aln9gCQfdSsAACAcTea4zfXBgIAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOC2vYWXWrFmKRCIJf/fcc08+iwQAABxTlO8CbNy4UatXr/ZvT548OY+lAQAArsl7WJkyZYpqamryXQwAAOCovPdZefDBB1VVVaVLL71UmzZtUl9fX9rl4/G4YrFYwh8AADhz5bVm5Rvf+IYWLFigiooK/frXv9a9996rgwcP6gc/+EHK52zevFn333//OJYSAADkU8TzPC+bK9ywYcOIYWLnzp264oorku5/8cUX9aUvfUldXV2qqqoKfW48Hlc8Hvdvx2Ix1dfXq7e3V2VlZadXeAAAMC5isZjKy8szOn5nvWZlzZo1WrFiRdplZs2aFXp/Y2OjJGnfvn0pw0o0GlU0Gj2tMgIAgIkj62Glurpa1dXVY3ru7t27JUnTp0/PZpEAAMAElrc+Kzt27FBbW5uuueYalZeXa+fOnbrjjjv0+c9/XjNmzMhXsQAAgGPyFlai0aheeOEF3X///YrH45o5c6ZWr16tu+++O19FAgAADspbWFmwYIHa2try9fIAAGCCyPs8KwAAAOkQVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNNyGlY2bdqk5uZmTZo0SVOnTg1dpr29XcuXL9e5556r6upqrV27Vn19fbksFgAAmECKcrnyvr4+3XjjjWpqatJTTz2V9Hh/f7+uu+46nXfeeXrzzTfV3d2tVatWyfM8/eu//msuiwYAACaInIaV+++/X5L0zDPPhD7+6quv6ne/+506OjpUW1srSfr+97+vL3/5y9q0aZPKyspyWTwAADAB5LXPyo4dOzR//nw/qEjSZz/7WcXjce3atSv0OfF4XLFYLOEPAACcufIaVjo7OzVt2rSE+yoqKlRcXKzOzs7Q52zevFnl5eX+X319/XgUFQAA5Mmow8qGDRsUiUTS/r399tsZry8SiSTd53le6P2SdO+996q3t9f/6+joGO0mAACACWTUfVbWrFmjFStWpF1m1qxZGa2rpqZGb731VsJ9R44c0SeffJJU42JEo1FFo9GM1g8AACa+UYeV6upqVVdXZ+XFm5qatGnTJn344YeaPn26pMFOt9FoVJdffnlWXgMAAExsOR0N1N7erp6eHrW3t6u/v1/vvPOOJOmCCy7Q5MmTtXTpUl144YW6+eab9dBDD6mnp0d33XWXVq9ezUggAAAgKcdh5dvf/raeffZZ//Zll10mSdq+fbuuvvpqFRYW6qWXXtKtt96qT3/60yotLdXKlSv1z//8z7ksFgAAmEAinud5+S7E6YjFYiovL1dvby+1MQAATBCjOX5zbSAAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHAaYQUAADiNsAIAAJxGWAEAAE4jrAAAAKcRVgAAgNMIKwAAwGmEFQAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACcRlgBAABOI6wAAACnEVYAAIDTCCsAAMBphBUAAOA0wgoAAHBaTsPKpk2b1NzcrEmTJmnq1Kmhy0QikaS/rVu35rJYAABgAinK5cr7+vp04403qqmpSU899VTK5Z5++ml97nOf82+Xl5fnslgAAGACyWlYuf/++yVJzzzzTNrlpk6dqpqamlwWBQAATFBO9FlZs2aNqqurtXDhQm3dulUDAwMpl43H44rFYgl/AADgzJXTmpVMfOc739HixYtVWlqqbdu26c4771RXV5e+9a1vhS6/efNmv8YGAACc+SKe53mjecKGDRtGDAs7d+7UFVdc4d9+5plndPvtt+vo0aMjrv/73/++Nm7cqN7e3tDH4/G44vG4f7u3t1czZsxQR0eHysrKMtsIAACQV7FYTPX19Tp69OiIfVVHXbOyZs0arVixIu0ys2bNGu1qfY2NjYrFYvroo480bdq0pMej0aii0ah/2zQD1dfXj/k1AQBAfhw7diz7YaW6ulrV1dVjLtRIdu/erZKSkpRDnYNqa2vV0dGhKVOmKBKJ5Kxc48mkzbOttuhs3G62+ezYZuns3G62+ezYZmls2+15no4dO6ba2toRl81pn5X29nb19PSovb1d/f39eueddyRJF1xwgSZPnqyf/exn6uzsVFNTk0pLS7V9+3atX79et9xyS0LtSToFBQWqq6vL4VbkT1lZ2Vn1ZTfOxu1mm88eZ+N2s81nj9Fud6ZTleQ0rHz729/Ws88+69++7LLLJEnbt2/X1VdfrXPOOUePP/641q1bp4GBATU0NGjjxo267bbbclksAAAwgeQ0rDzzzDNp51j53Oc+lzAZHAAAQJAT86wgUTQa1X333ZdxU9iZ4mzcbrb57HE2bjfbfPbI9XaPeugyAADAeKJmBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWHLNp0yY1Nzdr0qRJKWfxjUQiSX9bt24d34JmUSbb3N7eruXLl+vcc89VdXW11q5dq76+vvEtaI7NmjUr6XO955578l2srHv88cc1e/ZslZSU6PLLL9cvf/nLfBcpZzZs2JD0mdbU1OS7WFn3xhtvaPny5aqtrVUkEtFPf/rThMc9z9OGDRtUW1ur0tJSXX311Xr33XfzU9gsGWmbv/zlLyd99o2NjfkpbJZs3rxZCxcu1JQpU/SpT31Kf/3Xf63/+Z//SVgmV581YcUxfX19uvHGG/X3f//3aZd7+umn9eGHH/p/q1atGqcSZt9I29zf36/rrrtOx48f15tvvqnnn39eL774ou68885xLmnubdy4MeFzTXX18YnqhRde0O23367169dr9+7duvLKK7Vs2TK1t7fnu2g5c9FFFyV8pr/97W/zXaSsO378uC655BI99thjoY9/73vf08MPP6zHHntMO3fuVE1NjZYsWaJjx46Nc0mzZ6RtlgbnErM/+5dffnkcS5h9r7/+um677Ta1tbWppaVFp06d0tKlS3X8+HF/mZx91h6c9PTTT3vl5eWhj0nyfvKTn4xrecZDqm1++eWXvYKCAu/999/37/vRj37kRaNRr7e3dxxLmFszZ870HnnkkXwXI6f+4i/+wvva176WcN+f/umfevfcc0+eSpRb9913n3fJJZfkuxjjKrh/GhgY8Gpqarx/+qd/8u87efKkV15e7m3dujUPJcy+sH3yqlWrvOuvvz4v5Rkvhw8f9iR5r7/+uud5uf2sqVmZoNasWaPq6motXLhQW7du1cDAQL6LlDM7duzQ/PnzEy529dnPflbxeFy7du3KY8my78EHH1RVVZUuvfRSbdq06Yxq6urr69OuXbu0dOnShPuXLl2q1tbWPJUq9/bu3ava2lrNnj1bK1as0IEDB/JdpHF18OBBdXZ2Jnzu0WhUV1111Rn9uUvSL37xC33qU5/SvHnztHr1ah0+fDjfRcqq3t5eSVJlZaWk3H7WOZ1uH7nxne98R4sXL1Zpaam2bdumO++8U11dXWdck4HR2dmpadOmJdxXUVGh4uJidXZ25qlU2feNb3xDCxYsUEVFhX7961/r3nvv1cGDB/WDH/wg30XLiq6uLvX39yd9ltOmTTujPkfbokWL9MMf/lDz5s3TRx99pO9+97tqbm7Wu+++q6qqqnwXb1yYzzbsc3/vvffyUaRxsWzZMt14442aOXOmDh48qH/8x3/Utddeq127dp0Rs9t6nqd169bpM5/5jObPny8pt581NSvjIKyTXfDv7bffznh93/rWt9TU1KRLL71Ud955pzZu3KiHHnooh1swetne5kgkknSf53mh97tkNO/DHXfcoauuukp//ud/rq985SvaunWrnnrqKXV3d+d5K7Ir+JlNhM9xrJYtW6YbbrhBF198sf7qr/5KL730kiQlXOD1bHE2fe6SdNNNN+m6667T/PnztXz5cr3yyivas2eP/x2Y6NasWaP//u//1o9+9KOkx3LxWVOzMg7WrFmjFStWpF1m1qxZY15/Y2OjYrGYPvroo6REmy/Z3Oaamhq99dZbCfcdOXJEn3zyiTPbm8rpvA9m5MC+ffvOiLPw6upqFRYWJtWiHD582PnPMVvOPfdcXXzxxdq7d2++izJuzOinzs5OTZ8+3b//bPrcJWn69OmaOXPmGfHZf/3rX9d//ud/6o033lBdXZ1/fy4/a8LKOKiurlZ1dXXO1r97926VlJSkHPabD9nc5qamJm3atEkffvih/wN49dVXFY1Gdfnll2flNXLldN6H3bt3S1LCj34iKy4u1uWXX66WlhZ94Qtf8O9vaWnR9ddfn8eSjZ94PK7f//73uvLKK/NdlHEze/Zs1dTUqKWlRZdddpmkwf5Lr7/+uh588ME8l278dHd3q6OjY0L/nj3P09e//nX95Cc/0S9+8QvNnj074fFcftaEFce0t7erp6dH7e3t6u/v1zvvvCNJuuCCCzR58mT97Gc/U2dnp5qamlRaWqrt27dr/fr1uuWWWyZsO+hI27x06VJdeOGFuvnmm/XQQw+pp6dHd911l1avXq2ysrL8Fj5LduzYoba2Nl1zzTUqLy/Xzp07dccdd+jzn/+8ZsyYke/iZc26det0880364orrlBTU5OeeOIJtbe362tf+1q+i5YTd911l5YvX64ZM2bo8OHD+u53v6tYLDahpxoI88c//lH79u3zbx88eFDvvPOOKisrNWPGDN1+++164IEHNHfuXM2dO1cPPPCAJk2apJUrV+ax1Kcn3TZXVlZqw4YNuuGGGzR9+nT94Q9/0D/8wz+ouro6IahPNLfddpuee+45/cd//IemTJni15KWl5ertLRUkUgkd5/1aY0lQtatWrXKk5T0t337ds/zPO+VV17xLr30Um/y5MnepEmTvPnz53tbtmzxPvnkk/wW/DSMtM2e53nvvfeed91113mlpaVeZWWlt2bNGu/kyZP5K3SW7dq1y1u0aJFXXl7ulZSUeH/yJ3/i3Xfffd7x48fzXbSs+7d/+zdv5syZXnFxsbdgwQJ/2OOZ6KabbvKmT5/unXPOOV5tba33xS9+0Xv33XfzXays2759e+hveNWqVZ7nDQ5pve+++7yamhovGo16f/mXf+n99re/zW+hT1O6bf7444+9pUuXeuedd553zjnneDNmzPBWrVrltbe357vYpyVseyV5Tz/9tL9Mrj7ryFABAAAAnMRoIAAA4DTCCgAAcBphBQAAOI2wAgAAnEZYAQAATiOsAAAApxFWAACA0wgrAADAaYQVAADgNMIKAABwGmEFAAA4jbACAACc9v8DakZCaQ2DRD8AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": 8, + "id": "5ffa2256", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], "source": [ "# extract the relevant columns from the centered design matrix of dim n x 2\n", "x = X_centered[:,0]\n", @@ -1051,8 +1183,10 @@ }, { "cell_type": "markdown", - "id": "33abed26", - "metadata": {}, + "id": "b62b1ee4", + "metadata": { + "editable": true + }, "source": [ "## Exploring\n", "\n", @@ -1062,8 +1196,10 @@ }, { "cell_type": "markdown", - "id": "e99342eb", - "metadata": {}, + "id": "ad2ecea2", + "metadata": { + "editable": true + }, "source": [ "## Diagonalize the sample covariance matrix to obtain the principal components\n", "\n", @@ -1084,8 +1220,10 @@ }, { "cell_type": "markdown", - "id": "1f713cbc", - "metadata": {}, + "id": "f69809f8", + "metadata": { + "editable": true + }, "source": [ "$$\n", "x_i \\approx \\tilde{x}_i = \\mu_n + \\langle x_i, v_0 \\rangle v_0\n", @@ -1094,16 +1232,20 @@ }, { "cell_type": "markdown", - "id": "0835e685", - "metadata": {}, + "id": "6b75f94a", + "metadata": { + "editable": true + }, "source": [ "where $v_0$ is the first principal component." ] }, { "cell_type": "markdown", - "id": "d80188dc", - "metadata": {}, + "id": "c33341ec", + "metadata": { + "editable": true + }, "source": [ "## Collecting all Steps\n", "\n", @@ -1118,8 +1260,11 @@ { "cell_type": "code", "execution_count": 9, - "id": "fdfa7196", - "metadata": {}, + "id": "f3f4e414", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "# diagonalize and obtain eigenvalues, not necessarily sorted\n", @@ -1147,16 +1292,20 @@ }, { "cell_type": "markdown", - "id": "759948ad", - "metadata": {}, + "id": "7d7d6b3a", + "metadata": { + "editable": true + }, "source": [ "This code does not contain all the above elements, but it shows how we can use **Scikit-Learn** to extract the eigenvector which corresponds to the largest eigenvalue. Try to address the questions we pose before the above code. Try also to change the values of the covariance matrix by making one of the diagonal elements much larger than the other. What do you observe then?" ] }, { "cell_type": "markdown", - "id": "d2dd62e2", - "metadata": {}, + "id": "3e1e4037", + "metadata": { + "editable": true + }, "source": [ "## Classical PCA Theorem\n", "\n", @@ -1177,8 +1326,10 @@ }, { "cell_type": "markdown", - "id": "f8df57ba", - "metadata": {}, + "id": "4fdc8d7e", + "metadata": { + "editable": true + }, "source": [ "## The PCA Theorem\n", "\n", @@ -1199,8 +1350,10 @@ }, { "cell_type": "markdown", - "id": "5d2e8ad2", - "metadata": {}, + "id": "de3ffc83", + "metadata": { + "editable": true + }, "source": [ "$$\n", "J(\\boldsymbol{w}_0)= \\boldsymbol{w}_0^T\\boldsymbol{C}[\\boldsymbol{x}]\\boldsymbol{w}_0+\\lambda_0(1-\\boldsymbol{w}_0^T\\boldsymbol{w}_0).\n", @@ -1209,16 +1362,20 @@ }, { "cell_type": "markdown", - "id": "04ddd4a7", - "metadata": {}, + "id": "1be2f8d2", + "metadata": { + "editable": true + }, "source": [ "Taking the derivative with respect to $\\boldsymbol{w}_0$ we obtain" ] }, { "cell_type": "markdown", - "id": "302a5a03", - "metadata": {}, + "id": "eda2ce06", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\frac{\\partial J(\\boldsymbol{w}_0)}{\\partial \\boldsymbol{w}_0}= 2\\boldsymbol{C}[\\boldsymbol{x}]\\boldsymbol{w}_0-2\\lambda_0\\boldsymbol{w}_0=0,\n", @@ -1227,16 +1384,20 @@ }, { "cell_type": "markdown", - "id": "4844b665", - "metadata": {}, + "id": "fe3d6837", + "metadata": { + "editable": true + }, "source": [ "meaning that" ] }, { "cell_type": "markdown", - "id": "c4aea501", - "metadata": {}, + "id": "29136014", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{C}[\\boldsymbol{x}]\\boldsymbol{w}_0=\\lambda_0\\boldsymbol{w}_0.\n", @@ -1245,16 +1406,20 @@ }, { "cell_type": "markdown", - "id": "120dd213", - "metadata": {}, + "id": "2780a54e", + "metadata": { + "editable": true + }, "source": [ "**The direction that maximizes the variance (or minimizes the construction error) is an eigenvector of the covariance matrix**! If we left multiply with $\\boldsymbol{w}_0^T$ we have the variance of the projected data is" ] }, { "cell_type": "markdown", - "id": "87cc8b29", - "metadata": {}, + "id": "913d52e7", + "metadata": { + "editable": true + }, "source": [ "$$\n", "\\boldsymbol{w}_0^T\\boldsymbol{C}[\\boldsymbol{x}]\\boldsymbol{w}_0=\\lambda_0.\n", @@ -1263,8 +1428,10 @@ }, { "cell_type": "markdown", - "id": "127aa4c0", - "metadata": {}, + "id": "7644953d", + "metadata": { + "editable": true + }, "source": [ "If we want to maximize the variance (minimize the construction error)\n", "we simply pick the eigenvector of the covariance matrix with the\n", @@ -1287,8 +1454,10 @@ }, { "cell_type": "markdown", - "id": "f5c67d1b", - "metadata": {}, + "id": "4f21e3bc", + "metadata": { + "editable": true + }, "source": [ "## Geometric Interpretation and link with Singular Value Decomposition\n", "\n", @@ -1304,8 +1473,11 @@ { "cell_type": "code", "execution_count": 10, - "id": "0f2885aa", - "metadata": {}, + "id": "134950d3", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import numpy as np\n", @@ -1336,8 +1508,10 @@ }, { "cell_type": "markdown", - "id": "8a710c59", - "metadata": {}, + "id": "7325d161", + "metadata": { + "editable": true + }, "source": [ "PCA assumes that the dataset is centered around the origin. Scikit-Learn’s PCA classes take care of centering\n", "the data for you. However, if you implement PCA yourself (as in the preceding example), or if you use other libraries, don’t\n", @@ -1351,8 +1525,11 @@ { "cell_type": "code", "execution_count": 11, - "id": "9795e85c", - "metadata": {}, + "id": "ed151526", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "W2 = V.T[:, :2]\n", @@ -1361,8 +1538,10 @@ }, { "cell_type": "markdown", - "id": "8f2e67d3", - "metadata": {}, + "id": "6041ce6a", + "metadata": { + "editable": true + }, "source": [ "## PCA and scikit-learn\n", "\n", @@ -1374,8 +1553,11 @@ { "cell_type": "code", "execution_count": 12, - "id": "c7b6979d", - "metadata": {}, + "id": "4e9f65e9", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "#thereafter we do a PCA with Scikit-learn\n", @@ -1387,8 +1569,10 @@ }, { "cell_type": "markdown", - "id": "583397ec", - "metadata": {}, + "id": "7d7383b4", + "metadata": { + "editable": true + }, "source": [ "After fitting the PCA transformer to the dataset, you can access the principal components using the\n", "components variable (note that it contains the PCs as horizontal vectors, so, for example, the first\n", @@ -1398,8 +1582,11 @@ { "cell_type": "code", "execution_count": 13, - "id": "210037e3", - "metadata": {}, + "id": "c61889e6", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "pca.components_.T[:, 0]" @@ -1407,8 +1594,10 @@ }, { "cell_type": "markdown", - "id": "577bb61b", - "metadata": {}, + "id": "266f3c13", + "metadata": { + "editable": true + }, "source": [ "Another very useful piece of information is the explained variance ratio of each principal component,\n", "available via the $explained\\_variance\\_ratio$ variable. It indicates the proportion of the dataset’s\n", @@ -1417,8 +1606,10 @@ }, { "cell_type": "markdown", - "id": "e88d3db5", - "metadata": {}, + "id": "cc465156", + "metadata": { + "editable": true + }, "source": [ "## Back to the Cancer Data\n", "We can now repeat the above but applied to real data, in this case our breast cancer data.\n", @@ -1428,8 +1619,11 @@ { "cell_type": "code", "execution_count": 14, - "id": "8827956d", - "metadata": {}, + "id": "859d9ce2", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -1464,8 +1658,10 @@ }, { "cell_type": "markdown", - "id": "0a9a1a5b", - "metadata": {}, + "id": "502e1090", + "metadata": { + "editable": true + }, "source": [ "We see that our training data after the PCA decomposition has a performance similar to the non-scaled data. \n", "\n", @@ -1480,8 +1676,11 @@ { "cell_type": "code", "execution_count": 15, - "id": "caba8d7a", - "metadata": {}, + "id": "ce7f2415", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "pca = PCA()\n", @@ -1492,8 +1691,10 @@ }, { "cell_type": "markdown", - "id": "2227518c", - "metadata": {}, + "id": "ed80c53e", + "metadata": { + "editable": true + }, "source": [ "You could then set $n\\_components=d$ and run PCA again. However, there is a much better option: instead\n", "of specifying the number of principal components you want to preserve, you can set $n\\_components$ to be\n", @@ -1503,8 +1704,11 @@ { "cell_type": "code", "execution_count": 16, - "id": "c1e4e8af", - "metadata": {}, + "id": "a0e5c9a3", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "pca = PCA(n_components=0.95)\n", @@ -1513,8 +1717,10 @@ }, { "cell_type": "markdown", - "id": "36612657", - "metadata": {}, + "id": "6ea9e244", + "metadata": { + "editable": true + }, "source": [ "## Incremental PCA\n", "\n", @@ -1527,8 +1733,10 @@ }, { "cell_type": "markdown", - "id": "b4ae2de1", - "metadata": {}, + "id": "4148098e", + "metadata": { + "editable": true + }, "source": [ "### Randomized PCA\n", "\n", @@ -1540,8 +1748,10 @@ }, { "cell_type": "markdown", - "id": "2a3ddfdb", - "metadata": {}, + "id": "4bb617a3", + "metadata": { + "editable": true + }, "source": [ "### Kernel PCA\n", "\n", @@ -1559,8 +1769,11 @@ { "cell_type": "code", "execution_count": 17, - "id": "1a0eff2b", - "metadata": {}, + "id": "8ffe6575", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "from sklearn.decomposition import KernelPCA\n", @@ -1570,8 +1783,10 @@ }, { "cell_type": "markdown", - "id": "3ba826a2", - "metadata": {}, + "id": "598c9f74", + "metadata": { + "editable": true + }, "source": [ "## Other techniques\n", "\n", @@ -1589,8 +1804,10 @@ }, { "cell_type": "markdown", - "id": "e55c1fea", - "metadata": {}, + "id": "a8598b1f", + "metadata": { + "editable": true + }, "source": [ "## Clustering and Unsupervised Learning\n", "\n", @@ -1607,8 +1824,10 @@ }, { "cell_type": "markdown", - "id": "51491a2a", - "metadata": {}, + "id": "540d97f0", + "metadata": { + "editable": true + }, "source": [ "## Basic Idea of the $k$-means Clustering Algorithm\n", "\n", @@ -1621,8 +1840,10 @@ }, { "cell_type": "markdown", - "id": "05ddabda", - "metadata": {}, + "id": "304be3ce", + "metadata": { + "editable": true + }, "source": [ "## The $k$-means Algorithm\n", "\n", @@ -1632,8 +1853,10 @@ }, { "cell_type": "markdown", - "id": "61d59585", - "metadata": {}, + "id": "b71f9b24", + "metadata": { + "editable": true + }, "source": [ "$$\n", "k\\in\\{1, \\cdots, K \\}.\n", @@ -1642,8 +1865,10 @@ }, { "cell_type": "markdown", - "id": "525c822f", - "metadata": {}, + "id": "47118ffe", + "metadata": { + "editable": true + }, "source": [ "In the basic k-means algorithm each point is assigned to only\n", "one cluster $k$, and these assignments are *non-injective* i.e. many-to-one. We\n", @@ -1662,8 +1887,10 @@ }, { "cell_type": "markdown", - "id": "4bcec766", - "metadata": {}, + "id": "414f27d9", + "metadata": { + "editable": true + }, "source": [ "## Basic Math of the $k$-means Algorithm\n", "\n", @@ -1672,8 +1899,10 @@ }, { "cell_type": "markdown", - "id": "e3123956", - "metadata": {}, + "id": "47b62c37", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -1687,8 +1916,10 @@ }, { "cell_type": "markdown", - "id": "cd990c92", - "metadata": {}, + "id": "04fe3795", + "metadata": { + "editable": true + }, "source": [ "which we wish to group into $K < n$ clusters. For our dissimilarity measure we\n", "use the *squared Euclidean distance*" @@ -1696,8 +1927,10 @@ }, { "cell_type": "markdown", - "id": "3cdc5334", - "metadata": {}, + "id": "46514258", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -1712,8 +1945,10 @@ }, { "cell_type": "markdown", - "id": "9fd93eae", - "metadata": {}, + "id": "bb932e77", + "metadata": { + "editable": true + }, "source": [ "## Within Cluster Point Scatter\n", "\n", @@ -1724,8 +1959,10 @@ }, { "cell_type": "markdown", - "id": "10772d66", - "metadata": {}, + "id": "2e6e94ae", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -1741,8 +1978,10 @@ }, { "cell_type": "markdown", - "id": "8b6352eb", - "metadata": {}, + "id": "46c9f8d1", + "metadata": { + "editable": true + }, "source": [ "where $\\boldsymbol{\\overline{x_k}}$ is the mean vector associated with the $k$-th\n", "cluster, and $N_k = \\sum_{i=1}^nI(C(i) = k)$, where the $I()$ notation is\n", @@ -1756,8 +1995,10 @@ }, { "cell_type": "markdown", - "id": "40b02015", - "metadata": {}, + "id": "97fc8a6f", + "metadata": { + "editable": true + }, "source": [ "## More Details\n", "\n", @@ -1766,8 +2007,10 @@ }, { "cell_type": "markdown", - "id": "d7e33ce2", - "metadata": {}, + "id": "2ae904b2", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -1785,8 +2028,10 @@ }, { "cell_type": "markdown", - "id": "b56cc352", - "metadata": {}, + "id": "a4621209", + "metadata": { + "editable": true + }, "source": [ "This is a quantity that is conserved throughout the $k$-means algorithm. It can\n", "be thought of as the total amount of information in the data, and it is composed\n", @@ -1797,8 +2042,10 @@ }, { "cell_type": "markdown", - "id": "f02bc219", - "metadata": {}, + "id": "afef0b66", + "metadata": { + "editable": true + }, "source": [ "## Total Cluster Variance\n", "Given a cluster mean $\\boldsymbol{m_k}$ we define the **total cluster variance**" @@ -1806,8 +2053,10 @@ }, { "cell_type": "markdown", - "id": "81800ba9", - "metadata": {}, + "id": "925ec65d", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -1821,16 +2070,20 @@ }, { "cell_type": "markdown", - "id": "91dfc235", - "metadata": {}, + "id": "d0c934dd", + "metadata": { + "editable": true + }, "source": [ "Now we have all the pieces necessary to formally revisit the $k$-means algorithm." ] }, { "cell_type": "markdown", - "id": "72426487", - "metadata": {}, + "id": "34db61f9", + "metadata": { + "editable": true + }, "source": [ "## The $k$-means Clustering Algorithm\n", "\n", @@ -1845,8 +2098,10 @@ }, { "cell_type": "markdown", - "id": "0d2fa15d", - "metadata": {}, + "id": "08193f84", + "metadata": { + "editable": true + }, "source": [ "## Summarizing\n", "\n", @@ -1863,8 +2118,10 @@ }, { "cell_type": "markdown", - "id": "d9a45ff0", - "metadata": {}, + "id": "8170d2e7", + "metadata": { + "editable": true + }, "source": [ "## Writing our own Code, the Data Set\n", "\n", @@ -1881,8 +2138,11 @@ { "cell_type": "code", "execution_count": 18, - "id": "0391ce61", - "metadata": {}, + "id": "4cde86f0", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "import time\n", @@ -1898,8 +2158,10 @@ }, { "cell_type": "markdown", - "id": "8f1f9c5b", - "metadata": {}, + "id": "582d6422", + "metadata": { + "editable": true + }, "source": [ "Next we define functions, for ease of use later, to generate Gaussians and to\n", "set up our toy data set." @@ -1908,8 +2170,11 @@ { "cell_type": "code", "execution_count": 19, - "id": "38163479", - "metadata": {}, + "id": "40f27206", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "def gaussian_points(dim=2, n_points=1000, mean_vector=np.array([0, 0]),\n", @@ -1965,8 +2230,10 @@ }, { "cell_type": "markdown", - "id": "2a397151", - "metadata": {}, + "id": "df7aa5a4", + "metadata": { + "editable": true + }, "source": [ "## Implementing the $k$-means Algorithm\n", "\n", @@ -1977,8 +2244,11 @@ { "cell_type": "code", "execution_count": 20, - "id": "46e120cb", - "metadata": {}, + "id": "bd626561", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\n", @@ -2019,8 +2289,10 @@ }, { "cell_type": "markdown", - "id": "70bf2219", - "metadata": {}, + "id": "79d02c64", + "metadata": { + "editable": true + }, "source": [ "## Plotting" ] @@ -2028,8 +2300,11 @@ { "cell_type": "code", "execution_count": 21, - "id": "230ccf7a", - "metadata": {}, + "id": "4f8c9bd3", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "fig = plt.figure()\n", @@ -2049,8 +2324,10 @@ }, { "cell_type": "markdown", - "id": "0dc77504", - "metadata": {}, + "id": "10635a19", + "metadata": { + "editable": true + }, "source": [ "So what do we have so far? We have 'picked' $k$ centroids at random from our\n", "data points. There are other ways of more intelligently choosing their\n", @@ -2066,8 +2343,10 @@ }, { "cell_type": "markdown", - "id": "96cb7ce3", - "metadata": {}, + "id": "c51bd121", + "metadata": { + "editable": true + }, "source": [ "## Continuing" ] @@ -2075,8 +2354,11 @@ { "cell_type": "code", "execution_count": 22, - "id": "c0c6583a", - "metadata": {}, + "id": "b972b140", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "\n", @@ -2128,8 +2410,10 @@ }, { "cell_type": "markdown", - "id": "e0ad94a0", - "metadata": {}, + "id": "633c70f7", + "metadata": { + "editable": true + }, "source": [ "## Wrapping it up\n", "We now have a simple , un-optimized $k$-means\n", @@ -2139,8 +2423,11 @@ { "cell_type": "code", "execution_count": 23, - "id": "1f43fb46", - "metadata": {}, + "id": "2b5464e3", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "fig = plt.figure()\n", @@ -2161,8 +2448,11 @@ { "cell_type": "code", "execution_count": 24, - "id": "13d966fe", - "metadata": {}, + "id": "5a6ff4fc", + "metadata": { + "collapsed": false, + "editable": true + }, "outputs": [], "source": [ "def naive_kmeans(data, n_clusters=4, max_iterations=100, tolerance=1e-8):\n", @@ -2237,16 +2527,20 @@ }, { "cell_type": "markdown", - "id": "69426baf", - "metadata": {}, + "id": "ff1b4372", + "metadata": { + "editable": true + }, "source": [ "## Summary of course" ] }, { "cell_type": "markdown", - "id": "f1d1af84", - "metadata": {}, + "id": "0a755725", + "metadata": { + "editable": true + }, "source": [ "## What? Me worry? No final exam in this course!\n", "\n", @@ -2258,8 +2552,10 @@ }, { "cell_type": "markdown", - "id": "009ad2bf", - "metadata": {}, + "id": "6c7696f3", + "metadata": { + "editable": true + }, "source": [ "## What is the link between Artificial Intelligence and Machine Learning and some general Remarks\n", "\n", @@ -2272,8 +2568,10 @@ }, { "cell_type": "markdown", - "id": "e9146fc4", - "metadata": {}, + "id": "90b35824", + "metadata": { + "editable": true + }, "source": [ "## Going back to the beginning of the semester\n", "\n", @@ -2295,8 +2593,10 @@ }, { "cell_type": "markdown", - "id": "1f0a4a39", - "metadata": {}, + "id": "8efa8c76", + "metadata": { + "editable": true + }, "source": [ "## Not so sharp distinctions\n", "\n", @@ -2322,8 +2622,10 @@ }, { "cell_type": "markdown", - "id": "947159c1", - "metadata": {}, + "id": "bc8bdf59", + "metadata": { + "editable": true + }, "source": [ "## Topics we have covered this year\n", "\n", @@ -2336,8 +2638,10 @@ }, { "cell_type": "markdown", - "id": "fc11bb61", - "metadata": {}, + "id": "09d61ddb", + "metadata": { + "editable": true + }, "source": [ "## Statistical analysis and optimization of data\n", "\n", @@ -2359,8 +2663,10 @@ }, { "cell_type": "markdown", - "id": "1609dbcc", - "metadata": {}, + "id": "ea0ba52c", + "metadata": { + "editable": true + }, "source": [ "## Machine learning\n", "\n", @@ -2404,8 +2710,10 @@ }, { "cell_type": "markdown", - "id": "d723bf9f", - "metadata": {}, + "id": "f69dab53", + "metadata": { + "editable": true + }, "source": [ "## Learning outcomes and overarching aims of this course\n", "\n", @@ -2438,8 +2746,10 @@ }, { "cell_type": "markdown", - "id": "fd34c3e4", - "metadata": {}, + "id": "b8447f3c", + "metadata": { + "editable": true + }, "source": [ "## Perspective on Machine Learning\n", "\n", @@ -2459,8 +2769,10 @@ }, { "cell_type": "markdown", - "id": "78bb6872", - "metadata": {}, + "id": "653b631f", + "metadata": { + "editable": true + }, "source": [ "## Machine Learning Research\n", "\n", @@ -2480,8 +2792,10 @@ }, { "cell_type": "markdown", - "id": "79e69fc7", - "metadata": {}, + "id": "5e68a875", + "metadata": { + "editable": true + }, "source": [ "## Starting your Machine Learning Project\n", "\n", @@ -2498,8 +2812,10 @@ }, { "cell_type": "markdown", - "id": "3460e395", - "metadata": {}, + "id": "3e1cfe54", + "metadata": { + "editable": true + }, "source": [ "## Choose a Model and Algorithm\n", "\n", @@ -2512,8 +2828,10 @@ }, { "cell_type": "markdown", - "id": "9d474778", - "metadata": {}, + "id": "f9179cf9", + "metadata": { + "editable": true + }, "source": [ "## Preparing Your Data\n", "\n", @@ -2542,8 +2860,10 @@ }, { "cell_type": "markdown", - "id": "a1212230", - "metadata": {}, + "id": "a8efa03f", + "metadata": { + "editable": true + }, "source": [ "## Which Activation and Weights to Choose in Neural Networks\n", "\n", @@ -2562,8 +2882,10 @@ }, { "cell_type": "markdown", - "id": "263ca04a", - "metadata": {}, + "id": "89029bad", + "metadata": { + "editable": true + }, "source": [ "## Optimization Methods and Hyperparameters\n", "1. Stochastic gradient descent\n", @@ -2585,8 +2907,10 @@ }, { "cell_type": "markdown", - "id": "cc713d6f", - "metadata": {}, + "id": "32ad2a2b", + "metadata": { + "editable": true + }, "source": [ "## Resampling\n", "\n", @@ -2601,8 +2925,10 @@ }, { "cell_type": "markdown", - "id": "2a1064c4", - "metadata": {}, + "id": "8d1f9a34", + "metadata": { + "editable": true + }, "source": [ "## Other courses on Data science and Machine Learning at UiO\n", "\n", @@ -2627,8 +2953,10 @@ }, { "cell_type": "markdown", - "id": "d67ca1f2", - "metadata": {}, + "id": "195f6c61", + "metadata": { + "editable": true + }, "source": [ "## Additional courses of interest\n", "\n", @@ -2639,8 +2967,10 @@ }, { "cell_type": "markdown", - "id": "078c92cc", - "metadata": {}, + "id": "940bfecc", + "metadata": { + "editable": true + }, "source": [ "## What's the future like?\n", "\n", @@ -2663,8 +2993,10 @@ }, { "cell_type": "markdown", - "id": "0b549993", - "metadata": {}, + "id": "8ee522b4", + "metadata": { + "editable": true + }, "source": [ "## Types of Machine Learning, a repetition\n", "\n", @@ -2690,8 +3022,10 @@ }, { "cell_type": "markdown", - "id": "b0703acd", - "metadata": {}, + "id": "154c60af", + "metadata": { + "editable": true + }, "source": [ "## Why Boltzmann machines?\n", "\n", @@ -2705,8 +3039,10 @@ }, { "cell_type": "markdown", - "id": "7a5e1f12", - "metadata": {}, + "id": "28e36d07", + "metadata": { + "editable": true + }, "source": [ "## Boltzmann Machines\n", "\n", @@ -2725,8 +3061,10 @@ }, { "cell_type": "markdown", - "id": "ed9ba5d5", - "metadata": {}, + "id": "3e521a91", + "metadata": { + "editable": true + }, "source": [ "## Some similarities and differences from DNNs\n", "\n", @@ -2741,8 +3079,10 @@ }, { "cell_type": "markdown", - "id": "1572964e", - "metadata": {}, + "id": "f86a72bf", + "metadata": { + "editable": true + }, "source": [ "## Boltzmann machines (BM)\n", "\n", @@ -2762,8 +3102,10 @@ }, { "cell_type": "markdown", - "id": "ad39b364", - "metadata": {}, + "id": "239fb0f3", + "metadata": { + "editable": true + }, "source": [ "## A standard BM setup\n", "\n", @@ -2779,8 +3121,10 @@ }, { "cell_type": "markdown", - "id": "78ad73f3", - "metadata": {}, + "id": "748af18d", + "metadata": { + "editable": true + }, "source": [ "## The structure of the RBM network\n", "\n", @@ -2793,8 +3137,10 @@ }, { "cell_type": "markdown", - "id": "5fbe92ba", - "metadata": {}, + "id": "03e2c5aa", + "metadata": { + "editable": true + }, "source": [ "## The network\n", "\n", @@ -2806,8 +3152,10 @@ }, { "cell_type": "markdown", - "id": "0c61992d", - "metadata": {}, + "id": "4c350f63", + "metadata": { + "editable": true + }, "source": [ "## Goals\n", "\n", @@ -2828,8 +3176,10 @@ }, { "cell_type": "markdown", - "id": "4c0baf07", - "metadata": {}, + "id": "7e3219bc", + "metadata": { + "editable": true + }, "source": [ "## Joint distribution\n", "\n", @@ -2838,8 +3188,10 @@ }, { "cell_type": "markdown", - "id": "fe5131be", - "metadata": {}, + "id": "b5cd0a37", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2854,16 +3206,20 @@ }, { "cell_type": "markdown", - "id": "466c20e2", - "metadata": {}, + "id": "a877a5c1", + "metadata": { + "editable": true + }, "source": [ "where $Z$ is the normalization constant or partition function, defined as" ] }, { "cell_type": "markdown", - "id": "8c56a663", - "metadata": {}, + "id": "8dd20d9c", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2878,16 +3234,20 @@ }, { "cell_type": "markdown", - "id": "9aad45e0", - "metadata": {}, + "id": "8254835e", + "metadata": { + "editable": true + }, "source": [ "It is common to ignore $T_0$ by setting it to one." ] }, { "cell_type": "markdown", - "id": "a3e4545f", - "metadata": {}, + "id": "799714b0", + "metadata": { + "editable": true + }, "source": [ "## Network Elements, the energy function\n", "\n", @@ -2903,8 +3263,10 @@ }, { "cell_type": "markdown", - "id": "c59a927f", - "metadata": {}, + "id": "20277412", + "metadata": { + "editable": true + }, "source": [ "$$\n", "E(\\hat{x},\\hat{h}) = -\\sum_{ia}^{NA}b_i^a \\alpha_i^a(x_i)-\\sum_{jd}^{MD}c_j^d \\beta_j^d(h_j)-\\sum_{ijad}^{NAMD}b_i^a \\alpha_i^a(x_i)c_j^d \\beta_j^d(h_j)w_{ij}^{ad}.\n", @@ -2913,8 +3275,10 @@ }, { "cell_type": "markdown", - "id": "243fba20", - "metadata": {}, + "id": "bda5e6fb", + "metadata": { + "editable": true + }, "source": [ "Here $\\beta_j^d(h_j)$ and $\\alpha_i^a(x_j)$ are so-called transfer functions that map a given input value to a desired feature value. The labels $a$ and $d$ denote that there can be multiple transfer functions per variable. The first sum depends only on the visible units. The second on the hidden ones. **Note** that there is no connection between nodes in a layer.\n", "\n", @@ -2925,8 +3289,10 @@ }, { "cell_type": "markdown", - "id": "f47aa4b1", - "metadata": {}, + "id": "b3af16bf", + "metadata": { + "editable": true + }, "source": [ "## Defining different types of RBMs\n", "There are different variants of RBMs, and the differences lie in the types of visible and hidden units we choose as well as in the implementation of the energy function $E(\\mathbf{x},\\mathbf{h})$. \n", @@ -2938,8 +3304,10 @@ }, { "cell_type": "markdown", - "id": "cb41f814", - "metadata": {}, + "id": "95773541", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2954,8 +3322,10 @@ }, { "cell_type": "markdown", - "id": "27f2e905", - "metadata": {}, + "id": "d7608162", + "metadata": { + "editable": true + }, "source": [ "where the binary values taken on by the nodes are most commonly 0 and 1.\n", "\n", @@ -2966,8 +3336,10 @@ }, { "cell_type": "markdown", - "id": "03754b04", - "metadata": {}, + "id": "3a924890", + "metadata": { + "editable": true + }, "source": [ "\n", "
\n", @@ -2982,8 +3354,10 @@ }, { "cell_type": "markdown", - "id": "c5a5a9d7", - "metadata": {}, + "id": "8a386df4", + "metadata": { + "editable": true + }, "source": [ "## More about RBMs\n", "1. Useful when we model continuous data (i.e., we wish $\\mathbf{x}$ to be continuous)\n", @@ -3004,8 +3378,10 @@ }, { "cell_type": "markdown", - "id": "034a17d3", - "metadata": {}, + "id": "e23f14f5", + "metadata": { + "editable": true + }, "source": [ "## Autoencoders: Overarching view\n", "\n", @@ -3040,8 +3416,10 @@ }, { "cell_type": "markdown", - "id": "6a974e18", - "metadata": {}, + "id": "ace868c7", + "metadata": { + "editable": true + }, "source": [ "## Bayesian Machine Learning\n", "\n", @@ -3061,8 +3439,10 @@ }, { "cell_type": "markdown", - "id": "e6375edf", - "metadata": {}, + "id": "cde3892c", + "metadata": { + "editable": true + }, "source": [ "## Reinforcement Learning\n", "\n", @@ -3095,8 +3475,10 @@ }, { "cell_type": "markdown", - "id": "466b36c2", - "metadata": {}, + "id": "44bf4dbc", + "metadata": { + "editable": true + }, "source": [ "## Transfer learning\n", "\n", @@ -3114,8 +3496,10 @@ }, { "cell_type": "markdown", - "id": "f54342d9", - "metadata": {}, + "id": "11a175a9", + "metadata": { + "editable": true + }, "source": [ "## Adversarial learning\n", "\n", @@ -3134,8 +3518,10 @@ }, { "cell_type": "markdown", - "id": "8360b9e6", - "metadata": {}, + "id": "bdec4129", + "metadata": { + "editable": true + }, "source": [ "## Dual learning\n", "\n", @@ -3152,8 +3538,10 @@ }, { "cell_type": "markdown", - "id": "dad35e23", - "metadata": {}, + "id": "6ff589ba", + "metadata": { + "editable": true + }, "source": [ "## Distributed machine learning\n", "\n", @@ -3165,8 +3553,10 @@ }, { "cell_type": "markdown", - "id": "b48da1d7", - "metadata": {}, + "id": "b9c7ede8", + "metadata": { + "editable": true + }, "source": [ "## Meta learning\n", "\n", @@ -3181,8 +3571,10 @@ }, { "cell_type": "markdown", - "id": "a34c0717", - "metadata": {}, + "id": "13f8a2f6", + "metadata": { + "editable": true + }, "source": [ "## The Challenges Facing Machine Learning\n", "\n", @@ -3209,8 +3601,10 @@ }, { "cell_type": "markdown", - "id": "4f136fbe", - "metadata": {}, + "id": "54100bf2", + "metadata": { + "editable": true + }, "source": [ "## Explainable machine learning\n", "\n", @@ -3236,8 +3630,10 @@ }, { "cell_type": "markdown", - "id": "801ac0b3", - "metadata": {}, + "id": "58d52c0d", + "metadata": { + "editable": true + }, "source": [ "## Quantum machine learning\n", "\n", @@ -3265,8 +3661,10 @@ }, { "cell_type": "markdown", - "id": "86184acf", - "metadata": {}, + "id": "728d0de2", + "metadata": { + "editable": true + }, "source": [ "## Quantum machine learning algorithms based on linear algebra\n", "\n", @@ -3286,8 +3684,10 @@ }, { "cell_type": "markdown", - "id": "a1541609", - "metadata": {}, + "id": "29d83534", + "metadata": { + "editable": true + }, "source": [ "## Quantum reinforcement learning\n", "\n", @@ -3302,8 +3702,10 @@ }, { "cell_type": "markdown", - "id": "ea83f4f8", - "metadata": {}, + "id": "7e58fd32", + "metadata": { + "editable": true + }, "source": [ "## Quantum deep learning\n", "\n", @@ -3323,8 +3725,10 @@ }, { "cell_type": "markdown", - "id": "92de4ba3", - "metadata": {}, + "id": "2c80fe68", + "metadata": { + "editable": true + }, "source": [ "## Social machine learning\n", "\n", @@ -3342,8 +3746,10 @@ }, { "cell_type": "markdown", - "id": "72764dfe", - "metadata": {}, + "id": "ffe1e0bb", + "metadata": { + "editable": true + }, "source": [ "## The last words?\n", "\n", @@ -3356,8 +3762,10 @@ }, { "cell_type": "markdown", - "id": "c065f1a5", - "metadata": {}, + "id": "b5ecf668", + "metadata": { + "editable": true + }, "source": [ "## Best wishes to you all and thanks so much for your heroic efforts this semester\n", "\n", @@ -3369,25 +3777,7 @@ ] } ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.10" - } - }, + "metadata": {}, "nbformat": 4, "nbformat_minor": 5 } diff --git a/doc/src/week47/week47.do.txt b/doc/src/week47/week47.do.txt index a26df5f91..42e0b7dc8 100644 --- a/doc/src/week47/week47.do.txt +++ b/doc/src/week47/week47.do.txt @@ -6,6 +6,7 @@ DATE: today ===== Overview of week 47 ===== * _Thursday_: Dimensionality reduction and unsupervised learning: Principal Component analysis (PCA) and clustering + * "Video of lecture":"https://youtu.be/VJIsEQM2lCI" * _Friday_: PCA and clustering and Summary of Course !bblock