From 2fb0987483e57b893b49032bc3cd9e857a82b6e3 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Fri, 10 Sep 2021 08:37:02 +0200 Subject: [PATCH] added dots files --- ...._GenerativeAdversarialNetworks-bs000.html | 703 +++ .../GenerativeAdversarialNetworks-bs.html | 703 +++ .../GenerativeAdversarialNetworks-reveal.html | 884 ++++ ...nerativeAdversarialNetworks-solarized.html | 676 +++ .../html/GenerativeAdversarialNetworks.html | 681 +++ .../html/reveal.js/.gitignore | 8 + .../html/reveal.js/.travis.yml | 5 + .../html/reveal.js/CONTRIBUTING.md | 23 + .../html/reveal.js/Gruntfile.js | 140 + .../html/reveal.js/LICENSE | 19 + .../html/reveal.js/README.md | 1052 ++++ .../html/reveal.js/bower.json | 27 + .../html/reveal.js/css/images/cbc_footer.png | Bin 0 -> 10008 bytes .../html/reveal.js/css/images/cbc_symbol.png | Bin 0 -> 2946 bytes .../reveal.js/css/images/simula_footer.png | Bin 0 -> 2513 bytes .../html/reveal.js/css/images/simula_logo.png | Bin 0 -> 2138 bytes .../reveal.js/css/images/simula_symbol.png | Bin 0 -> 2138 bytes .../html/reveal.js/css/images/uio_footer.png | Bin 0 -> 18189 bytes .../html/reveal.js/css/images/uio_symbol.png | Bin 0 -> 11352 bytes .../html/reveal.js/css/print/paper.css | 202 + .../html/reveal.js/css/print/pdf.css | 157 + .../html/reveal.js/css/reveal.css | 1886 +++++++ .../html/reveal.js/css/reveal.scss | 1319 +++++ .../html/reveal.js/css/theme/README.md | 23 + .../html/reveal.js/css/theme/beige.css | 154 + .../html/reveal.js/css/theme/beigesmall.css | 155 + .../html/reveal.js/css/theme/black.css | 273 + .../html/reveal.js/css/theme/blood.css | 180 + .../html/reveal.js/css/theme/cbc.css | 144 + .../html/reveal.js/css/theme/darkgray.css | 153 + .../html/reveal.js/css/theme/default.css | 153 + .../html/reveal.js/css/theme/league.css | 279 + .../html/reveal.js/css/theme/moon.css | 153 + .../html/reveal.js/css/theme/night.css | 141 + .../html/reveal.js/css/theme/serif.css | 143 + .../html/reveal.js/css/theme/simple.css | 144 + .../html/reveal.js/css/theme/simula.css | 144 + .../html/reveal.js/css/theme/sky.css | 150 + .../html/reveal.js/css/theme/solarized.css | 153 + .../reveal.js/css/theme/source/beige.scss | 50 + .../css/theme/source/beigesmall.scss | 51 + .../reveal.js/css/theme/source/black.scss | 49 + .../reveal.js/css/theme/source/blood.scss | 91 + .../html/reveal.js/css/theme/source/cbc.scss | 39 + .../reveal.js/css/theme/source/darkgray.scss | 42 + .../reveal.js/css/theme/source/default.scss | 42 + .../reveal.js/css/theme/source/league.scss | 34 + .../html/reveal.js/css/theme/source/moon.scss | 68 + .../reveal.js/css/theme/source/night.scss | 35 + .../reveal.js/css/theme/source/serif.scss | 35 + .../reveal.js/css/theme/source/simple.scss | 38 + .../reveal.js/css/theme/source/simula.scss | 39 + .../html/reveal.js/css/theme/source/sky.scss | 46 + .../reveal.js/css/theme/source/solarized.scss | 74 + .../reveal.js/css/theme/source/white.scss | 49 + .../reveal.js/css/theme/template/mixins.scss | 29 + .../css/theme/template/settings.scss | 34 + .../reveal.js/css/theme/template/theme.scss | 171 + .../html/reveal.js/css/theme/white.css | 273 + .../html/reveal.js/index.html | 411 ++ .../html/reveal.js/js/reveal.js | 4508 +++++++++++++++++ .../html/reveal.js/lib/css/zenburn.css | 117 + .../reveal.js/lib/font/league-gothic/LICENSE | 2 + .../lib/font/league-gothic/league-gothic.css | 10 + .../lib/font/league-gothic/league-gothic.eot | Bin 0 -> 25696 bytes .../lib/font/league-gothic/league-gothic.ttf | Bin 0 -> 64256 bytes .../lib/font/league-gothic/league-gothic.woff | Bin 0 -> 30764 bytes .../lib/font/source-sans-pro/LICENSE | 45 + .../source-sans-pro-italic.eot | Bin 0 -> 75720 bytes .../source-sans-pro-italic.ttf | Bin 0 -> 238084 bytes .../source-sans-pro-italic.woff | Bin 0 -> 98556 bytes .../source-sans-pro-regular.eot | Bin 0 -> 88070 bytes .../source-sans-pro-regular.ttf | Bin 0 -> 288008 bytes .../source-sans-pro-regular.woff | Bin 0 -> 114324 bytes .../source-sans-pro-semibold.eot | Bin 0 -> 89897 bytes .../source-sans-pro-semibold.ttf | Bin 0 -> 284640 bytes .../source-sans-pro-semibold.woff | Bin 0 -> 115648 bytes .../source-sans-pro-semibolditalic.eot | Bin 0 -> 75706 bytes .../source-sans-pro-semibolditalic.ttf | Bin 0 -> 240944 bytes .../source-sans-pro-semibolditalic.woff | Bin 0 -> 98816 bytes .../font/source-sans-pro/source-sans-pro.css | 39 + .../html/reveal.js/lib/js/classList.js | 2 + .../html/reveal.js/lib/js/head.min.js | 8 + .../html/reveal.js/lib/js/html5shiv.js | 7 + .../html/reveal.js/package.json | 45 + .../reveal.js/plugin/highlight/highlight.js | 30 + .../html/reveal.js/plugin/leap/leap.js | 159 + .../reveal.js/plugin/markdown/example.html | 129 + .../html/reveal.js/plugin/markdown/example.md | 31 + .../reveal.js/plugin/markdown/markdown.js | 393 ++ .../html/reveal.js/plugin/markdown/marked.js | 6 + .../html/reveal.js/plugin/math/math.js | 64 + .../html/reveal.js/plugin/multiplex/client.js | 13 + .../html/reveal.js/plugin/multiplex/index.js | 56 + .../html/reveal.js/plugin/multiplex/master.js | 51 + .../reveal.js/plugin/notes-server/client.js | 60 + .../reveal.js/plugin/notes-server/index.js | 66 + .../reveal.js/plugin/notes-server/notes.html | 396 ++ .../html/reveal.js/plugin/notes/notes.html | 406 ++ .../html/reveal.js/plugin/notes/notes.js | 122 + .../reveal.js/plugin/print-pdf/print-pdf.js | 48 + .../html/reveal.js/plugin/remotes/remotes.js | 39 + .../html/reveal.js/plugin/search/search.js | 196 + .../html/reveal.js/plugin/zoom-js/zoom.js | 278 + .../reveal.js/test/examples/assets/image1.png | Bin 0 -> 21991 bytes .../reveal.js/test/examples/assets/image2.png | Bin 0 -> 10237 bytes .../reveal.js/test/examples/barebones.html | 41 + .../test/examples/embedded-media.html | 49 + .../html/reveal.js/test/examples/math.html | 185 + .../test/examples/slide-backgrounds.html | 144 + .../test/examples/slide-transitions.html | 101 + .../html/reveal.js/test/qunit-1.12.0.css | 244 + .../html/reveal.js/test/qunit-1.12.0.js | 2212 ++++++++ .../test-markdown-element-attributes.html | 134 + .../test/test-markdown-element-attributes.js | 46 + .../test/test-markdown-slide-attributes.html | 128 + .../test/test-markdown-slide-attributes.js | 47 + .../html/reveal.js/test/test-markdown.html | 52 + .../html/reveal.js/test/test-markdown.js | 15 + .../html/reveal.js/test/test-pdf.html | 83 + .../html/reveal.js/test/test-pdf.js | 15 + .../html/reveal.js/test/test.html | 85 + .../html/reveal.js/test/test.js | 589 +++ .../ipynb/GenerativeAdversarialNetworks.ipynb | 945 ++++ ...b-GenerativeAdversarialNetworks-src.tar.gz | Bin 0 -> 29 bytes .../Results/FigureFiles/Masses2016OLS.png | Bin 0 -> 11846 bytes doc/pub/week35/html/._week35-bs000.html | 458 ++ doc/pub/week35/html/._week35-bs001.html | 440 ++ doc/pub/week35/html/._week35-bs002.html | 444 ++ doc/pub/week35/html/._week35-bs003.html | 457 ++ doc/pub/week35/html/._week35-bs004.html | 458 ++ doc/pub/week35/html/._week35-bs005.html | 467 ++ doc/pub/week35/html/._week35-bs006.html | 464 ++ doc/pub/week35/html/._week35-bs007.html | 457 ++ doc/pub/week35/html/._week35-bs008.html | 457 ++ doc/pub/week35/html/._week35-bs009.html | 481 ++ doc/pub/week35/html/._week35-bs010.html | 471 ++ doc/pub/week35/html/._week35-bs011.html | 467 ++ doc/pub/week35/html/._week35-bs012.html | 468 ++ doc/pub/week35/html/._week35-bs013.html | 529 ++ doc/pub/week35/html/._week35-bs014.html | 475 ++ doc/pub/week35/html/._week35-bs015.html | 495 ++ doc/pub/week35/html/._week35-bs016.html | 484 ++ doc/pub/week35/html/._week35-bs017.html | 463 ++ doc/pub/week35/html/._week35-bs018.html | 473 ++ doc/pub/week35/html/._week35-bs019.html | 467 ++ doc/pub/week35/html/._week35-bs020.html | 480 ++ doc/pub/week35/html/._week35-bs021.html | 479 ++ doc/pub/week35/html/._week35-bs022.html | 465 ++ doc/pub/week35/html/._week35-bs023.html | 485 ++ doc/pub/week35/html/._week35-bs024.html | 466 ++ doc/pub/week35/html/._week35-bs025.html | 469 ++ doc/pub/week35/html/._week35-bs026.html | 595 +++ doc/pub/week35/html/._week35-bs027.html | 473 ++ doc/pub/week35/html/._week35-bs028.html | 468 ++ doc/pub/week35/html/._week35-bs029.html | 454 ++ doc/pub/week35/html/._week35-bs030.html | 470 ++ doc/pub/week35/html/._week35-bs031.html | 454 ++ doc/pub/week35/html/._week35-bs032.html | 479 ++ doc/pub/week35/html/._week35-bs033.html | 455 ++ doc/pub/week35/html/._week35-bs034.html | 503 ++ doc/pub/week35/html/._week35-bs035.html | 539 ++ doc/pub/week35/html/._week35-bs036.html | 479 ++ doc/pub/week35/html/._week35-bs037.html | 452 ++ doc/pub/week35/html/._week35-bs038.html | 464 ++ doc/pub/week35/html/._week35-bs039.html | 488 ++ doc/pub/week35/html/._week35-bs040.html | 489 ++ doc/pub/week35/html/._week35-bs041.html | 466 ++ doc/pub/week35/html/._week35-bs042.html | 477 ++ doc/pub/week35/html/._week35-bs043.html | 490 ++ doc/pub/week35/html/._week35-bs044.html | 463 ++ doc/pub/week35/html/._week35-bs045.html | 486 ++ doc/pub/week35/html/._week35-bs046.html | 459 ++ doc/pub/week35/html/._week35-bs047.html | 449 ++ doc/pub/week35/html/._week35-bs048.html | 480 ++ doc/pub/week35/html/._week35-bs049.html | 501 ++ doc/pub/week35/html/._week35-bs050.html | 495 ++ doc/pub/week35/html/._week35-bs051.html | 491 ++ doc/pub/week35/html/._week35-bs052.html | 470 ++ doc/pub/week35/html/._week35-bs053.html | 483 ++ doc/pub/week35/html/._week35-bs054.html | 469 ++ doc/pub/week35/html/._week35-bs055.html | 499 ++ doc/pub/week35/html/._week35-bs056.html | 479 ++ doc/pub/week35/html/._week35-bs057.html | 483 ++ doc/pub/week35/html/._week35-bs058.html | 466 ++ doc/pub/week35/html/._week35-bs059.html | 501 ++ doc/pub/week35/html/._week35-bs060.html | 479 ++ doc/pub/week35/html/._week35-bs061.html | 472 ++ doc/pub/week35/html/._week35-bs062.html | 477 ++ doc/pub/week35/html/._week35-bs063.html | 472 ++ doc/pub/week35/html/._week35-bs064.html | 490 ++ doc/pub/week35/html/._week35-bs065.html | 494 ++ doc/pub/week35/html/._week35-bs066.html | 452 ++ doc/pub/week35/html/._week35-bs067.html | 466 ++ doc/pub/week35/html/._week35-bs068.html | 459 ++ doc/pub/week35/html/._week35-bs069.html | 741 +++ doc/pub/week35/html/._week35-bs070.html | 494 ++ doc/pub/week35/html/._week35-bs071.html | 745 +++ doc/pub/week36/html/._week36-bs000.html | 452 ++ doc/pub/week36/html/._week36-bs001.html | 440 ++ doc/pub/week36/html/._week36-bs002.html | 429 ++ doc/pub/week36/html/._week36-bs003.html | 430 ++ doc/pub/week36/html/._week36-bs004.html | 463 ++ doc/pub/week36/html/._week36-bs005.html | 470 ++ doc/pub/week36/html/._week36-bs006.html | 467 ++ doc/pub/week36/html/._week36-bs007.html | 478 ++ doc/pub/week36/html/._week36-bs008.html | 484 ++ doc/pub/week36/html/._week36-bs009.html | 454 ++ doc/pub/week36/html/._week36-bs010.html | 472 ++ doc/pub/week36/html/._week36-bs011.html | 478 ++ doc/pub/week36/html/._week36-bs012.html | 454 ++ doc/pub/week36/html/._week36-bs013.html | 454 ++ doc/pub/week36/html/._week36-bs014.html | 469 ++ doc/pub/week36/html/._week36-bs015.html | 463 ++ doc/pub/week36/html/._week36-bs016.html | 452 ++ doc/pub/week36/html/._week36-bs017.html | 448 ++ doc/pub/week36/html/._week36-bs018.html | 458 ++ doc/pub/week36/html/._week36-bs019.html | 451 ++ doc/pub/week36/html/._week36-bs020.html | 453 ++ doc/pub/week36/html/._week36-bs021.html | 457 ++ doc/pub/week36/html/._week36-bs022.html | 466 ++ doc/pub/week36/html/._week36-bs023.html | 462 ++ doc/pub/week36/html/._week36-bs024.html | 462 ++ doc/pub/week36/html/._week36-bs025.html | 499 ++ doc/pub/week36/html/._week36-bs026.html | 497 ++ doc/pub/week36/html/._week36-bs027.html | 519 ++ doc/pub/week36/html/._week36-bs028.html | 437 ++ doc/pub/week36/html/._week36-bs029.html | 478 ++ doc/pub/week36/html/._week36-bs030.html | 452 ++ doc/pub/week36/html/._week36-bs031.html | 467 ++ doc/pub/week36/html/._week36-bs032.html | 516 ++ doc/pub/week36/html/._week36-bs033.html | 456 ++ doc/pub/week36/html/._week36-bs034.html | 469 ++ doc/pub/week36/html/._week36-bs035.html | 462 ++ doc/pub/week36/html/._week36-bs036.html | 464 ++ doc/pub/week36/html/._week36-bs037.html | 475 ++ doc/pub/week36/html/._week36-bs038.html | 449 ++ doc/pub/week36/html/._week36-bs039.html | 449 ++ doc/pub/week36/html/._week36-bs040.html | 451 ++ doc/pub/week36/html/._week36-bs041.html | 449 ++ doc/pub/week36/html/._week36-bs042.html | 459 ++ doc/pub/week36/html/._week36-bs043.html | 460 ++ doc/pub/week36/html/._week36-bs044.html | 446 ++ doc/pub/week36/html/._week36-bs045.html | 517 ++ doc/pub/week36/html/._week36-bs046.html | 467 ++ doc/pub/week36/html/._week36-bs047.html | 473 ++ doc/pub/week36/html/._week36-bs048.html | 467 ++ doc/pub/week36/html/._week36-bs049.html | 449 ++ doc/pub/week36/html/._week36-bs050.html | 466 ++ doc/pub/week36/html/._week36-bs051.html | 462 ++ doc/pub/week36/html/._week36-bs052.html | 449 ++ doc/pub/week36/html/._week36-bs053.html | 455 ++ doc/pub/week36/html/._week36-bs054.html | 461 ++ doc/pub/week36/html/._week36-bs055.html | 455 ++ doc/pub/week36/html/._week36-bs056.html | 451 ++ doc/pub/week36/html/._week36-bs057.html | 468 ++ doc/pub/week36/html/._week36-bs058.html | 454 ++ doc/pub/week36/html/._week36-bs059.html | 448 ++ doc/pub/week36/html/._week36-bs060.html | 454 ++ doc/pub/week36/html/._week36-bs061.html | 453 ++ doc/pub/week36/html/._week36-bs062.html | 457 ++ doc/pub/week36/html/._week36-bs063.html | 496 ++ doc/pub/week36/html/._week36-bs064.html | 453 ++ doc/pub/week36/html/._week36-bs065.html | 464 ++ doc/pub/week36/html/._week36-bs066.html | 452 ++ doc/pub/week36/html/._week36-bs067.html | 529 ++ doc/pub/week36/html/._week36-bs068.html | 494 ++ doc/pub/week36/html/._week36-bs069.html | 489 ++ doc/pub/week36/html/._week36-bs070.html | 480 ++ doc/pub/week36/html/._week36-bs071.html | 462 ++ doc/pub/week36/html/._week36-bs072.html | 503 ++ doc/pub/week36/html/._week36-bs073.html | 510 ++ doc/pub/week36/html/._week36-bs074.html | 498 ++ doc/pub/week36/html/._week36-bs075.html | 464 ++ .../week36-checkpoint.ipynb | 3108 ++++++++++++ 275 files changed, 98525 insertions(+) create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/._GenerativeAdversarialNetworks-bs000.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-bs.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-reveal.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-solarized.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.gitignore create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.travis.yml create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/CONTRIBUTING.md create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/Gruntfile.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/LICENSE create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/README.md create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/bower.json create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/images/cbc_footer.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/images/cbc_symbol.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/images/simula_footer.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/images/simula_logo.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/images/simula_symbol.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/images/uio_footer.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/images/uio_symbol.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/print/paper.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/print/pdf.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/reveal.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/reveal.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/README.md create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/beige.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/beigesmall.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/black.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/blood.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/cbc.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/darkgray.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/default.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/league.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/moon.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/night.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/serif.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/simple.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/simula.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/sky.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/solarized.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/beige.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/beigesmall.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/black.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/blood.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/cbc.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/darkgray.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/default.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/league.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/moon.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/night.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/serif.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/simple.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/simula.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/sky.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/solarized.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/source/white.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/template/mixins.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/template/settings.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/template/theme.scss create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/css/theme/white.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/index.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/js/reveal.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/css/zenburn.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/league-gothic/LICENSE create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/league-gothic/league-gothic.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/league-gothic/league-gothic.eot create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/league-gothic/league-gothic.ttf create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/league-gothic/league-gothic.woff create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/LICENSE create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-italic.eot create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-italic.ttf create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-italic.woff create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-regular.eot create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-regular.ttf create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-regular.woff create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-semibold.eot create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-semibold.ttf create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-semibold.woff create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-semibolditalic.eot create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-semibolditalic.ttf create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro-semibolditalic.woff create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/font/source-sans-pro/source-sans-pro.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/js/classList.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/js/head.min.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/lib/js/html5shiv.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/package.json create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/highlight/highlight.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/leap/leap.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/markdown/example.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/markdown/example.md create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/markdown/markdown.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/markdown/marked.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/math/math.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/multiplex/client.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/multiplex/index.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/multiplex/master.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/notes-server/client.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/notes-server/index.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/notes-server/notes.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/notes/notes.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/notes/notes.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/print-pdf/print-pdf.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/remotes/remotes.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/search/search.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/plugin/zoom-js/zoom.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/examples/assets/image1.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/examples/assets/image2.png create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/examples/barebones.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/examples/embedded-media.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/examples/math.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/examples/slide-backgrounds.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/examples/slide-transitions.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/qunit-1.12.0.css create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/qunit-1.12.0.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-markdown-element-attributes.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-markdown-element-attributes.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-markdown-slide-attributes.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-markdown-slide-attributes.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-markdown.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-markdown.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-pdf.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test-pdf.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test.html create mode 100644 doc/pub/GenerativeAdversarialNetworks/html/reveal.js/test/test.js create mode 100644 doc/pub/GenerativeAdversarialNetworks/ipynb/GenerativeAdversarialNetworks.ipynb create mode 100644 doc/pub/GenerativeAdversarialNetworks/ipynb/ipynb-GenerativeAdversarialNetworks-src.tar.gz create mode 100644 doc/pub/week34/ipynb/Results/FigureFiles/Masses2016OLS.png create mode 100644 doc/pub/week35/html/._week35-bs000.html create mode 100644 doc/pub/week35/html/._week35-bs001.html create mode 100644 doc/pub/week35/html/._week35-bs002.html create mode 100644 doc/pub/week35/html/._week35-bs003.html create mode 100644 doc/pub/week35/html/._week35-bs004.html create mode 100644 doc/pub/week35/html/._week35-bs005.html create mode 100644 doc/pub/week35/html/._week35-bs006.html create mode 100644 doc/pub/week35/html/._week35-bs007.html create mode 100644 doc/pub/week35/html/._week35-bs008.html create mode 100644 doc/pub/week35/html/._week35-bs009.html create mode 100644 doc/pub/week35/html/._week35-bs010.html create mode 100644 doc/pub/week35/html/._week35-bs011.html create mode 100644 doc/pub/week35/html/._week35-bs012.html create mode 100644 doc/pub/week35/html/._week35-bs013.html create mode 100644 doc/pub/week35/html/._week35-bs014.html create mode 100644 doc/pub/week35/html/._week35-bs015.html create mode 100644 doc/pub/week35/html/._week35-bs016.html create mode 100644 doc/pub/week35/html/._week35-bs017.html create mode 100644 doc/pub/week35/html/._week35-bs018.html create mode 100644 doc/pub/week35/html/._week35-bs019.html create mode 100644 doc/pub/week35/html/._week35-bs020.html create mode 100644 doc/pub/week35/html/._week35-bs021.html create mode 100644 doc/pub/week35/html/._week35-bs022.html create mode 100644 doc/pub/week35/html/._week35-bs023.html create mode 100644 doc/pub/week35/html/._week35-bs024.html create mode 100644 doc/pub/week35/html/._week35-bs025.html create mode 100644 doc/pub/week35/html/._week35-bs026.html create mode 100644 doc/pub/week35/html/._week35-bs027.html create mode 100644 doc/pub/week35/html/._week35-bs028.html create mode 100644 doc/pub/week35/html/._week35-bs029.html create mode 100644 doc/pub/week35/html/._week35-bs030.html create mode 100644 doc/pub/week35/html/._week35-bs031.html create mode 100644 doc/pub/week35/html/._week35-bs032.html create mode 100644 doc/pub/week35/html/._week35-bs033.html create mode 100644 doc/pub/week35/html/._week35-bs034.html create mode 100644 doc/pub/week35/html/._week35-bs035.html create mode 100644 doc/pub/week35/html/._week35-bs036.html create mode 100644 doc/pub/week35/html/._week35-bs037.html create mode 100644 doc/pub/week35/html/._week35-bs038.html create mode 100644 doc/pub/week35/html/._week35-bs039.html create mode 100644 doc/pub/week35/html/._week35-bs040.html create mode 100644 doc/pub/week35/html/._week35-bs041.html create mode 100644 doc/pub/week35/html/._week35-bs042.html create mode 100644 doc/pub/week35/html/._week35-bs043.html create mode 100644 doc/pub/week35/html/._week35-bs044.html create mode 100644 doc/pub/week35/html/._week35-bs045.html create mode 100644 doc/pub/week35/html/._week35-bs046.html create mode 100644 doc/pub/week35/html/._week35-bs047.html create mode 100644 doc/pub/week35/html/._week35-bs048.html create mode 100644 doc/pub/week35/html/._week35-bs049.html create mode 100644 doc/pub/week35/html/._week35-bs050.html create mode 100644 doc/pub/week35/html/._week35-bs051.html create mode 100644 doc/pub/week35/html/._week35-bs052.html create mode 100644 doc/pub/week35/html/._week35-bs053.html create mode 100644 doc/pub/week35/html/._week35-bs054.html create mode 100644 doc/pub/week35/html/._week35-bs055.html create mode 100644 doc/pub/week35/html/._week35-bs056.html create mode 100644 doc/pub/week35/html/._week35-bs057.html create mode 100644 doc/pub/week35/html/._week35-bs058.html create mode 100644 doc/pub/week35/html/._week35-bs059.html create mode 100644 doc/pub/week35/html/._week35-bs060.html create mode 100644 doc/pub/week35/html/._week35-bs061.html create mode 100644 doc/pub/week35/html/._week35-bs062.html create mode 100644 doc/pub/week35/html/._week35-bs063.html create mode 100644 doc/pub/week35/html/._week35-bs064.html create mode 100644 doc/pub/week35/html/._week35-bs065.html create mode 100644 doc/pub/week35/html/._week35-bs066.html create mode 100644 doc/pub/week35/html/._week35-bs067.html create mode 100644 doc/pub/week35/html/._week35-bs068.html create mode 100644 doc/pub/week35/html/._week35-bs069.html create mode 100644 doc/pub/week35/html/._week35-bs070.html create mode 100644 doc/pub/week35/html/._week35-bs071.html create mode 100644 doc/pub/week36/html/._week36-bs000.html create mode 100644 doc/pub/week36/html/._week36-bs001.html create mode 100644 doc/pub/week36/html/._week36-bs002.html create mode 100644 doc/pub/week36/html/._week36-bs003.html create mode 100644 doc/pub/week36/html/._week36-bs004.html create mode 100644 doc/pub/week36/html/._week36-bs005.html create mode 100644 doc/pub/week36/html/._week36-bs006.html create mode 100644 doc/pub/week36/html/._week36-bs007.html create mode 100644 doc/pub/week36/html/._week36-bs008.html create mode 100644 doc/pub/week36/html/._week36-bs009.html create mode 100644 doc/pub/week36/html/._week36-bs010.html create mode 100644 doc/pub/week36/html/._week36-bs011.html create mode 100644 doc/pub/week36/html/._week36-bs012.html create mode 100644 doc/pub/week36/html/._week36-bs013.html create mode 100644 doc/pub/week36/html/._week36-bs014.html create mode 100644 doc/pub/week36/html/._week36-bs015.html create mode 100644 doc/pub/week36/html/._week36-bs016.html create mode 100644 doc/pub/week36/html/._week36-bs017.html create mode 100644 doc/pub/week36/html/._week36-bs018.html create mode 100644 doc/pub/week36/html/._week36-bs019.html create mode 100644 doc/pub/week36/html/._week36-bs020.html create mode 100644 doc/pub/week36/html/._week36-bs021.html create mode 100644 doc/pub/week36/html/._week36-bs022.html create mode 100644 doc/pub/week36/html/._week36-bs023.html create mode 100644 doc/pub/week36/html/._week36-bs024.html create mode 100644 doc/pub/week36/html/._week36-bs025.html create mode 100644 doc/pub/week36/html/._week36-bs026.html create mode 100644 doc/pub/week36/html/._week36-bs027.html create mode 100644 doc/pub/week36/html/._week36-bs028.html create mode 100644 doc/pub/week36/html/._week36-bs029.html create mode 100644 doc/pub/week36/html/._week36-bs030.html create mode 100644 doc/pub/week36/html/._week36-bs031.html create mode 100644 doc/pub/week36/html/._week36-bs032.html create mode 100644 doc/pub/week36/html/._week36-bs033.html create mode 100644 doc/pub/week36/html/._week36-bs034.html create mode 100644 doc/pub/week36/html/._week36-bs035.html create mode 100644 doc/pub/week36/html/._week36-bs036.html create mode 100644 doc/pub/week36/html/._week36-bs037.html create mode 100644 doc/pub/week36/html/._week36-bs038.html create mode 100644 doc/pub/week36/html/._week36-bs039.html create mode 100644 doc/pub/week36/html/._week36-bs040.html create mode 100644 doc/pub/week36/html/._week36-bs041.html create mode 100644 doc/pub/week36/html/._week36-bs042.html create mode 100644 doc/pub/week36/html/._week36-bs043.html create mode 100644 doc/pub/week36/html/._week36-bs044.html create mode 100644 doc/pub/week36/html/._week36-bs045.html create mode 100644 doc/pub/week36/html/._week36-bs046.html create mode 100644 doc/pub/week36/html/._week36-bs047.html create mode 100644 doc/pub/week36/html/._week36-bs048.html create mode 100644 doc/pub/week36/html/._week36-bs049.html create mode 100644 doc/pub/week36/html/._week36-bs050.html create mode 100644 doc/pub/week36/html/._week36-bs051.html create mode 100644 doc/pub/week36/html/._week36-bs052.html create mode 100644 doc/pub/week36/html/._week36-bs053.html create mode 100644 doc/pub/week36/html/._week36-bs054.html create mode 100644 doc/pub/week36/html/._week36-bs055.html create mode 100644 doc/pub/week36/html/._week36-bs056.html create mode 100644 doc/pub/week36/html/._week36-bs057.html create mode 100644 doc/pub/week36/html/._week36-bs058.html create mode 100644 doc/pub/week36/html/._week36-bs059.html create mode 100644 doc/pub/week36/html/._week36-bs060.html create mode 100644 doc/pub/week36/html/._week36-bs061.html create mode 100644 doc/pub/week36/html/._week36-bs062.html create mode 100644 doc/pub/week36/html/._week36-bs063.html create mode 100644 doc/pub/week36/html/._week36-bs064.html create mode 100644 doc/pub/week36/html/._week36-bs065.html create mode 100644 doc/pub/week36/html/._week36-bs066.html create mode 100644 doc/pub/week36/html/._week36-bs067.html create mode 100644 doc/pub/week36/html/._week36-bs068.html create mode 100644 doc/pub/week36/html/._week36-bs069.html create mode 100644 doc/pub/week36/html/._week36-bs070.html create mode 100644 doc/pub/week36/html/._week36-bs071.html create mode 100644 doc/pub/week36/html/._week36-bs072.html create mode 100644 doc/pub/week36/html/._week36-bs073.html create mode 100644 doc/pub/week36/html/._week36-bs074.html create mode 100644 doc/pub/week36/html/._week36-bs075.html create mode 100644 doc/pub/week36/ipynb/.ipynb_checkpoints/week36-checkpoint.ipynb diff --git a/doc/pub/GenerativeAdversarialNetworks/html/._GenerativeAdversarialNetworks-bs000.html b/doc/pub/GenerativeAdversarialNetworks/html/._GenerativeAdversarialNetworks-bs000.html new file mode 100644 index 000000000..cc92cfe8b --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/._GenerativeAdversarialNetworks-bs000.html @@ -0,0 +1,703 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +

 

 

 

+ + + + +

Generative Models

+Generative models describe a class of statistical models that are a contrast +to discriminative models. Informally we say that generative models can +generate new data instances while discriminative models discriminate between +different kinds of data instances. A generative model could generate new photos +of animals that look like 'real' animals while a discriminative model could tell +a dog from a cat. More formally, given a data set \( x \) and a set of labels / +targets \( y \). Generative models capture the joint probability \( p(x, y) \), or +just \( p(x) \) if there are no labels, while discriminative models capture the +conditional probability \( p(y | x) \). Discriminative models generally try to draw +boundaries in the data space (often high dimensional), while generative models +try to model how data is placed throughout the space. + +

Generative Adversarial Networks

+Generative Adversarial Networks are a type of unsupervised machine learning +algorithm proposed by Goodfellow et. al +in 2014 (Read the paper first it's only 6 pages). The simplest formulation of +the model is based on a game theoretic approach, zero sum game, where we pit +two neural networks against one another. We define two rival networks, one +generator \( g \), and one discriminator \( d \). The generator directly produces +samples +$$ +\begin{equation} + x = g(z; \theta^{(g)}) +\tag{1} +\end{equation} +$$ + +The discriminator attempts to distinguish between samples drawn from the +training data and samples drawn from the generator. In other words, it tries to +tell the difference between the fake data produced by \( g \) and the actual data +samples we want to do prediction on. The discriminator outputs a probability +value given by + +$$ +\begin{equation} + d(x; \theta^{(d)}) +\tag{2} +\end{equation} +$$ + +

+indicating the probability that \( x \) is a real training example rather than a +fake sample the generator has generated. The simplest way to formulate the +learning process in a generative adversarial network is a zero-sum game, in +which a function + +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) +\tag{3} +\end{equation} +$$ + +

+determines the reward for the discriminator, while the generator gets the +conjugate reward + +$$ +\begin{equation} + -v(\theta^{(g)}, \theta^{(d)}) +\tag{4} +\end{equation} +$$ + +

+During learning both of the networks maximize their own reward function, so that +the generator gets better and better at tricking the discriminator, while the +discriminator gets better and better at telling the difference between the fake +and real data. The generator and discriminator alternate on which one trains at +one time (i.e. for one epoch). In other words, we keep the generator constant +and train the discriminator, then we keep the discriminator constant to train +the generator and repeat. It is this back and forth dynamic which lets GANs +tackle otherwise intractable generative problems. As the generator improves with + training, the discriminator's performance gets worse because it cannot easily + tell the difference between real and fake. If the generator ends up succeeding + perfectly, the the discriminator will do no better than random guessing i.e. + 50\%. This progression in the training poses a problem for the convergence + criteria for GANs. The discriminator feedback gets less meaningful over time, + if we continue training after this point then the generator is effectively + training on junk data which can undo the learning up to that point. Therefore, + we stop training when the discriminator starts outputting \( 1/2 \) everywhere. + At convergence we have + +$$ +\begin{equation} + g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\tag{5} +\end{equation} +$$ + +The default choice for \( v \) is +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x) + + \mathbb{E}_{x\sim p_\mathrm{model}} + \log (1 - d(x)) +\tag{6} +\end{equation} +$$ + +The main motivation for the design of GANs is that the learning process requires +neither approximate inference (variational autoencoders for example) nor +approximation of a partition function. In the case where +$$ +\begin{equation} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\tag{7} +\end{equation} +$$ + +is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is +asymptotically consistent +( Seth Lloyd on QuGANs ). This is in +general not the case and it is possible to get situations where the training +process never converges because the generator and discriminator chase one +another around in the parameter space indefinitely. A much deeper discussion on +the currently open research problem of GAN convergence is available +here. To +anyone interested in learning more about GANs it is a highly recommended read. +Direct quote: "In this best-performing formulation, the generator aims to +increase the log probability that the discriminator makes a mistake, rather than +aiming to decrease the log probability that the discriminator makes the correct +prediction." Another interesting read + +

Writing Our First Generative Adversarial Network

+Let us now move on to actually implementing a GAN in tensorflow. We will study +the performance of our GAN on the MNIST dataset. This code is based on and +adapted from the +google tutorial + +

+First we import our libraries + +

+ + +

import os
+import time
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+from tensorflow.keras import layers
+from tensorflow.keras.utils import plot_model
+
+

+Next we define our hyperparameters and import our data the usual way + +

+ + +

BUFFER_SIZE = 60000
+BATCH_SIZE = 256
+EPOCHS = 30
+
+data = tf.keras.datasets.mnist.load_data()
+(train_images, train_labels), (test_images, test_labels) = data
+train_images = np.reshape(train_images, (train_images.shape[0],
+                                         28,
+                                         28,
+                                         1)).astype('float32')
+
+# we normalize between -1 and 1
+train_images = (train_images - 127.5) / 127.5
+training_dataset = tf.data.Dataset.from_tensor_slices(
+                      train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
+
+

+Let's have a quick look + +

+ + +

plt.imshow(train_images[0], cmap='Greys')
+plt.show()
+
+

+Now we define our two models. This is where the 'magic' happens. There are a +huge amount of possible formulations for both models. A lot of engineering and +trial and error can be done here to try to produce better performing models. For +more advanced GANs this is by far the step where you can 'make or break' a +model. + +

+We start with the generator. As stated in the introductory text the generator +\( g \) upsamples from a random sample to the shape of what we want to predict. In +our case we are trying to predict MNIST images (\( 28\times 28 \) pixels). + +

+ + +

def generator_model():
+    """
+    The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to
+    produce an image from a random seed. We start with a Dense layer taking this
+    random sample as an input and subsequently upsample through multiple
+    convolutional layers.
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+
+
+    # adding our input layer. Dense means that every neuron is connected and
+    # the input shape is the shape of our random noise. The units need to match
+    # in some sense the upsampling strides to reach our desired output shape.
+    # we are using 100 random numbers as our seed
+    model.add(layers.Dense(units=7*7*BATCH_SIZE,
+                           use_bias=False,
+                           input_shape=(100, )))
+    # we normalize the output form the Dense layer
+    model.add(layers.BatchNormalization())
+    # and add an activation function to our 'layer'. LeakyReLU avoids vanishing
+    # gradient problem
+    model.add(layers.LeakyReLU())
+    model.add(layers.Reshape((7, 7, BATCH_SIZE)))
+    assert model.output_shape == (None, 7, 7, BATCH_SIZE)
+    # even though we just added four keras layers we think of everything above
+    # as 'one' layer
+
+    # next we add our upscaling convolutional layers
+    model.add(layers.Conv2DTranspose(filters=128,
+                                     kernel_size=(5, 5),
+                                     strides=(1, 1),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 7, 7, 128)
+
+    model.add(layers.Conv2DTranspose(filters=64,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 14, 14, 64)
+
+    model.add(layers.Conv2DTranspose(filters=1,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False,
+                                     activation='tanh'))
+    assert model.output_shape == (None, 28, 28, 1)
+
+    return model
+
+

+And there we have our 'simple' generator model. Now we move on to defining our +discriminator model \( d \), which is a convolutional neural network based image +classifier. + +

+ + +

def discriminator_model():
+    """
+    The discriminator is a convolutional neural network based image classifier
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+    model.add(layers.Conv2D(filters=64,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same',
+                            input_shape=[28, 28, 1]))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+
+    model.add(layers.Conv2D(filters=128,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same'))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+    model.add(layers.Flatten())
+    model.add(layers.Dense(1))
+
+    return model
+
+

+Let us take a look at our models. Note: double click images for bigger view. + +

+ + +

generator = generator_model()
+plot_model(generator, show_shapes=True, rankdir='LR')
+
+

+ + +

discriminator = discriminator_model()
+plot_model(discriminator, show_shapes=True, rankdir='LR')
+
+

+Next we need a few helper objects we will use in training + +

+ + +

cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)
+generator_optimizer = tf.keras.optimizers.Adam(1e-4)
+discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)
+
+

+The first object, cross_entropy is our loss function and the two others are +our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This +is because they need to improve their accuracy at approximately equal speeds to +get convergence (not necessarily exactly equal). Now we define our loss +functions + +

+ + +

def generator_loss(fake_output):
+    loss = cross_entropy(tf.ones_like(fake_output), fake_output)
+
+    return loss
+
+

+ + +

def discriminator_loss(real_output, fake_output):
+    real_loss = cross_entropy(tf.ones_like(real_output), real_output)
+    fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
+    total_loss = real_loss + fake_loss
+
+    return total_loss
+
+

+Next we define a kind of seed to help us compare the learning process over +multiple training epochs. + +

+ + +

noise_dimension = 100
+n_examples_to_generate = 16
+seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
+
+

+Now we have everything we need to define our training step, which we will apply +for every step in our training loop. Notice the @tf.function flag signifying +that the function is tensorflow 'compiled'. Removing this flag doubles the +computation time. + +

+ + +

@tf.function
+def train_step(images):
+    noise = tf.random.normal([BATCH_SIZE, noise_dimension])
+
+    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
+        generated_images = generator(noise, training=True)
+
+        real_output = discriminator(images, training=True)
+        fake_output = discriminator(generated_images, training=True)
+
+        gen_loss = generator_loss(fake_output)
+        disc_loss = discriminator_loss(real_output, fake_output)
+
+    gradients_of_generator = gen_tape.gradient(gen_loss,
+                                            generator.trainable_variables)
+    gradients_of_discriminator = disc_tape.gradient(disc_loss,
+                                            discriminator.trainable_variables)
+    generator_optimizer.apply_gradients(zip(gradients_of_generator,
+                                            generator.trainable_variables))
+    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator,
+                                            discriminator.trainable_variables))
+
+    return gen_loss, disc_loss
+
+

+Next we define a helper function to produce an output over our training epochs +to see the predictive progression of our generator model. Note: I am including +this code here, but comment it out in the training loop. +

+ + +

def generate_and_save_images(model, epoch, test_input):
+    # we're making inferences here
+    predictions = model(test_input, training=False)
+
+    fig = plt.figure(figsize=(4, 4))
+
+    for i in range(predictions.shape[0]):
+        plt.subplot(4, 4, i+1)
+        plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')
+        plt.axis('off')
+
+    plt.savefig(f'./images_from_seed_images/image_at_epoch_{str(epoch).zfill(3)}.png')
+    plt.close()
+    #plt.show()
+
+

+Setting up checkpoints to periodically save our model during training so that +everything is not lost even if the program were to somehow terminate while +training. + +

+ + +

# Setting up checkpoints to save model during training
+checkpoint_dir = './training_checkpoints'
+checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')
+checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
+                            discriminator_optimizer=discriminator_optimizer,
+                            generator=generator,
+                            discriminator=discriminator)
+
+

+Now we define our training loop + +

+ + +

def train(dataset, epochs):
+    generator_loss_list = []
+    discriminator_loss_list = []
+
+    for epoch in range(epochs):
+        start = time.time()
+
+        for image_batch in dataset:
+            gen_loss, disc_loss = train_step(image_batch)
+            generator_loss_list.append(gen_loss.numpy())
+            discriminator_loss_list.append(disc_loss.numpy())
+
+        #generate_and_save_images(generator, epoch + 1, seed_images)
+
+        if (epoch + 1) % 15 == 0:
+            checkpoint.save(file_prefix=checkpoint_prefix)
+
+        print(f'Time for epoch {epoch} is {time.time() - start}')
+
+    #generate_and_save_images(generator, epochs, seed_images)
+
+    loss_file = './data/lossfile.txt'
+    with open(loss_file, 'w') as outfile:
+        outfile.write(str(generator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+        outfile.write(str(discriminator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+
+

+To train simply call this function. Warning: this might take a long time so +there is a folder of a pretrained network already included in the repository. + +

+ + +

train(train_dataset, EPOCHS)
+
+

+And here is the result of training our model for 100 epochs + +

+ + +

+ +

+Now to avoid having to train and everything, which will take a while depending +on your computer setup we now load in the model which produced the above gif. + +

+ + +

checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
+restored_generator = checkpoint.generator
+restored_discriminator = checkpoint.discriminator
+
+print(restored_generator)
+print(restored_discriminator)
+
+ +

Exploring the Latent Space

+ +

+So we have successfully loaded in our latest model. Let us now play around a bit +and see what kind of things we can learn about this model. Our generator takes +an array of 100 numbers. One idea can be to try to systematically change our +input. Let us try and see what we get + +

+ + +

def generate_latent_points(number=100, scale_means=1, scale_stds=1):
+    latent_dim = 100
+    means = scale_means * tf.linspace(-1, 1, num=latent_dim)
+    stds = scale_stds * tf.linspace(-1, 1, num=latent_dim)
+    latent_space_value_range = tf.random.normal([number, number],
+                                                means,
+                                                stds,
+                                                dtype=tf.float64)
+
+    return latent_space_value_range
+
+def generate_images(latent_points):
+    # notice we set training to false because we are making inferences
+    generated_images = restored_generator(latent_space_value_range,
+                                          training=False)
+
+    return generated_images
+
+

+ + +

def plot_result(generated_images, number):
+    # obviously this assumes sqrt number is an int
+    fig, axs = plt.subplots(int(np.sqrt(number)), int(np.sqrt(number)),
+                            figsize=(10, 10))
+
+    for i in range(int(np.sqrt(number))):
+        for j in range(int(np.sqrt(number))):
+            axs[i, j].imshow(generated_images[i*j], cmap='Greys')
+            axs[i, j].axis('off')
+
+    plt.show()
+
+

+ + +

generated_images = generate_images(generate_latent_points())
+plot_result(generated_images, number)
+
+

+Interesting! We see that the generator generates images that look like MNIST +numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able +to generate a similar plot where we generate every MNIST number. Let us now try +to 'move' a bit around in the latent space. Note: decrease the plot number if +these following cells take too long to run on your computer. + +

+ + +

plot_number = 225
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=-5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=5))
+plot_result(generated_images, plot_number)
+
+

+Again, we have found something interesting. Moving around using our means +takes us from digit to digit, while moving around using our standard +deviations seem to increase the number of different digits! In the last image +above, we can barely make out every MNIST digit. Let us make on last plot using +this information by upping the standard deviation of our Gaussian noises. + +

+ + +

plot_number = 400
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=10))
+
+

+A pretty cool result! We see that our generator indeed has learned a +distribution which qualitatively looks a whole lot like the MNIST dataset. + +

Interpolating Between MNIST Digits

+Another interesting way to explore the latent space of our generator model is by +interpolating between the MNIST digits. This section is largely based on +"this excellent blogpost": https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/ +by Jason Brownlee. + +

+So let us start + +

+ +

+ + +
+ + + + + + + + + + + diff --git a/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-bs.html b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-bs.html new file mode 100644 index 000000000..cc92cfe8b --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-bs.html @@ -0,0 +1,703 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +

 

 

 

+ + + + +

Generative Models

+Generative models describe a class of statistical models that are a contrast +to discriminative models. Informally we say that generative models can +generate new data instances while discriminative models discriminate between +different kinds of data instances. A generative model could generate new photos +of animals that look like 'real' animals while a discriminative model could tell +a dog from a cat. More formally, given a data set \( x \) and a set of labels / +targets \( y \). Generative models capture the joint probability \( p(x, y) \), or +just \( p(x) \) if there are no labels, while discriminative models capture the +conditional probability \( p(y | x) \). Discriminative models generally try to draw +boundaries in the data space (often high dimensional), while generative models +try to model how data is placed throughout the space. + +

Generative Adversarial Networks

+Generative Adversarial Networks are a type of unsupervised machine learning +algorithm proposed by Goodfellow et. al +in 2014 (Read the paper first it's only 6 pages). The simplest formulation of +the model is based on a game theoretic approach, zero sum game, where we pit +two neural networks against one another. We define two rival networks, one +generator \( g \), and one discriminator \( d \). The generator directly produces +samples +$$ +\begin{equation} + x = g(z; \theta^{(g)}) +\tag{1} +\end{equation} +$$ + +The discriminator attempts to distinguish between samples drawn from the +training data and samples drawn from the generator. In other words, it tries to +tell the difference between the fake data produced by \( g \) and the actual data +samples we want to do prediction on. The discriminator outputs a probability +value given by + +$$ +\begin{equation} + d(x; \theta^{(d)}) +\tag{2} +\end{equation} +$$ + +

+indicating the probability that \( x \) is a real training example rather than a +fake sample the generator has generated. The simplest way to formulate the +learning process in a generative adversarial network is a zero-sum game, in +which a function + +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) +\tag{3} +\end{equation} +$$ + +

+determines the reward for the discriminator, while the generator gets the +conjugate reward + +$$ +\begin{equation} + -v(\theta^{(g)}, \theta^{(d)}) +\tag{4} +\end{equation} +$$ + +

+During learning both of the networks maximize their own reward function, so that +the generator gets better and better at tricking the discriminator, while the +discriminator gets better and better at telling the difference between the fake +and real data. The generator and discriminator alternate on which one trains at +one time (i.e. for one epoch). In other words, we keep the generator constant +and train the discriminator, then we keep the discriminator constant to train +the generator and repeat. It is this back and forth dynamic which lets GANs +tackle otherwise intractable generative problems. As the generator improves with + training, the discriminator's performance gets worse because it cannot easily + tell the difference between real and fake. If the generator ends up succeeding + perfectly, the the discriminator will do no better than random guessing i.e. + 50\%. This progression in the training poses a problem for the convergence + criteria for GANs. The discriminator feedback gets less meaningful over time, + if we continue training after this point then the generator is effectively + training on junk data which can undo the learning up to that point. Therefore, + we stop training when the discriminator starts outputting \( 1/2 \) everywhere. + At convergence we have + +$$ +\begin{equation} + g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\tag{5} +\end{equation} +$$ + +The default choice for \( v \) is +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x) + + \mathbb{E}_{x\sim p_\mathrm{model}} + \log (1 - d(x)) +\tag{6} +\end{equation} +$$ + +The main motivation for the design of GANs is that the learning process requires +neither approximate inference (variational autoencoders for example) nor +approximation of a partition function. In the case where +$$ +\begin{equation} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\tag{7} +\end{equation} +$$ + +is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is +asymptotically consistent +( Seth Lloyd on QuGANs ). This is in +general not the case and it is possible to get situations where the training +process never converges because the generator and discriminator chase one +another around in the parameter space indefinitely. A much deeper discussion on +the currently open research problem of GAN convergence is available +here. To +anyone interested in learning more about GANs it is a highly recommended read. +Direct quote: "In this best-performing formulation, the generator aims to +increase the log probability that the discriminator makes a mistake, rather than +aiming to decrease the log probability that the discriminator makes the correct +prediction." Another interesting read + +

Writing Our First Generative Adversarial Network

+Let us now move on to actually implementing a GAN in tensorflow. We will study +the performance of our GAN on the MNIST dataset. This code is based on and +adapted from the +google tutorial + +

+First we import our libraries + +

+ + +

import os
+import time
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+from tensorflow.keras import layers
+from tensorflow.keras.utils import plot_model
+
+

+Next we define our hyperparameters and import our data the usual way + +

+ + +

BUFFER_SIZE = 60000
+BATCH_SIZE = 256
+EPOCHS = 30
+
+data = tf.keras.datasets.mnist.load_data()
+(train_images, train_labels), (test_images, test_labels) = data
+train_images = np.reshape(train_images, (train_images.shape[0],
+                                         28,
+                                         28,
+                                         1)).astype('float32')
+
+# we normalize between -1 and 1
+train_images = (train_images - 127.5) / 127.5
+training_dataset = tf.data.Dataset.from_tensor_slices(
+                      train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
+
+

+Let's have a quick look + +

+ + +

plt.imshow(train_images[0], cmap='Greys')
+plt.show()
+
+

+Now we define our two models. This is where the 'magic' happens. There are a +huge amount of possible formulations for both models. A lot of engineering and +trial and error can be done here to try to produce better performing models. For +more advanced GANs this is by far the step where you can 'make or break' a +model. + +

+We start with the generator. As stated in the introductory text the generator +\( g \) upsamples from a random sample to the shape of what we want to predict. In +our case we are trying to predict MNIST images (\( 28\times 28 \) pixels). + +

+ + +

def generator_model():
+    """
+    The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to
+    produce an image from a random seed. We start with a Dense layer taking this
+    random sample as an input and subsequently upsample through multiple
+    convolutional layers.
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+
+
+    # adding our input layer. Dense means that every neuron is connected and
+    # the input shape is the shape of our random noise. The units need to match
+    # in some sense the upsampling strides to reach our desired output shape.
+    # we are using 100 random numbers as our seed
+    model.add(layers.Dense(units=7*7*BATCH_SIZE,
+                           use_bias=False,
+                           input_shape=(100, )))
+    # we normalize the output form the Dense layer
+    model.add(layers.BatchNormalization())
+    # and add an activation function to our 'layer'. LeakyReLU avoids vanishing
+    # gradient problem
+    model.add(layers.LeakyReLU())
+    model.add(layers.Reshape((7, 7, BATCH_SIZE)))
+    assert model.output_shape == (None, 7, 7, BATCH_SIZE)
+    # even though we just added four keras layers we think of everything above
+    # as 'one' layer
+
+    # next we add our upscaling convolutional layers
+    model.add(layers.Conv2DTranspose(filters=128,
+                                     kernel_size=(5, 5),
+                                     strides=(1, 1),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 7, 7, 128)
+
+    model.add(layers.Conv2DTranspose(filters=64,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 14, 14, 64)
+
+    model.add(layers.Conv2DTranspose(filters=1,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False,
+                                     activation='tanh'))
+    assert model.output_shape == (None, 28, 28, 1)
+
+    return model
+
+

+And there we have our 'simple' generator model. Now we move on to defining our +discriminator model \( d \), which is a convolutional neural network based image +classifier. + +

+ + +

def discriminator_model():
+    """
+    The discriminator is a convolutional neural network based image classifier
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+    model.add(layers.Conv2D(filters=64,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same',
+                            input_shape=[28, 28, 1]))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+
+    model.add(layers.Conv2D(filters=128,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same'))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+    model.add(layers.Flatten())
+    model.add(layers.Dense(1))
+
+    return model
+
+

+Let us take a look at our models. Note: double click images for bigger view. + +

+ + +

generator = generator_model()
+plot_model(generator, show_shapes=True, rankdir='LR')
+
+

+ + +

discriminator = discriminator_model()
+plot_model(discriminator, show_shapes=True, rankdir='LR')
+
+

+Next we need a few helper objects we will use in training + +

+ + +

cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)
+generator_optimizer = tf.keras.optimizers.Adam(1e-4)
+discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)
+
+

+The first object, cross_entropy is our loss function and the two others are +our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This +is because they need to improve their accuracy at approximately equal speeds to +get convergence (not necessarily exactly equal). Now we define our loss +functions + +

+ + +

def generator_loss(fake_output):
+    loss = cross_entropy(tf.ones_like(fake_output), fake_output)
+
+    return loss
+
+

+ + +

def discriminator_loss(real_output, fake_output):
+    real_loss = cross_entropy(tf.ones_like(real_output), real_output)
+    fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
+    total_loss = real_loss + fake_loss
+
+    return total_loss
+
+

+Next we define a kind of seed to help us compare the learning process over +multiple training epochs. + +

+ + +

noise_dimension = 100
+n_examples_to_generate = 16
+seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
+
+

+Now we have everything we need to define our training step, which we will apply +for every step in our training loop. Notice the @tf.function flag signifying +that the function is tensorflow 'compiled'. Removing this flag doubles the +computation time. + +

+ + +

@tf.function
+def train_step(images):
+    noise = tf.random.normal([BATCH_SIZE, noise_dimension])
+
+    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
+        generated_images = generator(noise, training=True)
+
+        real_output = discriminator(images, training=True)
+        fake_output = discriminator(generated_images, training=True)
+
+        gen_loss = generator_loss(fake_output)
+        disc_loss = discriminator_loss(real_output, fake_output)
+
+    gradients_of_generator = gen_tape.gradient(gen_loss,
+                                            generator.trainable_variables)
+    gradients_of_discriminator = disc_tape.gradient(disc_loss,
+                                            discriminator.trainable_variables)
+    generator_optimizer.apply_gradients(zip(gradients_of_generator,
+                                            generator.trainable_variables))
+    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator,
+                                            discriminator.trainable_variables))
+
+    return gen_loss, disc_loss
+
+

+Next we define a helper function to produce an output over our training epochs +to see the predictive progression of our generator model. Note: I am including +this code here, but comment it out in the training loop. +

+ + +

def generate_and_save_images(model, epoch, test_input):
+    # we're making inferences here
+    predictions = model(test_input, training=False)
+
+    fig = plt.figure(figsize=(4, 4))
+
+    for i in range(predictions.shape[0]):
+        plt.subplot(4, 4, i+1)
+        plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')
+        plt.axis('off')
+
+    plt.savefig(f'./images_from_seed_images/image_at_epoch_{str(epoch).zfill(3)}.png')
+    plt.close()
+    #plt.show()
+
+

+Setting up checkpoints to periodically save our model during training so that +everything is not lost even if the program were to somehow terminate while +training. + +

+ + +

# Setting up checkpoints to save model during training
+checkpoint_dir = './training_checkpoints'
+checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')
+checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
+                            discriminator_optimizer=discriminator_optimizer,
+                            generator=generator,
+                            discriminator=discriminator)
+
+

+Now we define our training loop + +

+ + +

def train(dataset, epochs):
+    generator_loss_list = []
+    discriminator_loss_list = []
+
+    for epoch in range(epochs):
+        start = time.time()
+
+        for image_batch in dataset:
+            gen_loss, disc_loss = train_step(image_batch)
+            generator_loss_list.append(gen_loss.numpy())
+            discriminator_loss_list.append(disc_loss.numpy())
+
+        #generate_and_save_images(generator, epoch + 1, seed_images)
+
+        if (epoch + 1) % 15 == 0:
+            checkpoint.save(file_prefix=checkpoint_prefix)
+
+        print(f'Time for epoch {epoch} is {time.time() - start}')
+
+    #generate_and_save_images(generator, epochs, seed_images)
+
+    loss_file = './data/lossfile.txt'
+    with open(loss_file, 'w') as outfile:
+        outfile.write(str(generator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+        outfile.write(str(discriminator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+
+

+To train simply call this function. Warning: this might take a long time so +there is a folder of a pretrained network already included in the repository. + +

+ + +

train(train_dataset, EPOCHS)
+
+

+And here is the result of training our model for 100 epochs + +

+ + +

+ +

+Now to avoid having to train and everything, which will take a while depending +on your computer setup we now load in the model which produced the above gif. + +

+ + +

checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
+restored_generator = checkpoint.generator
+restored_discriminator = checkpoint.discriminator
+
+print(restored_generator)
+print(restored_discriminator)
+
+ +

Exploring the Latent Space

+ +

+So we have successfully loaded in our latest model. Let us now play around a bit +and see what kind of things we can learn about this model. Our generator takes +an array of 100 numbers. One idea can be to try to systematically change our +input. Let us try and see what we get + +

+ + +

def generate_latent_points(number=100, scale_means=1, scale_stds=1):
+    latent_dim = 100
+    means = scale_means * tf.linspace(-1, 1, num=latent_dim)
+    stds = scale_stds * tf.linspace(-1, 1, num=latent_dim)
+    latent_space_value_range = tf.random.normal([number, number],
+                                                means,
+                                                stds,
+                                                dtype=tf.float64)
+
+    return latent_space_value_range
+
+def generate_images(latent_points):
+    # notice we set training to false because we are making inferences
+    generated_images = restored_generator(latent_space_value_range,
+                                          training=False)
+
+    return generated_images
+
+

+ + +

def plot_result(generated_images, number):
+    # obviously this assumes sqrt number is an int
+    fig, axs = plt.subplots(int(np.sqrt(number)), int(np.sqrt(number)),
+                            figsize=(10, 10))
+
+    for i in range(int(np.sqrt(number))):
+        for j in range(int(np.sqrt(number))):
+            axs[i, j].imshow(generated_images[i*j], cmap='Greys')
+            axs[i, j].axis('off')
+
+    plt.show()
+
+

+ + +

generated_images = generate_images(generate_latent_points())
+plot_result(generated_images, number)
+
+

+Interesting! We see that the generator generates images that look like MNIST +numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able +to generate a similar plot where we generate every MNIST number. Let us now try +to 'move' a bit around in the latent space. Note: decrease the plot number if +these following cells take too long to run on your computer. + +

+ + +

plot_number = 225
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=-5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=5))
+plot_result(generated_images, plot_number)
+
+

+Again, we have found something interesting. Moving around using our means +takes us from digit to digit, while moving around using our standard +deviations seem to increase the number of different digits! In the last image +above, we can barely make out every MNIST digit. Let us make on last plot using +this information by upping the standard deviation of our Gaussian noises. + +

+ + +

plot_number = 400
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=10))
+
+

+A pretty cool result! We see that our generator indeed has learned a +distribution which qualitatively looks a whole lot like the MNIST dataset. + +

Interpolating Between MNIST Digits

+Another interesting way to explore the latent space of our generator model is by +interpolating between the MNIST digits. This section is largely based on +"this excellent blogpost": https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/ +by Jason Brownlee. + +

+So let us start + +

+ +

+ + +
+ + + + + + + + + + + diff --git a/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-reveal.html b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-reveal.html new file mode 100644 index 000000000..ae4679c8b --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-reveal.html @@ -0,0 +1,884 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ + + + + + + + + + + + + + +
+ + +

Generative Models

+Generative models describe a class of statistical models that are a contrast +to discriminative models. Informally we say that generative models can +generate new data instances while discriminative models discriminate between +different kinds of data instances. A generative model could generate new photos +of animals that look like 'real' animals while a discriminative model could tell +a dog from a cat. More formally, given a data set \( x \) and a set of labels / +targets \( y \). Generative models capture the joint probability \( p(x, y) \), or +just \( p(x) \) if there are no labels, while discriminative models capture the +conditional probability \( p(y | x) \). Discriminative models generally try to draw +boundaries in the data space (often high dimensional), while generative models +try to model how data is placed throughout the space. + +

Generative Adversarial Networks

+Generative Adversarial Networks are a type of unsupervised machine learning +algorithm proposed by Goodfellow et. al +in 2014 (Read the paper first it's only 6 pages). The simplest formulation of +the model is based on a game theoretic approach, zero sum game, where we pit +two neural networks against one another. We define two rival networks, one +generator \( g \), and one discriminator \( d \). The generator directly produces +samples +

 
+$$ +\begin{equation} + x = g(z; \theta^{(g)}) +\tag{1} +\end{equation} +$$ +

 
+ +The discriminator attempts to distinguish between samples drawn from the +training data and samples drawn from the generator. In other words, it tries to +tell the difference between the fake data produced by \( g \) and the actual data +samples we want to do prediction on. The discriminator outputs a probability +value given by + +

 
+$$ +\begin{equation} + d(x; \theta^{(d)}) +\tag{2} +\end{equation} +$$ +

 
+ +

+indicating the probability that \( x \) is a real training example rather than a +fake sample the generator has generated. The simplest way to formulate the +learning process in a generative adversarial network is a zero-sum game, in +which a function + +

 
+$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) +\tag{3} +\end{equation} +$$ +

 
+ +

+determines the reward for the discriminator, while the generator gets the +conjugate reward + +

 
+$$ +\begin{equation} + -v(\theta^{(g)}, \theta^{(d)}) +\tag{4} +\end{equation} +$$ +

 
+ +

+During learning both of the networks maximize their own reward function, so that +the generator gets better and better at tricking the discriminator, while the +discriminator gets better and better at telling the difference between the fake +and real data. The generator and discriminator alternate on which one trains at +one time (i.e. for one epoch). In other words, we keep the generator constant +and train the discriminator, then we keep the discriminator constant to train +the generator and repeat. It is this back and forth dynamic which lets GANs +tackle otherwise intractable generative problems. As the generator improves with + training, the discriminator's performance gets worse because it cannot easily + tell the difference between real and fake. If the generator ends up succeeding + perfectly, the the discriminator will do no better than random guessing i.e. + 50\%. This progression in the training poses a problem for the convergence + criteria for GANs. The discriminator feedback gets less meaningful over time, + if we continue training after this point then the generator is effectively + training on junk data which can undo the learning up to that point. Therefore, + we stop training when the discriminator starts outputting \( 1/2 \) everywhere. + At convergence we have + +

 
+$$ +\begin{equation} + g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\tag{5} +\end{equation} +$$ +

 
+ +The default choice for \( v \) is +

 
+$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x) + + \mathbb{E}_{x\sim p_\mathrm{model}} + \log (1 - d(x)) +\tag{6} +\end{equation} +$$ +

 
+ +The main motivation for the design of GANs is that the learning process requires +neither approximate inference (variational autoencoders for example) nor +approximation of a partition function. In the case where +

 
+$$ +\begin{equation} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\tag{7} +\end{equation} +$$ +

 
+ +is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is +asymptotically consistent +( Seth Lloyd on QuGANs ). This is in +general not the case and it is possible to get situations where the training +process never converges because the generator and discriminator chase one +another around in the parameter space indefinitely. A much deeper discussion on +the currently open research problem of GAN convergence is available +here. To +anyone interested in learning more about GANs it is a highly recommended read. +Direct quote: "In this best-performing formulation, the generator aims to +increase the log probability that the discriminator makes a mistake, rather than +aiming to decrease the log probability that the discriminator makes the correct +prediction." Another interesting read + +

Writing Our First Generative Adversarial Network

+Let us now move on to actually implementing a GAN in tensorflow. We will study +the performance of our GAN on the MNIST dataset. This code is based on and +adapted from the +google tutorial + +

+First we import our libraries + +

+ + +

import os
+import time
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+from tensorflow.keras import layers
+from tensorflow.keras.utils import plot_model
+
+

+Next we define our hyperparameters and import our data the usual way + +

+ + +

BUFFER_SIZE = 60000
+BATCH_SIZE = 256
+EPOCHS = 30
+
+data = tf.keras.datasets.mnist.load_data()
+(train_images, train_labels), (test_images, test_labels) = data
+train_images = np.reshape(train_images, (train_images.shape[0],
+                                         28,
+                                         28,
+                                         1)).astype('float32')
+
+# we normalize between -1 and 1
+train_images = (train_images - 127.5) / 127.5
+training_dataset = tf.data.Dataset.from_tensor_slices(
+                      train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
+
+

+Let's have a quick look + +

+ + +

plt.imshow(train_images[0], cmap='Greys')
+plt.show()
+
+

+Now we define our two models. This is where the 'magic' happens. There are a +huge amount of possible formulations for both models. A lot of engineering and +trial and error can be done here to try to produce better performing models. For +more advanced GANs this is by far the step where you can 'make or break' a +model. + +

+We start with the generator. As stated in the introductory text the generator +\( g \) upsamples from a random sample to the shape of what we want to predict. In +our case we are trying to predict MNIST images (\( 28\times 28 \) pixels). + +

+ + +

def generator_model():
+    """
+    The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to
+    produce an image from a random seed. We start with a Dense layer taking this
+    random sample as an input and subsequently upsample through multiple
+    convolutional layers.
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+
+
+    # adding our input layer. Dense means that every neuron is connected and
+    # the input shape is the shape of our random noise. The units need to match
+    # in some sense the upsampling strides to reach our desired output shape.
+    # we are using 100 random numbers as our seed
+    model.add(layers.Dense(units=7*7*BATCH_SIZE,
+                           use_bias=False,
+                           input_shape=(100, )))
+    # we normalize the output form the Dense layer
+    model.add(layers.BatchNormalization())
+    # and add an activation function to our 'layer'. LeakyReLU avoids vanishing
+    # gradient problem
+    model.add(layers.LeakyReLU())
+    model.add(layers.Reshape((7, 7, BATCH_SIZE)))
+    assert model.output_shape == (None, 7, 7, BATCH_SIZE)
+    # even though we just added four keras layers we think of everything above
+    # as 'one' layer
+
+    # next we add our upscaling convolutional layers
+    model.add(layers.Conv2DTranspose(filters=128,
+                                     kernel_size=(5, 5),
+                                     strides=(1, 1),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 7, 7, 128)
+
+    model.add(layers.Conv2DTranspose(filters=64,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 14, 14, 64)
+
+    model.add(layers.Conv2DTranspose(filters=1,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False,
+                                     activation='tanh'))
+    assert model.output_shape == (None, 28, 28, 1)
+
+    return model
+
+

+And there we have our 'simple' generator model. Now we move on to defining our +discriminator model \( d \), which is a convolutional neural network based image +classifier. + +

+ + +

def discriminator_model():
+    """
+    The discriminator is a convolutional neural network based image classifier
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+    model.add(layers.Conv2D(filters=64,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same',
+                            input_shape=[28, 28, 1]))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+
+    model.add(layers.Conv2D(filters=128,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same'))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+    model.add(layers.Flatten())
+    model.add(layers.Dense(1))
+
+    return model
+
+

+Let us take a look at our models. Note: double click images for bigger view. + +

+ + +

generator = generator_model()
+plot_model(generator, show_shapes=True, rankdir='LR')
+
+

+ + +

discriminator = discriminator_model()
+plot_model(discriminator, show_shapes=True, rankdir='LR')
+
+

+Next we need a few helper objects we will use in training + +

+ + +

cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)
+generator_optimizer = tf.keras.optimizers.Adam(1e-4)
+discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)
+
+

+The first object, cross_entropy is our loss function and the two others are +our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This +is because they need to improve their accuracy at approximately equal speeds to +get convergence (not necessarily exactly equal). Now we define our loss +functions + +

+ + +

def generator_loss(fake_output):
+    loss = cross_entropy(tf.ones_like(fake_output), fake_output)
+
+    return loss
+
+

+ + +

def discriminator_loss(real_output, fake_output):
+    real_loss = cross_entropy(tf.ones_like(real_output), real_output)
+    fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
+    total_loss = real_loss + fake_loss
+
+    return total_loss
+
+

+Next we define a kind of seed to help us compare the learning process over +multiple training epochs. + +

+ + +

noise_dimension = 100
+n_examples_to_generate = 16
+seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
+
+

+Now we have everything we need to define our training step, which we will apply +for every step in our training loop. Notice the @tf.function flag signifying +that the function is tensorflow 'compiled'. Removing this flag doubles the +computation time. + +

+ + +

@tf.function
+def train_step(images):
+    noise = tf.random.normal([BATCH_SIZE, noise_dimension])
+
+    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
+        generated_images = generator(noise, training=True)
+
+        real_output = discriminator(images, training=True)
+        fake_output = discriminator(generated_images, training=True)
+
+        gen_loss = generator_loss(fake_output)
+        disc_loss = discriminator_loss(real_output, fake_output)
+
+    gradients_of_generator = gen_tape.gradient(gen_loss,
+                                            generator.trainable_variables)
+    gradients_of_discriminator = disc_tape.gradient(disc_loss,
+                                            discriminator.trainable_variables)
+    generator_optimizer.apply_gradients(zip(gradients_of_generator,
+                                            generator.trainable_variables))
+    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator,
+                                            discriminator.trainable_variables))
+
+    return gen_loss, disc_loss
+
+

+Next we define a helper function to produce an output over our training epochs +to see the predictive progression of our generator model. Note: I am including +this code here, but comment it out in the training loop. +

+ + +

def generate_and_save_images(model, epoch, test_input):
+    # we're making inferences here
+    predictions = model(test_input, training=False)
+
+    fig = plt.figure(figsize=(4, 4))
+
+    for i in range(predictions.shape[0]):
+        plt.subplot(4, 4, i+1)
+        plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')
+        plt.axis('off')
+
+    plt.savefig(f'./images_from_seed_images/image_at_epoch_{str(epoch).zfill(3)}.png')
+    plt.close()
+    #plt.show()
+
+

+Setting up checkpoints to periodically save our model during training so that +everything is not lost even if the program were to somehow terminate while +training. + +

+ + +

# Setting up checkpoints to save model during training
+checkpoint_dir = './training_checkpoints'
+checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')
+checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
+                            discriminator_optimizer=discriminator_optimizer,
+                            generator=generator,
+                            discriminator=discriminator)
+
+

+Now we define our training loop + +

+ + +

def train(dataset, epochs):
+    generator_loss_list = []
+    discriminator_loss_list = []
+
+    for epoch in range(epochs):
+        start = time.time()
+
+        for image_batch in dataset:
+            gen_loss, disc_loss = train_step(image_batch)
+            generator_loss_list.append(gen_loss.numpy())
+            discriminator_loss_list.append(disc_loss.numpy())
+
+        #generate_and_save_images(generator, epoch + 1, seed_images)
+
+        if (epoch + 1) % 15 == 0:
+            checkpoint.save(file_prefix=checkpoint_prefix)
+
+        print(f'Time for epoch {epoch} is {time.time() - start}')
+
+    #generate_and_save_images(generator, epochs, seed_images)
+
+    loss_file = './data/lossfile.txt'
+    with open(loss_file, 'w') as outfile:
+        outfile.write(str(generator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+        outfile.write(str(discriminator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+
+

+To train simply call this function. Warning: this might take a long time so +there is a folder of a pretrained network already included in the repository. + +

+ + +

train(train_dataset, EPOCHS)
+
+

+And here is the result of training our model for 100 epochs + +

+ + +

+ +

+Now to avoid having to train and everything, which will take a while depending +on your computer setup we now load in the model which produced the above gif. + +

+ + +

checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
+restored_generator = checkpoint.generator
+restored_discriminator = checkpoint.discriminator
+
+print(restored_generator)
+print(restored_discriminator)
+
+ +

Exploring the Latent Space

+ +

+So we have successfully loaded in our latest model. Let us now play around a bit +and see what kind of things we can learn about this model. Our generator takes +an array of 100 numbers. One idea can be to try to systematically change our +input. Let us try and see what we get + +

+ + +

def generate_latent_points(number=100, scale_means=1, scale_stds=1):
+    latent_dim = 100
+    means = scale_means * tf.linspace(-1, 1, num=latent_dim)
+    stds = scale_stds * tf.linspace(-1, 1, num=latent_dim)
+    latent_space_value_range = tf.random.normal([number, number],
+                                                means,
+                                                stds,
+                                                dtype=tf.float64)
+
+    return latent_space_value_range
+
+def generate_images(latent_points):
+    # notice we set training to false because we are making inferences
+    generated_images = restored_generator(latent_space_value_range,
+                                          training=False)
+
+    return generated_images
+
+

+ + +

def plot_result(generated_images, number):
+    # obviously this assumes sqrt number is an int
+    fig, axs = plt.subplots(int(np.sqrt(number)), int(np.sqrt(number)),
+                            figsize=(10, 10))
+
+    for i in range(int(np.sqrt(number))):
+        for j in range(int(np.sqrt(number))):
+            axs[i, j].imshow(generated_images[i*j], cmap='Greys')
+            axs[i, j].axis('off')
+
+    plt.show()
+
+

+ + +

generated_images = generate_images(generate_latent_points())
+plot_result(generated_images, number)
+
+

+Interesting! We see that the generator generates images that look like MNIST +numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able +to generate a similar plot where we generate every MNIST number. Let us now try +to 'move' a bit around in the latent space. Note: decrease the plot number if +these following cells take too long to run on your computer. + +

+ + +

plot_number = 225
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=-5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=5))
+plot_result(generated_images, plot_number)
+
+

+Again, we have found something interesting. Moving around using our means +takes us from digit to digit, while moving around using our standard +deviations seem to increase the number of different digits! In the last image +above, we can barely make out every MNIST digit. Let us make on last plot using +this information by upping the standard deviation of our Gaussian noises. + +

+ + +

plot_number = 400
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=10))
+
+

+A pretty cool result! We see that our generator indeed has learned a +distribution which qualitatively looks a whole lot like the MNIST dataset. + +

Interpolating Between MNIST Digits

+Another interesting way to explore the latent space of our generator model is by +interpolating between the MNIST digits. This section is largely based on +"this excellent blogpost": https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/ +by Jason Brownlee. + +

+So let us start +

+ + + +
+
+ + + + + + + + + + + + diff --git a/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-solarized.html b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-solarized.html new file mode 100644 index 000000000..b0dbacff8 --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks-solarized.html @@ -0,0 +1,676 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Generative Models

+Generative models describe a class of statistical models that are a contrast +to discriminative models. Informally we say that generative models can +generate new data instances while discriminative models discriminate between +different kinds of data instances. A generative model could generate new photos +of animals that look like 'real' animals while a discriminative model could tell +a dog from a cat. More formally, given a data set \( x \) and a set of labels / +targets \( y \). Generative models capture the joint probability \( p(x, y) \), or +just \( p(x) \) if there are no labels, while discriminative models capture the +conditional probability \( p(y | x) \). Discriminative models generally try to draw +boundaries in the data space (often high dimensional), while generative models +try to model how data is placed throughout the space. + +

Generative Adversarial Networks

+Generative Adversarial Networks are a type of unsupervised machine learning +algorithm proposed by Goodfellow et. al +in 2014 (Read the paper first it's only 6 pages). The simplest formulation of +the model is based on a game theoretic approach, zero sum game, where we pit +two neural networks against one another. We define two rival networks, one +generator \( g \), and one discriminator \( d \). The generator directly produces +samples +$$ +\begin{equation} + x = g(z; \theta^{(g)}) +\label{_auto1} +\end{equation} +$$ + +The discriminator attempts to distinguish between samples drawn from the +training data and samples drawn from the generator. In other words, it tries to +tell the difference between the fake data produced by \( g \) and the actual data +samples we want to do prediction on. The discriminator outputs a probability +value given by + +$$ +\begin{equation} + d(x; \theta^{(d)}) +\label{_auto2} +\end{equation} +$$ + +

+indicating the probability that \( x \) is a real training example rather than a +fake sample the generator has generated. The simplest way to formulate the +learning process in a generative adversarial network is a zero-sum game, in +which a function + +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) +\label{_auto3} +\end{equation} +$$ + +

+determines the reward for the discriminator, while the generator gets the +conjugate reward + +$$ +\begin{equation} + -v(\theta^{(g)}, \theta^{(d)}) +\label{_auto4} +\end{equation} +$$ + +

+During learning both of the networks maximize their own reward function, so that +the generator gets better and better at tricking the discriminator, while the +discriminator gets better and better at telling the difference between the fake +and real data. The generator and discriminator alternate on which one trains at +one time (i.e. for one epoch). In other words, we keep the generator constant +and train the discriminator, then we keep the discriminator constant to train +the generator and repeat. It is this back and forth dynamic which lets GANs +tackle otherwise intractable generative problems. As the generator improves with + training, the discriminator's performance gets worse because it cannot easily + tell the difference between real and fake. If the generator ends up succeeding + perfectly, the the discriminator will do no better than random guessing i.e. + 50\%. This progression in the training poses a problem for the convergence + criteria for GANs. The discriminator feedback gets less meaningful over time, + if we continue training after this point then the generator is effectively + training on junk data which can undo the learning up to that point. Therefore, + we stop training when the discriminator starts outputting \( 1/2 \) everywhere. + At convergence we have + +$$ +\begin{equation} + g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\label{_auto5} +\end{equation} +$$ + +The default choice for \( v \) is +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x) + + \mathbb{E}_{x\sim p_\mathrm{model}} + \log (1 - d(x)) +\label{_auto6} +\end{equation} +$$ + +The main motivation for the design of GANs is that the learning process requires +neither approximate inference (variational autoencoders for example) nor +approximation of a partition function. In the case where +$$ +\begin{equation} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\label{_auto7} +\end{equation} +$$ + +is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is +asymptotically consistent +( Seth Lloyd on QuGANs ). This is in +general not the case and it is possible to get situations where the training +process never converges because the generator and discriminator chase one +another around in the parameter space indefinitely. A much deeper discussion on +the currently open research problem of GAN convergence is available +here. To +anyone interested in learning more about GANs it is a highly recommended read. +Direct quote: "In this best-performing formulation, the generator aims to +increase the log probability that the discriminator makes a mistake, rather than +aiming to decrease the log probability that the discriminator makes the correct +prediction." Another interesting read + +

Writing Our First Generative Adversarial Network

+Let us now move on to actually implementing a GAN in tensorflow. We will study +the performance of our GAN on the MNIST dataset. This code is based on and +adapted from the +google tutorial + +

+First we import our libraries + +

+ + +

import os
+import time
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+from tensorflow.keras import layers
+from tensorflow.keras.utils import plot_model
+
+

+Next we define our hyperparameters and import our data the usual way + +

+ + +

BUFFER_SIZE = 60000
+BATCH_SIZE = 256
+EPOCHS = 30
+
+data = tf.keras.datasets.mnist.load_data()
+(train_images, train_labels), (test_images, test_labels) = data
+train_images = np.reshape(train_images, (train_images.shape[0],
+                                         28,
+                                         28,
+                                         1)).astype('float32')
+
+# we normalize between -1 and 1
+train_images = (train_images - 127.5) / 127.5
+training_dataset = tf.data.Dataset.from_tensor_slices(
+                      train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
+
+

+Let's have a quick look + +

+ + +

plt.imshow(train_images[0], cmap='Greys')
+plt.show()
+
+

+Now we define our two models. This is where the 'magic' happens. There are a +huge amount of possible formulations for both models. A lot of engineering and +trial and error can be done here to try to produce better performing models. For +more advanced GANs this is by far the step where you can 'make or break' a +model. + +

+We start with the generator. As stated in the introductory text the generator +\( g \) upsamples from a random sample to the shape of what we want to predict. In +our case we are trying to predict MNIST images (\( 28\times 28 \) pixels). + +

+ + +

def generator_model():
+    """
+    The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to
+    produce an image from a random seed. We start with a Dense layer taking this
+    random sample as an input and subsequently upsample through multiple
+    convolutional layers.
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+
+
+    # adding our input layer. Dense means that every neuron is connected and
+    # the input shape is the shape of our random noise. The units need to match
+    # in some sense the upsampling strides to reach our desired output shape.
+    # we are using 100 random numbers as our seed
+    model.add(layers.Dense(units=7*7*BATCH_SIZE,
+                           use_bias=False,
+                           input_shape=(100, )))
+    # we normalize the output form the Dense layer
+    model.add(layers.BatchNormalization())
+    # and add an activation function to our 'layer'. LeakyReLU avoids vanishing
+    # gradient problem
+    model.add(layers.LeakyReLU())
+    model.add(layers.Reshape((7, 7, BATCH_SIZE)))
+    assert model.output_shape == (None, 7, 7, BATCH_SIZE)
+    # even though we just added four keras layers we think of everything above
+    # as 'one' layer
+
+    # next we add our upscaling convolutional layers
+    model.add(layers.Conv2DTranspose(filters=128,
+                                     kernel_size=(5, 5),
+                                     strides=(1, 1),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 7, 7, 128)
+
+    model.add(layers.Conv2DTranspose(filters=64,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 14, 14, 64)
+
+    model.add(layers.Conv2DTranspose(filters=1,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False,
+                                     activation='tanh'))
+    assert model.output_shape == (None, 28, 28, 1)
+
+    return model
+
+

+And there we have our 'simple' generator model. Now we move on to defining our +discriminator model \( d \), which is a convolutional neural network based image +classifier. + +

+ + +

def discriminator_model():
+    """
+    The discriminator is a convolutional neural network based image classifier
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+    model.add(layers.Conv2D(filters=64,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same',
+                            input_shape=[28, 28, 1]))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+
+    model.add(layers.Conv2D(filters=128,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same'))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+    model.add(layers.Flatten())
+    model.add(layers.Dense(1))
+
+    return model
+
+

+Let us take a look at our models. Note: double click images for bigger view. + +

+ + +

generator = generator_model()
+plot_model(generator, show_shapes=True, rankdir='LR')
+
+

+ + +

discriminator = discriminator_model()
+plot_model(discriminator, show_shapes=True, rankdir='LR')
+
+

+Next we need a few helper objects we will use in training + +

+ + +

cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)
+generator_optimizer = tf.keras.optimizers.Adam(1e-4)
+discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)
+
+

+The first object, cross_entropy is our loss function and the two others are +our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This +is because they need to improve their accuracy at approximately equal speeds to +get convergence (not necessarily exactly equal). Now we define our loss +functions + +

+ + +

def generator_loss(fake_output):
+    loss = cross_entropy(tf.ones_like(fake_output), fake_output)
+
+    return loss
+
+

+ + +

def discriminator_loss(real_output, fake_output):
+    real_loss = cross_entropy(tf.ones_like(real_output), real_output)
+    fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
+    total_loss = real_loss + fake_loss
+
+    return total_loss
+
+

+Next we define a kind of seed to help us compare the learning process over +multiple training epochs. + +

+ + +

noise_dimension = 100
+n_examples_to_generate = 16
+seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
+
+

+Now we have everything we need to define our training step, which we will apply +for every step in our training loop. Notice the @tf.function flag signifying +that the function is tensorflow 'compiled'. Removing this flag doubles the +computation time. + +

+ + +

@tf.function
+def train_step(images):
+    noise = tf.random.normal([BATCH_SIZE, noise_dimension])
+
+    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
+        generated_images = generator(noise, training=True)
+
+        real_output = discriminator(images, training=True)
+        fake_output = discriminator(generated_images, training=True)
+
+        gen_loss = generator_loss(fake_output)
+        disc_loss = discriminator_loss(real_output, fake_output)
+
+    gradients_of_generator = gen_tape.gradient(gen_loss,
+                                            generator.trainable_variables)
+    gradients_of_discriminator = disc_tape.gradient(disc_loss,
+                                            discriminator.trainable_variables)
+    generator_optimizer.apply_gradients(zip(gradients_of_generator,
+                                            generator.trainable_variables))
+    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator,
+                                            discriminator.trainable_variables))
+
+    return gen_loss, disc_loss
+
+

+Next we define a helper function to produce an output over our training epochs +to see the predictive progression of our generator model. Note: I am including +this code here, but comment it out in the training loop. +

+ + +

def generate_and_save_images(model, epoch, test_input):
+    # we're making inferences here
+    predictions = model(test_input, training=False)
+
+    fig = plt.figure(figsize=(4, 4))
+
+    for i in range(predictions.shape[0]):
+        plt.subplot(4, 4, i+1)
+        plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')
+        plt.axis('off')
+
+    plt.savefig(f'./images_from_seed_images/image_at_epoch_{str(epoch).zfill(3)}.png')
+    plt.close()
+    #plt.show()
+
+

+Setting up checkpoints to periodically save our model during training so that +everything is not lost even if the program were to somehow terminate while +training. + +

+ + +

# Setting up checkpoints to save model during training
+checkpoint_dir = './training_checkpoints'
+checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')
+checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
+                            discriminator_optimizer=discriminator_optimizer,
+                            generator=generator,
+                            discriminator=discriminator)
+
+

+Now we define our training loop + +

+ + +

def train(dataset, epochs):
+    generator_loss_list = []
+    discriminator_loss_list = []
+
+    for epoch in range(epochs):
+        start = time.time()
+
+        for image_batch in dataset:
+            gen_loss, disc_loss = train_step(image_batch)
+            generator_loss_list.append(gen_loss.numpy())
+            discriminator_loss_list.append(disc_loss.numpy())
+
+        #generate_and_save_images(generator, epoch + 1, seed_images)
+
+        if (epoch + 1) % 15 == 0:
+            checkpoint.save(file_prefix=checkpoint_prefix)
+
+        print(f'Time for epoch {epoch} is {time.time() - start}')
+
+    #generate_and_save_images(generator, epochs, seed_images)
+
+    loss_file = './data/lossfile.txt'
+    with open(loss_file, 'w') as outfile:
+        outfile.write(str(generator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+        outfile.write(str(discriminator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+
+

+To train simply call this function. Warning: this might take a long time so +there is a folder of a pretrained network already included in the repository. + +

+ + +

train(train_dataset, EPOCHS)
+
+

+And here is the result of training our model for 100 epochs + +

+ + +

+ +

+Now to avoid having to train and everything, which will take a while depending +on your computer setup we now load in the model which produced the above gif. + +

+ + +

checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
+restored_generator = checkpoint.generator
+restored_discriminator = checkpoint.discriminator
+
+print(restored_generator)
+print(restored_discriminator)
+
+ +

Exploring the Latent Space

+ +

+So we have successfully loaded in our latest model. Let us now play around a bit +and see what kind of things we can learn about this model. Our generator takes +an array of 100 numbers. One idea can be to try to systematically change our +input. Let us try and see what we get + +

+ + +

def generate_latent_points(number=100, scale_means=1, scale_stds=1):
+    latent_dim = 100
+    means = scale_means * tf.linspace(-1, 1, num=latent_dim)
+    stds = scale_stds * tf.linspace(-1, 1, num=latent_dim)
+    latent_space_value_range = tf.random.normal([number, number],
+                                                means,
+                                                stds,
+                                                dtype=tf.float64)
+
+    return latent_space_value_range
+
+def generate_images(latent_points):
+    # notice we set training to false because we are making inferences
+    generated_images = restored_generator(latent_space_value_range,
+                                          training=False)
+
+    return generated_images
+
+

+ + +

def plot_result(generated_images, number):
+    # obviously this assumes sqrt number is an int
+    fig, axs = plt.subplots(int(np.sqrt(number)), int(np.sqrt(number)),
+                            figsize=(10, 10))
+
+    for i in range(int(np.sqrt(number))):
+        for j in range(int(np.sqrt(number))):
+            axs[i, j].imshow(generated_images[i*j], cmap='Greys')
+            axs[i, j].axis('off')
+
+    plt.show()
+
+

+ + +

generated_images = generate_images(generate_latent_points())
+plot_result(generated_images, number)
+
+

+Interesting! We see that the generator generates images that look like MNIST +numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able +to generate a similar plot where we generate every MNIST number. Let us now try +to 'move' a bit around in the latent space. Note: decrease the plot number if +these following cells take too long to run on your computer. + +

+ + +

plot_number = 225
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=-5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=5))
+plot_result(generated_images, plot_number)
+
+

+Again, we have found something interesting. Moving around using our means +takes us from digit to digit, while moving around using our standard +deviations seem to increase the number of different digits! In the last image +above, we can barely make out every MNIST digit. Let us make on last plot using +this information by upping the standard deviation of our Gaussian noises. + +

+ + +

plot_number = 400
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=10))
+
+

+A pretty cool result! We see that our generator indeed has learned a +distribution which qualitatively looks a whole lot like the MNIST dataset. + +

Interpolating Between MNIST Digits

+Another interesting way to explore the latent space of our generator model is by +interpolating between the MNIST digits. This section is largely based on +"this excellent blogpost": https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/ +by Jason Brownlee. + +

+So let us start + + + + + + + + diff --git a/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks.html b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks.html new file mode 100644 index 000000000..c709845aa --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/GenerativeAdversarialNetworks.html @@ -0,0 +1,681 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Generative Models

+Generative models describe a class of statistical models that are a contrast +to discriminative models. Informally we say that generative models can +generate new data instances while discriminative models discriminate between +different kinds of data instances. A generative model could generate new photos +of animals that look like 'real' animals while a discriminative model could tell +a dog from a cat. More formally, given a data set \( x \) and a set of labels / +targets \( y \). Generative models capture the joint probability \( p(x, y) \), or +just \( p(x) \) if there are no labels, while discriminative models capture the +conditional probability \( p(y | x) \). Discriminative models generally try to draw +boundaries in the data space (often high dimensional), while generative models +try to model how data is placed throughout the space. + +

Generative Adversarial Networks

+Generative Adversarial Networks are a type of unsupervised machine learning +algorithm proposed by Goodfellow et. al +in 2014 (Read the paper first it's only 6 pages). The simplest formulation of +the model is based on a game theoretic approach, zero sum game, where we pit +two neural networks against one another. We define two rival networks, one +generator \( g \), and one discriminator \( d \). The generator directly produces +samples +$$ +\begin{equation} + x = g(z; \theta^{(g)}) +\label{_auto1} +\end{equation} +$$ + +The discriminator attempts to distinguish between samples drawn from the +training data and samples drawn from the generator. In other words, it tries to +tell the difference between the fake data produced by \( g \) and the actual data +samples we want to do prediction on. The discriminator outputs a probability +value given by + +$$ +\begin{equation} + d(x; \theta^{(d)}) +\label{_auto2} +\end{equation} +$$ + +

+indicating the probability that \( x \) is a real training example rather than a +fake sample the generator has generated. The simplest way to formulate the +learning process in a generative adversarial network is a zero-sum game, in +which a function + +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) +\label{_auto3} +\end{equation} +$$ + +

+determines the reward for the discriminator, while the generator gets the +conjugate reward + +$$ +\begin{equation} + -v(\theta^{(g)}, \theta^{(d)}) +\label{_auto4} +\end{equation} +$$ + +

+During learning both of the networks maximize their own reward function, so that +the generator gets better and better at tricking the discriminator, while the +discriminator gets better and better at telling the difference between the fake +and real data. The generator and discriminator alternate on which one trains at +one time (i.e. for one epoch). In other words, we keep the generator constant +and train the discriminator, then we keep the discriminator constant to train +the generator and repeat. It is this back and forth dynamic which lets GANs +tackle otherwise intractable generative problems. As the generator improves with + training, the discriminator's performance gets worse because it cannot easily + tell the difference between real and fake. If the generator ends up succeeding + perfectly, the the discriminator will do no better than random guessing i.e. + 50\%. This progression in the training poses a problem for the convergence + criteria for GANs. The discriminator feedback gets less meaningful over time, + if we continue training after this point then the generator is effectively + training on junk data which can undo the learning up to that point. Therefore, + we stop training when the discriminator starts outputting \( 1/2 \) everywhere. + At convergence we have + +$$ +\begin{equation} + g^* = \underset{g}{\mathrm{argmin}}\hspace{2pt} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\label{_auto5} +\end{equation} +$$ + +The default choice for \( v \) is +$$ +\begin{equation} + v(\theta^{(g)}, \theta^{(d)}) = \mathbb{E}_{x\sim p_\mathrm{data}}\log d(x) + + \mathbb{E}_{x\sim p_\mathrm{model}} + \log (1 - d(x)) +\label{_auto6} +\end{equation} +$$ + +The main motivation for the design of GANs is that the learning process requires +neither approximate inference (variational autoencoders for example) nor +approximation of a partition function. In the case where +$$ +\begin{equation} + \underset{d}{\mathrm{max}}v(\theta^{(g)}, \theta^{(d)}) +\label{_auto7} +\end{equation} +$$ + +is convex in $\theta^{(g)} then the procedure is guaranteed to converge and is +asymptotically consistent +( Seth Lloyd on QuGANs ). This is in +general not the case and it is possible to get situations where the training +process never converges because the generator and discriminator chase one +another around in the parameter space indefinitely. A much deeper discussion on +the currently open research problem of GAN convergence is available +here. To +anyone interested in learning more about GANs it is a highly recommended read. +Direct quote: "In this best-performing formulation, the generator aims to +increase the log probability that the discriminator makes a mistake, rather than +aiming to decrease the log probability that the discriminator makes the correct +prediction." Another interesting read + +

Writing Our First Generative Adversarial Network

+Let us now move on to actually implementing a GAN in tensorflow. We will study +the performance of our GAN on the MNIST dataset. This code is based on and +adapted from the +google tutorial + +

+First we import our libraries + +

+ + +

import os
+import time
+import numpy as np
+import tensorflow as tf
+import matplotlib.pyplot as plt
+from tensorflow.keras import layers
+from tensorflow.keras.utils import plot_model
+
+

+Next we define our hyperparameters and import our data the usual way + +

+ + +

BUFFER_SIZE = 60000
+BATCH_SIZE = 256
+EPOCHS = 30
+
+data = tf.keras.datasets.mnist.load_data()
+(train_images, train_labels), (test_images, test_labels) = data
+train_images = np.reshape(train_images, (train_images.shape[0],
+                                         28,
+                                         28,
+                                         1)).astype('float32')
+
+# we normalize between -1 and 1
+train_images = (train_images - 127.5) / 127.5
+training_dataset = tf.data.Dataset.from_tensor_slices(
+                      train_images).shuffle(BUFFER_SIZE).batch(BATCH_SIZE)
+
+

+Let's have a quick look + +

+ + +

plt.imshow(train_images[0], cmap='Greys')
+plt.show()
+
+

+Now we define our two models. This is where the 'magic' happens. There are a +huge amount of possible formulations for both models. A lot of engineering and +trial and error can be done here to try to produce better performing models. For +more advanced GANs this is by far the step where you can 'make or break' a +model. + +

+We start with the generator. As stated in the introductory text the generator +\( g \) upsamples from a random sample to the shape of what we want to predict. In +our case we are trying to predict MNIST images (\( 28\times 28 \) pixels). + +

+ + +

def generator_model():
+    """
+    The generator uses upsampling layers tf.keras.layers.Conv2DTranspose() to
+    produce an image from a random seed. We start with a Dense layer taking this
+    random sample as an input and subsequently upsample through multiple
+    convolutional layers.
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+
+
+    # adding our input layer. Dense means that every neuron is connected and
+    # the input shape is the shape of our random noise. The units need to match
+    # in some sense the upsampling strides to reach our desired output shape.
+    # we are using 100 random numbers as our seed
+    model.add(layers.Dense(units=7*7*BATCH_SIZE,
+                           use_bias=False,
+                           input_shape=(100, )))
+    # we normalize the output form the Dense layer
+    model.add(layers.BatchNormalization())
+    # and add an activation function to our 'layer'. LeakyReLU avoids vanishing
+    # gradient problem
+    model.add(layers.LeakyReLU())
+    model.add(layers.Reshape((7, 7, BATCH_SIZE)))
+    assert model.output_shape == (None, 7, 7, BATCH_SIZE)
+    # even though we just added four keras layers we think of everything above
+    # as 'one' layer
+
+    # next we add our upscaling convolutional layers
+    model.add(layers.Conv2DTranspose(filters=128,
+                                     kernel_size=(5, 5),
+                                     strides=(1, 1),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 7, 7, 128)
+
+    model.add(layers.Conv2DTranspose(filters=64,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False))
+    model.add(layers.BatchNormalization())
+    model.add(layers.LeakyReLU())
+    assert model.output_shape == (None, 14, 14, 64)
+
+    model.add(layers.Conv2DTranspose(filters=1,
+                                     kernel_size=(5, 5),
+                                     strides=(2, 2),
+                                     padding='same',
+                                     use_bias=False,
+                                     activation='tanh'))
+    assert model.output_shape == (None, 28, 28, 1)
+
+    return model
+
+

+And there we have our 'simple' generator model. Now we move on to defining our +discriminator model \( d \), which is a convolutional neural network based image +classifier. + +

+ + +

def discriminator_model():
+    """
+    The discriminator is a convolutional neural network based image classifier
+    """
+
+    # we define our model
+    model = tf.keras.Sequential()
+    model.add(layers.Conv2D(filters=64,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same',
+                            input_shape=[28, 28, 1]))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+
+    model.add(layers.Conv2D(filters=128,
+                            kernel_size=(5, 5),
+                            strides=(2, 2),
+                            padding='same'))
+    model.add(layers.LeakyReLU())
+    # adding a dropout layer as you do in conv-nets
+    model.add(layers.Dropout(0.3))
+
+    model.add(layers.Flatten())
+    model.add(layers.Dense(1))
+
+    return model
+
+

+Let us take a look at our models. Note: double click images for bigger view. + +

+ + +

generator = generator_model()
+plot_model(generator, show_shapes=True, rankdir='LR')
+
+

+ + +

discriminator = discriminator_model()
+plot_model(discriminator, show_shapes=True, rankdir='LR')
+
+

+Next we need a few helper objects we will use in training + +

+ + +

cross_entropy = tf.keras.losses.BinaryCrossentropy(from_logits=True)
+generator_optimizer = tf.keras.optimizers.Adam(1e-4)
+discriminator_optimizer = tf.keras.optimizers.Adam(1e-4)
+
+

+The first object, cross_entropy is our loss function and the two others are +our optimizers. Notice we use the same learning rate for both \( g \) and \( d \). This +is because they need to improve their accuracy at approximately equal speeds to +get convergence (not necessarily exactly equal). Now we define our loss +functions + +

+ + +

def generator_loss(fake_output):
+    loss = cross_entropy(tf.ones_like(fake_output), fake_output)
+
+    return loss
+
+

+ + +

def discriminator_loss(real_output, fake_output):
+    real_loss = cross_entropy(tf.ones_like(real_output), real_output)
+    fake_loss = cross_entropy(tf.zeros_liks(fake_output), fake_output)
+    total_loss = real_loss + fake_loss
+
+    return total_loss
+
+

+Next we define a kind of seed to help us compare the learning process over +multiple training epochs. + +

+ + +

noise_dimension = 100
+n_examples_to_generate = 16
+seed_images = tf.random.normal([n_examples_to_generate, noise_dimension])
+
+

+Now we have everything we need to define our training step, which we will apply +for every step in our training loop. Notice the @tf.function flag signifying +that the function is tensorflow 'compiled'. Removing this flag doubles the +computation time. + +

+ + +

@tf.function
+def train_step(images):
+    noise = tf.random.normal([BATCH_SIZE, noise_dimension])
+
+    with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
+        generated_images = generator(noise, training=True)
+
+        real_output = discriminator(images, training=True)
+        fake_output = discriminator(generated_images, training=True)
+
+        gen_loss = generator_loss(fake_output)
+        disc_loss = discriminator_loss(real_output, fake_output)
+
+    gradients_of_generator = gen_tape.gradient(gen_loss,
+                                            generator.trainable_variables)
+    gradients_of_discriminator = disc_tape.gradient(disc_loss,
+                                            discriminator.trainable_variables)
+    generator_optimizer.apply_gradients(zip(gradients_of_generator,
+                                            generator.trainable_variables))
+    discriminator_optimizer.apply_gradients(zip(gradients_of_discriminator,
+                                            discriminator.trainable_variables))
+
+    return gen_loss, disc_loss
+
+

+Next we define a helper function to produce an output over our training epochs +to see the predictive progression of our generator model. Note: I am including +this code here, but comment it out in the training loop. +

+ + +

def generate_and_save_images(model, epoch, test_input):
+    # we're making inferences here
+    predictions = model(test_input, training=False)
+
+    fig = plt.figure(figsize=(4, 4))
+
+    for i in range(predictions.shape[0]):
+        plt.subplot(4, 4, i+1)
+        plt.imshow(predictions[i, :, :, 0] * 127.5 + 127.5, cmap='gray')
+        plt.axis('off')
+
+    plt.savefig(f'./images_from_seed_images/image_at_epoch_{str(epoch).zfill(3)}.png')
+    plt.close()
+    #plt.show()
+
+

+Setting up checkpoints to periodically save our model during training so that +everything is not lost even if the program were to somehow terminate while +training. + +

+ + +

# Setting up checkpoints to save model during training
+checkpoint_dir = './training_checkpoints'
+checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')
+checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,
+                            discriminator_optimizer=discriminator_optimizer,
+                            generator=generator,
+                            discriminator=discriminator)
+
+

+Now we define our training loop + +

+ + +

def train(dataset, epochs):
+    generator_loss_list = []
+    discriminator_loss_list = []
+
+    for epoch in range(epochs):
+        start = time.time()
+
+        for image_batch in dataset:
+            gen_loss, disc_loss = train_step(image_batch)
+            generator_loss_list.append(gen_loss.numpy())
+            discriminator_loss_list.append(disc_loss.numpy())
+
+        #generate_and_save_images(generator, epoch + 1, seed_images)
+
+        if (epoch + 1) % 15 == 0:
+            checkpoint.save(file_prefix=checkpoint_prefix)
+
+        print(f'Time for epoch {epoch} is {time.time() - start}')
+
+    #generate_and_save_images(generator, epochs, seed_images)
+
+    loss_file = './data/lossfile.txt'
+    with open(loss_file, 'w') as outfile:
+        outfile.write(str(generator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+        outfile.write(str(discriminator_loss_list))
+        outfile.write('\n')
+        outfile.write('\n')
+
+

+To train simply call this function. Warning: this might take a long time so +there is a folder of a pretrained network already included in the repository. + +

+ + +

train(train_dataset, EPOCHS)
+
+

+And here is the result of training our model for 100 epochs + +

+ + +

+ +

+Now to avoid having to train and everything, which will take a while depending +on your computer setup we now load in the model which produced the above gif. + +

+ + +

checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))
+restored_generator = checkpoint.generator
+restored_discriminator = checkpoint.discriminator
+
+print(restored_generator)
+print(restored_discriminator)
+
+ +

Exploring the Latent Space

+ +

+So we have successfully loaded in our latest model. Let us now play around a bit +and see what kind of things we can learn about this model. Our generator takes +an array of 100 numbers. One idea can be to try to systematically change our +input. Let us try and see what we get + +

+ + +

def generate_latent_points(number=100, scale_means=1, scale_stds=1):
+    latent_dim = 100
+    means = scale_means * tf.linspace(-1, 1, num=latent_dim)
+    stds = scale_stds * tf.linspace(-1, 1, num=latent_dim)
+    latent_space_value_range = tf.random.normal([number, number],
+                                                means,
+                                                stds,
+                                                dtype=tf.float64)
+
+    return latent_space_value_range
+
+def generate_images(latent_points):
+    # notice we set training to false because we are making inferences
+    generated_images = restored_generator(latent_space_value_range,
+                                          training=False)
+
+    return generated_images
+
+

+ + +

def plot_result(generated_images, number):
+    # obviously this assumes sqrt number is an int
+    fig, axs = plt.subplots(int(np.sqrt(number)), int(np.sqrt(number)),
+                            figsize=(10, 10))
+
+    for i in range(int(np.sqrt(number))):
+        for j in range(int(np.sqrt(number))):
+            axs[i, j].imshow(generated_images[i*j], cmap='Greys')
+            axs[i, j].axis('off')
+
+    plt.show()
+
+

+ + +

generated_images = generate_images(generate_latent_points())
+plot_result(generated_images, number)
+
+

+Interesting! We see that the generator generates images that look like MNIST +numbers: \( 1, 4, 7, 9 \). Let's try to tweak it a bit more to see if we are able +to generate a similar plot where we generate every MNIST number. Let us now try +to 'move' a bit around in the latent space. Note: decrease the plot number if +these following cells take too long to run on your computer. + +

+ + +

plot_number = 225
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=-5,
+                                                          scale_stds=1))
+plot_result(generated_images, plot_number)
+
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=5))
+plot_result(generated_images, plot_number)
+
+

+Again, we have found something interesting. Moving around using our means +takes us from digit to digit, while moving around using our standard +deviations seem to increase the number of different digits! In the last image +above, we can barely make out every MNIST digit. Let us make on last plot using +this information by upping the standard deviation of our Gaussian noises. + +

+ + +

plot_number = 400
+generated_images = generate_images(generate_latent_points(number=plot_number,
+                                                          scale_means=1,
+                                                          scale_stds=10))
+
+

+A pretty cool result! We see that our generator indeed has learned a +distribution which qualitatively looks a whole lot like the MNIST dataset. + +

Interpolating Between MNIST Digits

+Another interesting way to explore the latent space of our generator model is by +interpolating between the MNIST digits. This section is largely based on +"this excellent blogpost": https://machinelearningmastery.com/how-to-interpolate-and-perform-vector-arithmetic-with-faces-using-a-generative-adversarial-network/ +by Jason Brownlee. + +

+So let us start + + + + + + + + diff --git a/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.gitignore b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.gitignore new file mode 100644 index 000000000..a5df3133d --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +.svn +log/*.log +tmp/** +node_modules/ +.sass-cache +css/reveal.min.css +js/reveal.min.js diff --git a/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.travis.yml b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.travis.yml new file mode 100644 index 000000000..165d9ae9f --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/.travis.yml @@ -0,0 +1,5 @@ +language: node_js +node_js: + - 0.10 +before_script: + - npm install -g grunt-cli \ No newline at end of file diff --git a/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/CONTRIBUTING.md b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/CONTRIBUTING.md new file mode 100644 index 000000000..c2091e88f --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/CONTRIBUTING.md @@ -0,0 +1,23 @@ +## Contributing + +Please keep the [issue tracker](http://github.com/hakimel/reveal.js/issues) limited to **bug reports**, **feature requests** and **pull requests**. + + +### Personal Support +If you have personal support or setup questions the best place to ask those are [StackOverflow](http://stackoverflow.com/questions/tagged/reveal.js). + + +### Bug Reports +When reporting a bug make sure to include information about which browser and operating system you are on as well as the necessary steps to reproduce the issue. If possible please include a link to a sample presentation where the bug can be tested. + + +### Pull Requests +- Should follow the coding style of the file you work in, most importantly: + - Tabs to indent + - Single-quoted strings +- Should be made towards the **dev branch** +- Should be submitted from a feature/topic branch (not your master) + + +### Plugins +Please do not submit plugins as pull requests. They should be maintained in their own separate repository. More information here: https://github.com/hakimel/reveal.js/wiki/Plugin-Guidelines diff --git a/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/Gruntfile.js b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/Gruntfile.js new file mode 100644 index 000000000..b257e8f32 --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/Gruntfile.js @@ -0,0 +1,140 @@ +/* global module:false */ +module.exports = function(grunt) { + var port = grunt.option('port') || 8000; + // Project configuration + grunt.initConfig({ + pkg: grunt.file.readJSON('package.json'), + meta: { + banner: + '/*!\n' + + ' * reveal.js <%= pkg.version %> (<%= grunt.template.today("yyyy-mm-dd, HH:MM") %>)\n' + + ' * http://lab.hakim.se/reveal-js\n' + + ' * MIT licensed\n' + + ' *\n' + + ' * Copyright (C) 2014 Hakim El Hattab, http://hakim.se\n' + + ' */' + }, + + qunit: { + files: [ 'test/*.html' ] + }, + + uglify: { + options: { + banner: '<%= meta.banner %>\n' + }, + build: { + src: 'js/reveal.js', + dest: 'js/reveal.min.js' + } + }, + + cssmin: { + compress: { + files: { + 'css/reveal.min.css': [ 'css/reveal.css' ] + } + } + }, + + sass: { + main: { + files: { + 'css/theme/darkgray.css': 'css/theme/source/darkgray.scss', + 'css/theme/beigesmall.css': 'css/theme/source/beigesmall.scss', + 'css/theme/cbc.css': 'css/theme/source/cbc.scss', + 'css/theme/default.css': 'css/theme/source/default.scss', + 'css/theme/beige.css': 'css/theme/source/beige.scss', + 'css/theme/night.css': 'css/theme/source/night.scss', + 'css/theme/serif.css': 'css/theme/source/serif.scss', + 'css/theme/simple.css': 'css/theme/source/simple.scss', + 'css/theme/sky.css': 'css/theme/source/sky.scss', + 'css/theme/moon.css': 'css/theme/source/moon.scss', + 'css/theme/solarized.css': 'css/theme/source/solarized.scss', + 'css/theme/blood.css': 'css/theme/source/blood.scss' + } + } + }, + + jshint: { + options: { + curly: false, + eqeqeq: true, + immed: true, + latedef: true, + newcap: true, + noarg: true, + sub: true, + undef: true, + eqnull: true, + browser: true, + expr: true, + globals: { + head: false, + module: false, + console: false, + unescape: false + } + }, + files: [ 'Gruntfile.js', 'js/reveal.js' ] + }, + + connect: { + server: { + options: { + port: port, + base: '.' + } + } + }, + + zip: { + 'reveal-js-presentation.zip': [ + 'index.html', + 'css/**', + 'js/**', + 'lib/**', + 'images/**', + 'plugin/**' + ] + }, + + watch: { + main: { + files: [ 'Gruntfile.js', 'js/reveal.js', 'css/reveal.css' ], + tasks: 'default' + }, + theme: { + files: [ 'css/theme/source/*.scss', 'css/theme/template/*.scss' ], + tasks: 'themes' + } + } + + }); + + // Dependencies + grunt.loadNpmTasks( 'grunt-contrib-qunit' ); + grunt.loadNpmTasks( 'grunt-contrib-jshint' ); + grunt.loadNpmTasks( 'grunt-contrib-cssmin' ); + grunt.loadNpmTasks( 'grunt-contrib-uglify' ); + grunt.loadNpmTasks( 'grunt-contrib-watch' ); + grunt.loadNpmTasks( 'grunt-contrib-sass' ); + grunt.loadNpmTasks( 'grunt-contrib-connect' ); + grunt.loadNpmTasks( 'grunt-zip' ); + + // Default task + grunt.registerTask( 'default', [ 'jshint', 'cssmin', 'uglify', 'qunit' ] ); + + // Theme task + grunt.registerTask( 'themes', [ 'sass' ] ); + + // Package presentation to archive + grunt.registerTask( 'package', [ 'default', 'zip' ] ); + + // Serve presentation locally + grunt.registerTask( 'serve', [ 'connect', 'watch' ] ); + + // Run tests + grunt.registerTask( 'test', [ 'jshint', 'qunit' ] ); + +}; diff --git a/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/LICENSE b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/LICENSE new file mode 100644 index 000000000..09623076f --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2015 Hakim El Hattab, http://hakim.se + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/README.md b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/README.md new file mode 100644 index 000000000..573b19597 --- /dev/null +++ b/doc/pub/GenerativeAdversarialNetworks/html/reveal.js/README.md @@ -0,0 +1,1052 @@ +# reveal.js [![Build Status](https://travis-ci.org/hakimel/reveal.js.svg?branch=master)](https://travis-ci.org/hakimel/reveal.js) + +A framework for easily creating beautiful presentations using HTML. [Check out the live demo](http://lab.hakim.se/reveal-js/). + +reveal.js comes with a broad range of features including [nested slides](https://github.com/hakimel/reveal.js#markup), [Markdown contents](https://github.com/hakimel/reveal.js#markdown), [PDF export](https://github.com/hakimel/reveal.js#pdf-export), [speaker notes](https://github.com/hakimel/reveal.js#speaker-notes) and a [JavaScript API](https://github.com/hakimel/reveal.js#api). It's best viewed in a modern browser but [fallbacks](https://github.com/hakimel/reveal.js/wiki/Browser-Support) are available to make sure your presentation can still be viewed elsewhere. + + +#### More reading: +- [Installation](#installation): Step-by-step instructions for getting reveal.js running on your computer. +- [Changelog](https://github.com/hakimel/reveal.js/releases): Up-to-date version history. +- [Examples](https://github.com/hakimel/reveal.js/wiki/Example-Presentations): Presentations created with reveal.js, add your own! +- [Browser Support](https://github.com/hakimel/reveal.js/wiki/Browser-Support): Explanation of browser support and fallbacks. +- [Plugins](https://github.com/hakimel/reveal.js/wiki/Plugins,-Tools-and-Hardware): A list of plugins that can be used to extend reveal.js. + +## Online Editor + +Presentations are written using HTML or Markdown but there's also an online editor for those of you who prefer a graphical interface. Give it a try at [http://slides.com](http://slides.com). + + +## Instructions + +### Markup + +Markup hierarchy needs to be ``

`` where the ``
`` represents one slide and can be repeated indefinitely. If you place multiple ``
``'s inside of another ``
`` they will be shown as vertical slides. The first of the vertical slides is the "root" of the others (at the top), and it will be included in the horizontal sequence. For example: + +```html +
+
+
Single Horizontal Slide
+
+
Vertical Slide 1
+
Vertical Slide 2
+
+
+
+``` + +### Markdown + +It's possible to write your slides using Markdown. To enable Markdown, add the ```data-markdown``` attribute to your ```
``` elements and wrap the contents in a ``` +
+``` + +#### External Markdown + +You can write your content as a separate file and have reveal.js load it at runtime. Note the separator arguments which determine how slides are delimited in the external file. The ```data-charset``` attribute is optional and specifies which charset to use when loading the external file. + +When used locally, this feature requires that reveal.js [runs from a local web server](#full-setup). + +```html +
+
+``` + +#### Element Attributes + +Special syntax (in html comment) is available for adding attributes to Markdown elements. This is useful for fragments, amongst other things. + +```html +
+ +
+``` + +#### Slide Attributes + +Special syntax (in html comment) is available for adding attributes to the slide `
` elements generated by your Markdown. + +```html +
+ +
+``` + + +### Configuration + +At the end of your page you need to initialize reveal by running the following code. Note that all config values are optional and will default as specified below. + +```javascript +Reveal.initialize({ + + // Display controls in the bottom right corner + controls: true, + + // Display a presentation progress bar + progress: true, + + // Display the page number of the current slide + slideNumber: false, + + // Push each slide change to the browser history + history: false, + + // Enable keyboard shortcuts for navigation + keyboard: true, + + // Enable the slide overview mode + overview: true, + + // Vertical centering of slides + center: true, + + // Enables touch navigation on devices with touch input + touch: true, + + // Loop the presentation + loop: false, + + // Change the presentation direction to be RTL + rtl: false, + + // Turns fragments on and off globally + fragments: true, + + // Flags if the presentation is running in an embedded mode, + // i.e. contained within a limited portion of the screen + embedded: false, + + // Flags if we should show a help overlay when the questionmark + // key is pressed + help: true, + + // Number of milliseconds between automatically proceeding to the + // next slide, disabled when set to 0, this value can be overwritten + // by using a data-autoslide attribute on your slides + autoSlide: 0, + + // Stop auto-sliding after user input + autoSlideStoppable: true, + + // Enable slide navigation via mouse wheel + mouseWheel: false, + + // Hides the address bar on mobile devices + hideAddressBar: true, + + // Opens links in an iframe preview overlay + previewLinks: false, + + // Transition style + transition: 'default', // none/fade/slide/convex/concave/zoom + + // Transition speed + transitionSpeed: 'default', // default/fast/slow + + // Transition style for full page slide backgrounds + backgroundTransition: 'default', // none/fade/slide/convex/concave/zoom + + // Number of slides away from the current that are visible + viewDistance: 3, + + // Parallax background image + parallaxBackgroundImage: '', // e.g. "'https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg'" + + // Parallax background size + parallaxBackgroundSize: '', // CSS syntax, e.g. "2100px 900px" + + // Amount to move parallax background (horizontal and vertical) on slide change + // Number, e.g. 100 + parallaxBackgroundHorizontal: '', + parallaxBackgroundVertical: '' + +}); +``` + + +The configuration can be updated after initialization using the ```configure``` method: + +```javascript +// Turn autoSlide off +Reveal.configure({ autoSlide: 0 }); + +// Start auto-sliding every 5s +Reveal.configure({ autoSlide: 5000 }); +``` + + +### Dependencies + +Reveal.js doesn't _rely_ on any third party scripts to work but a few optional libraries are included by default. These libraries are loaded as dependencies in the order they appear, for example: + +```javascript +Reveal.initialize({ + dependencies: [ + // Cross-browser shim that fully implements classList - https://github.com/eligrey/classList.js/ + { src: 'lib/js/classList.js', condition: function() { return !document.body.classList; } }, + + // Interpret Markdown in
elements + { src: 'plugin/markdown/marked.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } }, + { src: 'plugin/markdown/markdown.js', condition: function() { return !!document.querySelector( '[data-markdown]' ); } }, + + // Syntax highlight for elements + { src: 'plugin/highlight/highlight.js', async: true, callback: function() { hljs.initHighlightingOnLoad(); } }, + + // Zoom in and out with Alt+click + { src: 'plugin/zoom-js/zoom.js', async: true }, + + // Speaker notes + { src: 'plugin/notes/notes.js', async: true }, + + // Remote control your reveal.js presentation using a touch device + { src: 'plugin/remotes/remotes.js', async: true }, + + // MathJax + { src: 'plugin/math/math.js', async: true } + ] +}); +``` + +You can add your own extensions using the same syntax. The following properties are available for each dependency object: +- **src**: Path to the script to load +- **async**: [optional] Flags if the script should load after reveal.js has started, defaults to false +- **callback**: [optional] Function to execute when the script has loaded +- **condition**: [optional] Function which must return true for the script to be loaded + + +### Ready Event + +A 'ready' event is fired when reveal.js has loaded all non-async dependencies and is ready to start navigating. To check if reveal.js is already 'ready' you can call `Reveal.isReady()`. + +```javascript +Reveal.addEventListener( 'ready', function( event ) { + // event.currentSlide, event.indexh, event.indexv +} ); +``` + + +### Presentation Size + +All presentations have a normal size, that is the resolution at which they are authored. The framework will automatically scale presentations uniformly based on this size to ensure that everything fits on any given display or viewport. + +See below for a list of configuration options related to sizing, including default values: + +```javascript +Reveal.initialize({ + + ... + + // The "normal" size of the presentation, aspect ratio will be preserved + // when the presentation is scaled to fit different resolutions. Can be + // specified using percentage units. + width: 960, + height: 700, + + // Factor of the display size that should remain empty around the content + margin: 0.1, + + // Bounds for smallest/largest possible scale to apply to content + minScale: 0.2, + maxScale: 1.5 + +}); +``` + + +### Auto-sliding + +Presentations can be configured to progress through slides automatically, without any user input. To enable this you will need to tell the framework how many milliseconds it should wait between slides: + +```javascript +// Slide every five seconds +Reveal.configure({ + autoSlide: 5000 +}); +``` +When this is turned on a control element will appear that enables users to pause and resume auto-sliding. Alternatively, sliding can be paused or resumed by pressing »a« on the keyboard. Sliding is paused automatically as soon as the user starts navigating. You can disable these controls by specifying ```autoSlideStoppable: false``` in your reveal.js config. + +You can also override the slide duration for individual slides and fragments by using the ```data-autoslide``` attribute: + +```html +
+

After 2 seconds the first fragment will be shown.

+

After 10 seconds the next fragment will be shown.

+

Now, the fragment is displayed for 2 seconds before the next slide is shown.

+
+``` + +Whenever the auto-slide mode is resumed or paused the ```autoslideresumed``` and ```autoslidepaused``` events are fired. + + +### Keyboard Bindings + +If you're unhappy with any of the default keyboard bindings you can override them using the ```keyboard``` config option: + +```javascript +Reveal.configure({ + keyboard: { + 13: 'next', // go to the next slide when the ENTER key is pressed + 27: function() {}, // do something custom when ESC is pressed + 32: null // don't do anything when SPACE is pressed (i.e. disable a reveal.js default binding) + } +}); +``` + +### Lazy Loading + +When working on presentation with a lot of media or iframe content it's important to load lazily. Lazy loading means that reveal.js will only load content for the few slides nearest to the current slide. The number of slides that are preloaded is determined by the `viewDistance` configuration option. + +To enable lazy loading all you need to do is change your "src" attributes to "data-src" as shown below. This is supported for image, video, audio and iframe elements. Lazy loaded iframes will also unload when the containing slide is no longer visible. + +```html +
+ + + +
+``` + + +### API + +The ``Reveal`` object exposes a JavaScript API for controlling navigation and reading state: + +```javascript +// Navigation +Reveal.slide( indexh, indexv, indexf ); +Reveal.left(); +Reveal.right(); +Reveal.up(); +Reveal.down(); +Reveal.prev(); +Reveal.next(); +Reveal.prevFragment(); +Reveal.nextFragment(); + +// Toggle presentation states, optionally pass true/false to force on/off +Reveal.toggleOverview(); +Reveal.togglePause(); +Reveal.toggleAutoSlide(); + +// Change a config value at runtime +Reveal.configure({ controls: true }); + +// Returns the present configuration options +Reveal.getConfig(); + +// Fetch the current scale of the presentation +Reveal.getScale(); + +// Retrieves the previous and current slide elements +Reveal.getPreviousSlide(); +Reveal.getCurrentSlide(); + +Reveal.getIndices(); // { h: 0, v: 0 } } +Reveal.getProgress(); // 0-1 +Reveal.getTotalSlides(); + +// State checks +Reveal.isFirstSlide(); +Reveal.isLastSlide(); +Reveal.isOverview(); +Reveal.isPaused(); +Reveal.isAutoSliding(); +``` + +### Slide Changed Event + +A 'slidechanged' event is fired each time the slide is changed (regardless of state). The event object holds the index values of the current slide as well as a reference to the previous and current slide HTML nodes. + +Some libraries, like MathJax (see [#226](https://github.com/hakimel/reveal.js/issues/226#issuecomment-10261609)), get confused by the transforms and display states of slides. Often times, this can be fixed by calling their update or render function from this callback. + +```javascript +Reveal.addEventListener( 'slidechanged', function( event ) { + // event.previousSlide, event.currentSlide, event.indexh, event.indexv +} ); +``` + +### Presentation State + +The presentation's current state can be fetched by using the `getState` method. A state object contains all of the information required to put the presentation back as it was when `getState` was first called. Sort of like a snapshot. It's a simple object that can easily be stringified and persisted or sent over the wire. + +```javascript +Reveal.slide( 1 ); +// we're on slide 1 + +var state = Reveal.getState(); + +Reveal.slide( 3 ); +// we're on slide 3 + +Reveal.setState( state ); +// we're back on slide 1 +``` + +### Slide States + +If you set ``data-state="somestate"`` on a slide ``
``, "somestate" will be applied as a class on the document element when that slide is opened. This allows you to apply broad style changes to the page based on the active slide. + +Furthermore you can also listen to these changes in state via JavaScript: + +```javascript +Reveal.addEventListener( 'somestate', function() { + // TODO: Sprinkle magic +}, false ); +``` + +### Slide Backgrounds + +Slides are contained within a limited portion of the screen by default to allow them to fit any display and scale uniformly. You can apply full page backgrounds outside of the slide area by adding a ```data-background``` attribute to your ```
``` elements. Four different types of backgrounds are supported: color, image, video and iframe. Below are a few examples. + +```html +
+

All CSS color formats are supported, like rgba() or hsl().

+
+
+

This slide will have a full-size background image.

+
+
+

This background image will be sized to 100px and repeated.

+
+
+

Video. Multiple sources can be defined using a comma separated list. Video will loop when the data-background-video-loop attribute is provided.

+
+
+

Embeds a web page as a background. Note that the page won't be interactive.

+
+``` + +Backgrounds transition using a fade animation by default. This can be changed to a linear sliding transition by passing ```backgroundTransition: 'slide'``` to the ```Reveal.initialize()``` call. Alternatively you can set ```data-background-transition``` on any section with a background to override that specific transition. + + +### Parallax Background + +If you want to use a parallax scrolling background, set the first two config properties below when initializing reveal.js (the other two are optional). + +```javascript +Reveal.initialize({ + + // Parallax background image + parallaxBackgroundImage: '', // e.g. "https://s3.amazonaws.com/hakim-static/reveal-js/reveal-parallax-1.jpg" + + // Parallax background size + parallaxBackgroundSize: '', // CSS syntax, e.g. "2100px 900px" - currently only pixels are supported (don't use % or auto) + + // Amount of pixels to move the parallax background per slide step, + // a value of 0 disables movement along the given axis + // These are optional, if they aren't specified they'll be calculated automatically + parallaxBackgroundHorizontal: 200, + parallaxBackgroundVertical: 50 + +}); +``` + +Make sure that the background size is much bigger than screen size to allow for some scrolling. [View example](http://lab.hakim.se/reveal-js/?parallaxBackgroundImage=https%3A%2F%2Fs3.amazonaws.com%2Fhakim-static%2Freveal-js%2Freveal-parallax-1.jpg¶llaxBackgroundSize=2100px%20900px). + + + +### Slide Transitions +The global presentation transition is set using the ```transition``` config value. You can override the global transition for a specific slide by using the ```data-transition``` attribute: + +```html +
+

This slide will override the presentation transition and zoom!

+
+ +
+

Choose from three transition speeds: default, fast or slow!

+
+``` + +You can also use different in and out transitions for the same slide: + +```html +
+ The train goes on … +
+
+ and on … +
+
+ and stops. +
+
+ (Passengers entering and leaving) +
+
+ And it starts again. +
+``` + + +Note that this does not work with the page and cube transitions. + + +### Internal links + +It's easy to link between slides. The first example below targets the index of another slide whereas the second targets a slide with an ID attribute (```
```): + +```html +Link +Link +``` + +You can also add relative navigation links, similar to the built in reveal.js controls, by appending one of the following classes on any element. Note that each element is automatically given an ```enabled``` class when it's a valid navigation route based on the current slide. + +```html + + + + + + +``` + + +### Fragments +Fragments are used to highlight individual elements on a slide. Every element with the class ```fragment``` will be stepped through before moving on to the next slide. Here's an example: http://lab.hakim.se/reveal-js/#/fragments + +The default fragment style is to start out invisible and fade in. This style can be changed by appending a different class to the fragment: + +```html +
+

grow

+

shrink

+

fade-out

+

visible only once

+

blue only once

+

highlight-red

+

highlight-green

+

highlight-blue

+
+``` + +Multiple fragments can be applied to the same element sequentially by wrapping it, this will fade in the text on the first step and fade it back out on the second. + +```html +
+ + I'll fade in, then out + +
+``` + +The display order of fragments can be controlled using the ```data-fragment-index``` attribute. + +```html +
+

Appears last

+

Appears first

+

Appears second

+
+``` + +### Fragment events + +When a slide fragment is either shown or hidden reveal.js will dispatch an event. + +Some libraries, like MathJax (see #505), get confused by the initially hidden fragment elements. Often times this can be fixed by calling their update or render function from this callback. + +```javascript +Reveal.addEventListener( 'fragmentshown', function( event ) { + // event.fragment = the fragment DOM element +} ); +Reveal.addEventListener( 'fragmenthidden', function( event ) { + // event.fragment = the fragment DOM element +} ); +``` + +### Code syntax highlighting + +By default, Reveal is configured with [highlight.js](http://softwaremaniacs.org/soft/highlight/en/) for code syntax highlighting. Below is an example with clojure code that will be syntax highlighted. When the `data-trim` attribute is present surrounding whitespace is automatically removed. + +```html +
+ 

+(def lazy-fib
+  (concat
+   [0 1]
+   ((fn rfib [a b]
+        (lazy-cons (+ a b) (rfib b (+ a b)))) 0 1)))
+
+
+``` + +### Slide number +If you would like to display the page number of the current slide you can do so using the ```slideNumber``` configuration value. + +```javascript +// Shows the slide number using default formatting +Reveal.configure({ slideNumber: true }); + +// Slide number formatting can be configured using these variables: +// h: current slide's horizontal index +// v: current slide's vertical index +// c: current slide index (flattened) +// t: total number of slides (flattened) +Reveal.configure({ slideNumber: 'c / t' }); + +``` + + +### Overview mode + +Press "Esc" or "o" keys to toggle the overview mode on and off. While you're in this mode, you can still navigate between slides, +as if you were at 1,000 feet above your presentation. The overview mode comes with a few API hooks: + +```javascript +Reveal.addEventListener( 'overviewshown', function( event ) { /* ... */ } ); +Reveal.addEventListener( 'overviewhidden', function( event ) { /* ... */ } ); + +// Toggle the overview mode programmatically +Reveal.toggleOverview(); +``` + +### Fullscreen mode +Just press »F« on your keyboard to show your presentation in fullscreen mode. Press the »ESC« key to exit fullscreen mode. + + +### Embedded media +Embedded HTML5 `
+ +
+ +

 

 

 

+ + + + + + +
+

Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression

+ +

+ + +

+Morten Hjorth-Jensen [1, 2] +
+ +

+ + +

[1] Department of Physics, University of Oslo
+
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
+
+

+

Sep 6, 2021

+
+

+ + +

Read »

+ + +
+ +

+ +

+ + +
+ + + + + + + +
+ © 1999-2021, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs001.html b/doc/pub/week35/html/._week35-bs001.html new file mode 100644 index 000000000..eeb8931d8 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs001.html @@ -0,0 +1,440 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+
+ +
+ +

 

 

 

+ + + + +

Plans for week 35, August 30 -September 3

+ +
    +
  • Thursday: Review of ordinary Least Squares with applications and discussion of Ridge Regression and Singular Value Decomposition
    • +
  • Video of lecture Thursday.
    • +
  • Friday: Analysis of Ridge and Lasso Regression and links with Singular Value Decomposition
    • +
  • Video of lecture Friday
    • +
+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs002.html b/doc/pub/week35/html/._week35-bs002.html new file mode 100644 index 000000000..d3cf64d04 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs002.html @@ -0,0 +1,444 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Thursday September 2

+ +

+The main topics on Thursday are: + +

    +
  1. Repetition from last week on linear regression
    2. +
  2. Discussion of how to prepare data and examples of applications of linear regression
    3. +
  3. Mathematical interpretations of Linear Regression
    4. +
  4. Start discussing Ridge and Lasso regression and Singular Value Decomposition
    5. +
+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs003.html b/doc/pub/week35/html/._week35-bs003.html new file mode 100644 index 000000000..7d55cf37b --- /dev/null +++ b/doc/pub/week35/html/._week35-bs003.html @@ -0,0 +1,457 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week

+ +

+We need first a reminder from last week about linear regression. + +

+Fitting a continuous function with linear parameterization in terms of the parameters \( \boldsymbol{\beta} \). + +

    +
  • Method of choice for fitting a continuous function!
    • +
  • Gives an excellent introduction to central Machine Learning features with understandable pedagogical links to other methods like Neural Networks, Support Vector Machines etc
    • +
  • Analytical expression for the fitting parameters \( \boldsymbol{\beta} \)
    • +
  • Analytical expressions for statistical propertiers like mean values, variances, confidence intervals and more
    • +
  • Analytical relation with probabilistic interpretations
    • +
  • Easy to introduce basic concepts like bias-variance tradeoff, cross-validation, resampling and regularization techniques and many other ML topics
    • +
  • Easy to code! And links well with classification problems and logistic regression and neural networks
    • +
  • Allows for easy hands-on understanding of gradient descent methods
    • +
  • and many more features
    • +
+ +For more discussions of Ridge and Lasso regression, Wessel van Wieringen's article is highly recommended. +Similarly, Mehta et al's article is also recommended. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs004.html b/doc/pub/week35/html/._week35-bs004.html new file mode 100644 index 000000000..a6cf527f8 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs004.html @@ -0,0 +1,458 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Regression analysis, overarching aims

+
+
+

+ +

+Regression modeling deals with the description of the sampling distribution of a given random variable \( y \) and how it varies as function of another variable or a set of such variables \( \boldsymbol{x} =[x_0, x_1,\dots, x_{n-1}]^T \). +The first variable is called the dependent, the outcome or the response variable while the set of variables \( \boldsymbol{x} \) is called the independent variable, or the predictor variable or the explanatory variable. + +

+A regression model aims at finding a likelihood function \( p(\boldsymbol{y}\vert \boldsymbol{x}) \), that is the conditional distribution for \( \boldsymbol{y} \) with a given \( \boldsymbol{x} \). The estimation of \( p(\boldsymbol{y}\vert \boldsymbol{x}) \) is made using a data set with + +

    +
  • \( n \) cases \( i = 0, 1, 2, \dots, n-1 \)
    • +
  • Response (target, dependent or outcome) variable \( y_i \) with \( i = 0, 1, 2, \dots, n-1 \)
    • +
  • \( p \) so-called explanatory (independent or predictor) variables \( \boldsymbol{x}_i=[x_{i0}, x_{i1}, \dots, x_{ip-1}] \) with \( i = 0, 1, 2, \dots, n-1 \) and explanatory variables running from \( 0 \) to \( p-1 \). See below for more explicit examples.
    • +
+ + The goal of the regression analysis is to extract/exploit relationship between \( \boldsymbol{y} \) and \( \boldsymbol{x} \) in or to infer causal dependencies, approximations to the likelihood functions, functional relationships and to make predictions, making fits and many other things. +
+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs005.html b/doc/pub/week35/html/._week35-bs005.html new file mode 100644 index 000000000..a8eb2ce10 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs005.html @@ -0,0 +1,467 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Regression analysis, overarching aims II

+
+
+

+ +

+Consider an experiment in which \( p \) characteristics of \( n \) samples are +measured. The data from this experiment, for various explanatory variables \( p \) are normally represented by a matrix +\( \mathbf{X} \). + +

+The matrix \( \mathbf{X} \) is called the design +matrix. Additional information of the samples is available in the +form of \( \boldsymbol{y} \) (also as above). The variable \( \boldsymbol{y} \) is +generally referred to as the response variable. The aim of +regression analysis is to explain \( \boldsymbol{y} \) in terms of +\( \boldsymbol{X} \) through a functional relationship like \( y_i = +f(\mathbf{X}_{i,\ast}) \). When no prior knowledge on the form of +\( f(\cdot) \) is available, it is common to assume a linear relationship +between \( \boldsymbol{X} \) and \( \boldsymbol{y} \). This assumption gives rise to +the linear regression model where \( \boldsymbol{\beta} = [\beta_0, \ldots, +\beta_{p-1}]^{T} \) are the regression parameters. + +

+Linear regression gives us a set of analytical equations for the parameters \( \beta_j \). + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs006.html b/doc/pub/week35/html/._week35-bs006.html new file mode 100644 index 000000000..788eb6068 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs006.html @@ -0,0 +1,464 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Examples

+
+
+

+In order to understand the relation among the predictors \( p \), the set of data \( n \) and the target (outcome, output etc) \( \boldsymbol{y} \), +consider the model we discussed for describing nuclear binding energies. + +

+There we assumed that we could parametrize the data using a polynomial approximation based on the liquid drop model. +Assuming +$$ +BE(A) = a_0+a_1A+a_2A^{2/3}+a_3A^{-1/3}+a_4A^{-1}, +$$ + +we have five predictors, that is the intercept, the \( A \) dependent term, the \( A^{2/3} \) term and the \( A^{-1/3} \) and \( A^{-1} \) terms. +This gives \( p=0,1,2,3,4 \). Furthermore we have \( n \) entries for each predictor. It means that our design matrix is a +\( p\times n \) matrix \( \boldsymbol{X} \). + +

+Here the predictors are based on a model we have made. A popular data set which is widely encountered in ML applications is the +so-called credit card default data from Taiwan. The data set contains data on \( n=30000 \) credit card holders with predictors like gender, marital status, age, profession, education, etc. In total there are \( 24 \) such predictors or attributes leading to a design matrix of dimensionality \( 24 \times 30000 \). This is however a classification problem and we will come back to it when we discuss Logistic Regression. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs007.html b/doc/pub/week35/html/._week35-bs007.html new file mode 100644 index 000000000..1d17a9f91 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs007.html @@ -0,0 +1,457 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

General linear models

+
+
+

+Before we proceed let us study a case from linear algebra where we aim at fitting a set of data \( \boldsymbol{y}=[y_0,y_1,\dots,y_{n-1}] \). We could think of these data as a result of an experiment or a complicated numerical experiment. These data are functions of a series of variables \( \boldsymbol{x}=[x_0,x_1,\dots,x_{n-1}] \), that is \( y_i = y(x_i) \) with \( i=0,1,2,\dots,n-1 \). The variables \( x_i \) could represent physical quantities like time, temperature, position etc. We assume that \( y(x) \) is a smooth function. + +

+Since obtaining these data points may not be trivial, we want to use these data to fit a function which can allow us to make predictions for values of \( y \) which are not in the present set. The perhaps simplest approach is to assume we can parametrize our function in terms of a polynomial of degree \( n-1 \) with \( n \) points, that is +$$ +y=y(x) \rightarrow y(x_i)=\tilde{y}_i+\epsilon_i=\sum_{j=0}^{n-1} \beta_j x_i^j+\epsilon_i, +$$ + +where \( \epsilon_i \) is the error in our approximation. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs008.html b/doc/pub/week35/html/._week35-bs008.html new file mode 100644 index 000000000..998e361fc --- /dev/null +++ b/doc/pub/week35/html/._week35-bs008.html @@ -0,0 +1,457 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Rewriting the fitting procedure as a linear algebra problem

+
+
+

+For every set of values \( y_i,x_i \) we have thus the corresponding set of equations +$$ +\begin{align*} +y_0&=\beta_0+\beta_1x_0^1+\beta_2x_0^2+\dots+\beta_{n-1}x_0^{n-1}+\epsilon_0\\ +y_1&=\beta_0+\beta_1x_1^1+\beta_2x_1^2+\dots+\beta_{n-1}x_1^{n-1}+\epsilon_1\\ +y_2&=\beta_0+\beta_1x_2^1+\beta_2x_2^2+\dots+\beta_{n-1}x_2^{n-1}+\epsilon_2\\ +\dots & \dots \\ +y_{n-1}&=\beta_0+\beta_1x_{n-1}^1+\beta_2x_{n-1}^2+\dots+\beta_{n-1}x_{n-1}^{n-1}+\epsilon_{n-1}.\\ +\end{align*} +$$ +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs009.html b/doc/pub/week35/html/._week35-bs009.html new file mode 100644 index 000000000..1b8e650a3 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs009.html @@ -0,0 +1,481 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Rewriting the fitting procedure as a linear algebra problem, more details

+
+
+

+Defining the vectors +$$ +\boldsymbol{y} = [y_0,y_1, y_2,\dots, y_{n-1}]^T, +$$ + +and +$$ +\boldsymbol{\beta} = [\beta_0,\beta_1, \beta_2,\dots, \beta_{n-1}]^T, +$$ + +and +$$ +\boldsymbol{\epsilon} = [\epsilon_0,\epsilon_1, \epsilon_2,\dots, \epsilon_{n-1}]^T, +$$ + +and the design matrix +$$ +\boldsymbol{X}= +\begin{bmatrix} +1& x_{0}^1 &x_{0}^2& \dots & \dots &x_{0}^{n-1}\\ +1& x_{1}^1 &x_{1}^2& \dots & \dots &x_{1}^{n-1}\\ +1& x_{2}^1 &x_{2}^2& \dots & \dots &x_{2}^{n-1}\\ +\dots& \dots &\dots& \dots & \dots &\dots\\ +1& x_{n-1}^1 &x_{n-1}^2& \dots & \dots &x_{n-1}^{n-1}\\ +\end{bmatrix} +$$ + +we can rewrite our equations as +$$ +\boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta}+\boldsymbol{\epsilon}. +$$ + +The above design matrix is called a Vandermonde matrix. +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs010.html b/doc/pub/week35/html/._week35-bs010.html new file mode 100644 index 000000000..5509a62e5 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs010.html @@ -0,0 +1,471 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Generalizing the fitting procedure as a linear algebra problem

+
+
+

+ +

+We are obviously not limited to the above polynomial expansions. We +could replace the various powers of \( x \) with elements of Fourier +series or instead of \( x_i^j \) we could have \( \cos{(j x_i)} \) or \( \sin{(j +x_i)} \), or time series or other orthogonal functions. For every set +of values \( y_i,x_i \) we can then generalize the equations to + +$$ +\begin{align*} +y_0&=\beta_0x_{00}+\beta_1x_{01}+\beta_2x_{02}+\dots+\beta_{n-1}x_{0n-1}+\epsilon_0\\ +y_1&=\beta_0x_{10}+\beta_1x_{11}+\beta_2x_{12}+\dots+\beta_{n-1}x_{1n-1}+\epsilon_1\\ +y_2&=\beta_0x_{20}+\beta_1x_{21}+\beta_2x_{22}+\dots+\beta_{n-1}x_{2n-1}+\epsilon_2\\ +\dots & \dots \\ +y_{i}&=\beta_0x_{i0}+\beta_1x_{i1}+\beta_2x_{i2}+\dots+\beta_{n-1}x_{in-1}+\epsilon_i\\ +\dots & \dots \\ +y_{n-1}&=\beta_0x_{n-1,0}+\beta_1x_{n-1,2}+\beta_2x_{n-1,2}+\dots+\beta_{n-1}x_{n-1,n-1}+\epsilon_{n-1}.\\ +\end{align*} +$$ + +

+Note that we have \( p=n \) here. The matrix is symmetric. This is generally not the case! +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs011.html b/doc/pub/week35/html/._week35-bs011.html new file mode 100644 index 000000000..f6a655a06 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs011.html @@ -0,0 +1,467 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Generalizing the fitting procedure as a linear algebra problem

+
+
+

+We redefine in turn the matrix \( \boldsymbol{X} \) as +$$ +\boldsymbol{X}= +\begin{bmatrix} +x_{00}& x_{01} &x_{02}& \dots & \dots &x_{0,n-1}\\ +x_{10}& x_{11} &x_{12}& \dots & \dots &x_{1,n-1}\\ +x_{20}& x_{21} &x_{22}& \dots & \dots &x_{2,n-1}\\ +\dots& \dots &\dots& \dots & \dots &\dots\\ +x_{n-1,0}& x_{n-1,1} &x_{n-1,2}& \dots & \dots &x_{n-1,n-1}\\ +\end{bmatrix} +$$ + +and without loss of generality we rewrite again our equations as +$$ +\boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta}+\boldsymbol{\epsilon}. +$$ + +The left-hand side of this equation is kwown. Our error vector \( \boldsymbol{\epsilon} \) and the parameter vector \( \boldsymbol{\beta} \) are our unknow quantities. How can we obtain the optimal set of \( \beta_i \) values? +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs012.html b/doc/pub/week35/html/._week35-bs012.html new file mode 100644 index 000000000..71c7e5f9b --- /dev/null +++ b/doc/pub/week35/html/._week35-bs012.html @@ -0,0 +1,468 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Optimizing our parameters

+
+
+

+We have defined the matrix \( \boldsymbol{X} \) via the equations +$$ +\begin{align*} +y_0&=\beta_0x_{00}+\beta_1x_{01}+\beta_2x_{02}+\dots+\beta_{n-1}x_{0n-1}+\epsilon_0\\ +y_1&=\beta_0x_{10}+\beta_1x_{11}+\beta_2x_{12}+\dots+\beta_{n-1}x_{1n-1}+\epsilon_1\\ +y_2&=\beta_0x_{20}+\beta_1x_{21}+\beta_2x_{22}+\dots+\beta_{n-1}x_{2n-1}+\epsilon_1\\ +\dots & \dots \\ +y_{i}&=\beta_0x_{i0}+\beta_1x_{i1}+\beta_2x_{i2}+\dots+\beta_{n-1}x_{in-1}+\epsilon_1\\ +\dots & \dots \\ +y_{n-1}&=\beta_0x_{n-1,0}+\beta_1x_{n-1,2}+\beta_2x_{n-1,2}+\dots+\beta_{n-1}x_{n-1,n-1}+\epsilon_{n-1}.\\ +\end{align*} +$$ + +

+As we noted above, we stayed with a system with the design matrix + \( \boldsymbol{X}\in {\mathbb{R}}^{n\times n} \), that is we have \( p=n \). For reasons to come later (algorithmic arguments) we will hereafter define +our matrix as \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \), with the predictors refering to the column numbers and the entries \( n \) being the row elements. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs013.html b/doc/pub/week35/html/._week35-bs013.html new file mode 100644 index 000000000..1cf3811e5 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs013.html @@ -0,0 +1,529 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Our model for the nuclear binding energies

+ +

+In our introductory notes we looked at the so-called liquid drop model. Let us remind ourselves about what we did by looking at the code. + +

+We restate the parts of the code we are most interested in. +

+ + +

# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from IPython.display import display
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("MassEval2016.dat"),'r')
+
+
+# Read the experimental data with Pandas
+Masses = pd.read_fwf(infile, usecols=(2,3,4,6,11),
+              names=('N', 'Z', 'A', 'Element', 'Ebinding'),
+              widths=(1,3,5,5,5,1,3,4,1,13,11,11,9,1,2,11,9,1,3,1,12,11,1),
+              header=39,
+              index_col=False)
+
+# Extrapolated values are indicated by '#' in place of the decimal place, so
+# the Ebinding column won't be numeric. Coerce to float and drop these entries.
+Masses['Ebinding'] = pd.to_numeric(Masses['Ebinding'], errors='coerce')
+Masses = Masses.dropna()
+# Convert from keV to MeV.
+Masses['Ebinding'] /= 1000
+
+# Group the DataFrame by nucleon number, A.
+Masses = Masses.groupby('A')
+# Find the rows of the grouped DataFrame with the maximum binding energy.
+Masses = Masses.apply(lambda t: t[t.Ebinding==t.Ebinding.max()])
+A = Masses['A']
+Z = Masses['Z']
+N = Masses['N']
+Element = Masses['Element']
+Energies = Masses['Ebinding']
+
+# Now we set up the design matrix X
+X = np.zeros((len(A),5))
+X[:,0] = 1
+X[:,1] = A
+X[:,2] = A**(2.0/3.0)
+X[:,3] = A**(-1.0/3.0)
+X[:,4] = A**(-1.0)
+# Then nice printout using pandas
+DesignMatrix = pd.DataFrame(X)
+DesignMatrix.index = A
+DesignMatrix.columns = ['1', 'A', 'A^(2/3)', 'A^(-1/3)', '1/A']
+display(DesignMatrix)
+
+

+With \( \boldsymbol{\beta}\in {\mathbb{R}}^{p\times 1} \), it means that we will hereafter write our equations for the approximation as +$$ +\boldsymbol{\tilde{y}}= \boldsymbol{X}\boldsymbol{\beta}, +$$ + +throughout these lectures. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs014.html b/doc/pub/week35/html/._week35-bs014.html new file mode 100644 index 000000000..058df6ab7 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs014.html @@ -0,0 +1,475 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Optimizing our parameters, more details

+
+
+

+With the above we use the design matrix to define the approximation \( \boldsymbol{\tilde{y}} \) via the unknown quantity \( \boldsymbol{\beta} \) as +$$ +\boldsymbol{\tilde{y}}= \boldsymbol{X}\boldsymbol{\beta}, +$$ + +and in order to find the optimal parameters \( \beta_i \) instead of solving the above linear algebra problem, we define a function which gives a measure of the spread between the values \( y_i \) (which represent hopefully the exact values) and the parameterized values \( \tilde{y}_i \), namely +$$ +C(\boldsymbol{\beta})=\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)^T\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)\right\}, +$$ + +or using the matrix \( \boldsymbol{X} \) and in a more compact matrix-vector notation as +$$ +C(\boldsymbol{\beta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}. +$$ + +This function is one possible way to define the so-called cost function. + +

+It is also common to define +the function \( C \) as + +$$ +C(\boldsymbol{\beta})=\frac{1}{2n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2, +$$ + +since when taking the first derivative with respect to the unknown parameters \( \beta \), the factor of \( 2 \) cancels out. +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs015.html b/doc/pub/week35/html/._week35-bs015.html new file mode 100644 index 000000000..b2bfcc61f --- /dev/null +++ b/doc/pub/week35/html/._week35-bs015.html @@ -0,0 +1,495 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Interpretations and optimizing our parameters

+
+
+

+ +

+The function +$$ +C(\boldsymbol{\beta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}, +$$ + +can be linked to the variance of the quantity \( y_i \) if we interpret the latter as the mean value. +When linking (see the discussion below) with the maximum likelihood approach below, we will indeed interpret \( y_i \) as a mean value +$$ +y_{i}=\langle y_i \rangle = \beta_0x_{i,0}+\beta_1x_{i,1}+\beta_2x_{i,2}+\dots+\beta_{n-1}x_{i,n-1}+\epsilon_i, +$$ + +

+where \( \langle y_i \rangle \) is the mean value. Keep in mind also that +till now we have treated \( y_i \) as the exact value. Normally, the +response (dependent or outcome) variable \( y_i \) the outcome of a +numerical experiment or another type of experiment and is thus only an +approximation to the true value. It is then always accompanied by an +error estimate, often limited to a statistical error estimate given by +the standard deviation discussed earlier. In the discussion here we +will treat \( y_i \) as our exact value for the response variable. + +

+In order to find the parameters \( \beta_i \) we will then minimize the spread of \( C(\boldsymbol{\beta}) \), that is we are going to solve the problem +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}. +$$ + +In practical terms it means we will require +$$ +\frac{\partial C(\boldsymbol{\beta})}{\partial \beta_j} = \frac{\partial }{\partial \beta_j}\left[ \frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)^2\right]=0, +$$ + +which results in +$$ +\frac{\partial C(\boldsymbol{\beta})}{\partial \beta_j} = -\frac{2}{n}\left[ \sum_{i=0}^{n-1}x_{ij}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)\right]=0, +$$ + +or in a matrix-vector form as +$$ +\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}} = 0 = \boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right). +$$ + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs016.html b/doc/pub/week35/html/._week35-bs016.html new file mode 100644 index 000000000..7c5c643b2 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs016.html @@ -0,0 +1,484 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Interpretations and optimizing our parameters

+
+
+

+We can rewrite +$$ +\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}} = 0 = \boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right), +$$ + +as +$$ +\boldsymbol{X}^T\boldsymbol{y} = \boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta}, +$$ + +and if the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) is invertible we have the solution +$$ +\boldsymbol{\beta} =\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

+We note also that since our design matrix is defined as \( \boldsymbol{X}\in +{\mathbb{R}}^{n\times p} \), the product \( \boldsymbol{X}^T\boldsymbol{X} \in +{\mathbb{R}}^{p\times p} \). In the above case we have that \( p \ll n \), +in our case \( p=5 \) meaning that we end up with inverting a small +\( 5\times 5 \) matrix. This is a rather common situation, in many cases we end up with low-dimensional +matrices to invert. The methods discussed here and for many other +supervised learning algorithms like classification with logistic +regression or support vector machines, exhibit dimensionalities which +allow for the usage of direct linear algebra methods such as LU decomposition or Singular Value Decomposition (SVD) for finding the inverse of the matrix +\( \boldsymbol{X}^T\boldsymbol{X} \). +

+
+ + +

+

+
+

+Small question: Do you think the example we have at hand here (the nuclear binding energies) can lead to problems in inverting the matrix \( \boldsymbol{X}^T\boldsymbol{X} \)? What kind of problems can we expect? +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs017.html b/doc/pub/week35/html/._week35-bs017.html new file mode 100644 index 000000000..d29decc17 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs017.html @@ -0,0 +1,463 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Some useful matrix and vector expressions

+ +

+The following matrix and vector relation will be useful here and for the rest of the course. Vectors are always written as boldfaced lower case letters and +matrices as upper case boldfaced letters. + +$$ +\frac{\partial (\boldsymbol{b}^T\boldsymbol{a})}{\partial \boldsymbol{a}} = \boldsymbol{b}, +$$ + +$$ +\frac{\partial (\boldsymbol{a}^T\boldsymbol{A}\boldsymbol{a})}{\partial \boldsymbol{a}} = (\boldsymbol{A}+\boldsymbol{A}^T)\boldsymbol{a}, +$$ + +$$ +\frac{\partial tr(\boldsymbol{B}\boldsymbol{A})}{\partial \boldsymbol{A}} = \boldsymbol{B}^T, +$$ + +$$ +\frac{\partial \log{\vert\boldsymbol{A}\vert}}{\partial \boldsymbol{A}} = (\boldsymbol{A}^{-1})^T. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs018.html b/doc/pub/week35/html/._week35-bs018.html new file mode 100644 index 000000000..e643774b0 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs018.html @@ -0,0 +1,473 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Meet the Hessian Matrix

+ +

+A very important matrix we will meet again and again in Machine +Learning is the Hessian. It is given by the second derivative of the +cost function with respect to the parameter \( \beta \). Using the above +expression for derivatives of vectors and matrices, we find that the +second derivative of the cost function is, + +$$ +\frac{\partial}{\partial \boldsymbol{\beta}^T}\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}} =\frac{\partial}{\partial \boldsymbol{\beta}}\left[-\frac{2}{n}\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right]=\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}. +$$ + +

+The Hessian matrix plays an important role and is defined here as + +$$ +\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}. +$$ + +

+For ordinary least squares, it is inversely proportional (derivation +next week) with the variance of the optimal parameters +\( \hat{\boldsymbol{\beta}} \). Furthermore, we will see later this week that is +(beside \( 1/n \)) equal to the covariance matrix. It plays also a very +important role in optmization algorithms and Principal Component +Analysis as a way to reduce the dimensionality of a machine learning +problem. + +

+Linear algebra question: Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs019.html b/doc/pub/week35/html/._week35-bs019.html new file mode 100644 index 000000000..87cb4f698 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs019.html @@ -0,0 +1,467 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Interpretations and optimizing our parameters

+
+
+

+The residuals \( \boldsymbol{\epsilon} \) are in turn given by +$$ +\boldsymbol{\epsilon} = \boldsymbol{y}-\boldsymbol{\tilde{y}} = \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}, +$$ + +and with +$$ +\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)= 0, +$$ + +we have +$$ +\boldsymbol{X}^T\boldsymbol{\epsilon}=\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)= 0, +$$ + +meaning that the solution for \( \boldsymbol{\beta} \) is the one which minimizes the residuals. Later we will link this with the maximum likelihood approach. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs020.html b/doc/pub/week35/html/._week35-bs020.html new file mode 100644 index 000000000..8a59b55b2 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs020.html @@ -0,0 +1,480 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Own code for Ordinary Least Squares

+ +

+It is rather straightforward to implement the matrix inversion and obtain the parameters \( \boldsymbol{\beta} \). After having defined the matrix \( \boldsymbol{X} \) we simply need to +write +

+ + +

# matrix inversion to find beta
+beta = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Energies)
+# and then make the prediction
+ytilde = X @ beta
+
+

+Alternatively, you can use the least squares functionality in Numpy as +

+ + +

fit = np.linalg.lstsq(X, Energies, rcond =None)[0]
+ytildenp = np.dot(fit,X.T)
+
+

+And finally we plot our fit with and compare with data +

+ + +

Masses['Eapprox']  = ytilde
+# Generate a plot comparing the experimental with the fitted values values.
+fig, ax = plt.subplots()
+ax.set_xlabel(r'$A = N + Z$')
+ax.set_ylabel(r'$E_\mathrm{bind}\,/\mathrm{MeV}$')
+ax.plot(Masses['A'], Masses['Ebinding'], alpha=0.7, lw=2,
+            label='Ame2016')
+ax.plot(Masses['A'], Masses['Eapprox'], alpha=0.7, lw=2, c='m',
+            label='Fit')
+ax.legend()
+save_fig("Masses2016OLS")
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs021.html b/doc/pub/week35/html/._week35-bs021.html new file mode 100644 index 000000000..68b046921 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs021.html @@ -0,0 +1,479 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Adding error analysis and training set up

+ +

+We can easily test our fit by computing the \( R2 \) score that we discussed in connection with the functionality of Scikit-Learn in the introductory slides. +Since we are not using Scikit-Learn here we can define our own \( R2 \) function as +

+ + +

def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+
+

+and we would be using it as +

+ + +

print(R2(Energies,ytilde))
+
+

+We can easily add our MSE score as +

+ + +

def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+print(MSE(Energies,ytilde))
+
+

+and finally the relative error as +

+ + +

def RelativeError(y_data,y_model):
+    return abs((y_data-y_model)/y_data)
+print(RelativeError(Energies, ytilde))
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs022.html b/doc/pub/week35/html/._week35-bs022.html new file mode 100644 index 000000000..a2c0752dd --- /dev/null +++ b/doc/pub/week35/html/._week35-bs022.html @@ -0,0 +1,465 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Splitting our Data in Training and Test data

+ +

+

+
+

+ +

+It is normal in essentially all Machine Learning studies to split the +data in a training set and a test set (sometimes also an additional +validation set). Scikit-Learn has an own function for this. There +is no explicit recipe for how much data should be included as training +data and say test data. An accepted rule of thumb is to use +approximately \( 2/3 \) to \( 4/5 \) of the data as training data. We will +postpone a discussion of this splitting to the end of these notes and +our discussion of the so-called bias-variance tradeoff. Here we +limit ourselves to repeat the above equation of state fitting example +but now splitting the data into a training set and a test set. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs023.html b/doc/pub/week35/html/._week35-bs023.html new file mode 100644 index 000000000..23ac0efee --- /dev/null +++ b/doc/pub/week35/html/._week35-bs023.html @@ -0,0 +1,485 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Examples

+ +

+ + +

import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+x = np.random.rand(100)
+y = 2.0+5*x*x+0.1*np.random.randn(100)
+
+
+#  The design matrix now as function of a given polynomial
+X = np.zeros((len(x),3))
+X[:,0] = 1.0
+X[:,1] = x
+X[:,2] = x**2
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+# matrix inversion to find beta
+beta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
+print(beta)
+# and then make the prediction
+ytilde = X_train @ beta
+print("Training R2")
+print(R2(y_train,ytilde))
+print("Training MSE")
+print(MSE(y_train,ytilde))
+ypredict = X_test @ beta
+print("Test R2")
+print(R2(y_test,ypredict))
+print("Test MSE")
+print(MSE(y_test,ypredict))
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs024.html b/doc/pub/week35/html/._week35-bs024.html new file mode 100644 index 000000000..feb16e1ab --- /dev/null +++ b/doc/pub/week35/html/._week35-bs024.html @@ -0,0 +1,466 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Making your own test-train splitting

+ +

+ + +

# equivalently in numpy
+def train_test_split_numpy(inputs, labels, train_size, test_size):
+    n_inputs = len(inputs)
+    inputs_shuffled = inputs.copy()
+    labels_shuffled = labels.copy()
+
+    np.random.shuffle(inputs_shuffled)
+    np.random.shuffle(labels_shuffled)
+
+    train_end = int(n_inputs*train_size)
+    X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]
+    Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]
+
+    return X_train, X_test, Y_train, Y_test
+
+

+But since scikit-learn has its own function for doing this and since +it interfaces easily with tensorflow and other libraries, we +normally recommend using the latter functionality. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs025.html b/doc/pub/week35/html/._week35-bs025.html new file mode 100644 index 000000000..8d522c052 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs025.html @@ -0,0 +1,469 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The Boston housing data example

+ +

+The Boston housing +data set was originally a part of UCI Machine Learning Repository +and has been removed now. The data set is now included in Scikit-Learn's +library. There are 506 samples and 13 feature (predictor) variables +in this data set. The objective is to predict the value of prices of +the house using the features (predictors) listed here. + +

+The features/predictors are + +

    +
  1. CRIM: Per capita crime rate by town
    2. +
  2. ZN: Proportion of residential land zoned for lots over 25000 square feet
    3. +
  3. INDUS: Proportion of non-retail business acres per town
    4. +
  4. CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)
    5. +
  5. NOX: Nitric oxide concentration (parts per 10 million)
    6. +
  6. RM: Average number of rooms per dwelling
    7. +
  7. AGE: Proportion of owner-occupied units built prior to 1940
    8. +
  8. DIS: Weighted distances to five Boston employment centers
    9. +
  9. RAD: Index of accessibility to radial highways
    10. +
  10. TAX: Full-value property tax rate per USD10000
    11. +
  11. B: \( 1000(Bk - 0.63)^2 \), where \( Bk \) is the proportion of [people of African American descent] by town
    12. +
  12. LSTAT: Percentage of lower status of the population
    13. +
  13. MEDV: Median value of owner-occupied homes in USD 1000s
    14. +
+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs026.html b/doc/pub/week35/html/._week35-bs026.html new file mode 100644 index 000000000..9badf1799 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs026.html @@ -0,0 +1,595 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Housing data, the code

+We start by importing the libraries +

+ + +

import numpy as np
+import matplotlib.pyplot as plt 
+
+import pandas as pd  
+import seaborn as sns 
+
+

+and load the Boston Housing DataSet from Scikit-Learn + +

+ + +

from sklearn.datasets import load_boston
+
+boston_dataset = load_boston()
+
+# boston_dataset is a dictionary
+# let's check what it contains
+boston_dataset.keys()
+
+

+Then we invoke Pandas +

+ + +

boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)
+boston.head()
+boston['MEDV'] = boston_dataset.target
+
+

+and preprocess the data +

+ + +

# check for missing values in all the columns
+boston.isnull().sum()
+
+

+We can then visualize the data +

+ + +

# set the size of the figure
+sns.set(rc={'figure.figsize':(11.7,8.27)})
+
+# plot a histogram showing the distribution of the target values
+sns.distplot(boston['MEDV'], bins=30)
+plt.show()
+
+

+It is now useful to look at the correlation matrix +

+ + +

# compute the pair wise correlation for all columns  
+correlation_matrix = boston.corr().round(2)
+# use the heatmap function from seaborn to plot the correlation matrix
+# annot = True to print the values inside the square
+sns.heatmap(data=correlation_matrix, annot=True)
+
+

+From the above coorelation plot we can see that MEDV is strongly correlated to LSTAT and RM. We see also that RAD and TAX are stronly correlated, but we don't include this in our features together to avoid multi-colinearity + +

+ + +

plt.figure(figsize=(20, 5))
+
+features = ['LSTAT', 'RM']
+target = boston['MEDV']
+
+for i, col in enumerate(features):
+    plt.subplot(1, len(features) , i+1)
+    x = boston[col]
+    y = target
+    plt.scatter(x, y, marker='o')
+    plt.title(col)
+    plt.xlabel(col)
+    plt.ylabel('MEDV')
+
+

+Now we start training our model +

+ + +

X = pd.DataFrame(np.c_[boston['LSTAT'], boston['RM']], columns = ['LSTAT','RM'])
+Y = boston['MEDV']
+
+

+We split the data into training and test sets + +

+ + +

from sklearn.model_selection import train_test_split
+
+# splits the training and test data set in 80% : 20%
+# assign random_state to any value.This ensures consistency.
+X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state=5)
+print(X_train.shape)
+print(X_test.shape)
+print(Y_train.shape)
+print(Y_test.shape)
+
+

+Then we use the linear regression functionality from Scikit-Learn +

+ + +

from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error, r2_score
+
+lin_model = LinearRegression()
+lin_model.fit(X_train, Y_train)
+
+# model evaluation for training set
+
+y_train_predict = lin_model.predict(X_train)
+rmse = (np.sqrt(mean_squared_error(Y_train, y_train_predict)))
+r2 = r2_score(Y_train, y_train_predict)
+
+print("The model performance for training set")
+print("--------------------------------------")
+print('RMSE is {}'.format(rmse))
+print('R2 score is {}'.format(r2))
+print("\n")
+
+# model evaluation for testing set
+
+y_test_predict = lin_model.predict(X_test)
+# root mean square error of the model
+rmse = (np.sqrt(mean_squared_error(Y_test, y_test_predict)))
+
+# r-squared score of the model
+r2 = r2_score(Y_test, y_test_predict)
+
+print("The model performance for testing set")
+print("--------------------------------------")
+print('RMSE is {}'.format(rmse))
+print('R2 score is {}'.format(r2))
+
+

+ + +

# plotting the y_test vs y_pred
+# ideally should have been a straight line
+plt.scatter(Y_test, y_test_predict)
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs027.html b/doc/pub/week35/html/._week35-bs027.html new file mode 100644 index 000000000..b487508ae --- /dev/null +++ b/doc/pub/week35/html/._week35-bs027.html @@ -0,0 +1,473 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Reducing the number of degrees of freedom, overarching view

+
+
+

+ +

+Many Machine Learning problems involve thousands or even millions of +features for each training instance. Not only does this make training +extremely slow, it can also make it much harder to find a good +solution, as we will see. This problem is often referred to as the +curse of dimensionality. Fortunately, in real-world problems, it is +often possible to reduce the number of features considerably, turning +an intractable problem into a tractable one. + +

+Later we will discuss some of the most popular dimensionality reduction +techniques: the principal component analysis (PCA), Kernel PCA, and +Locally Linear Embedding (LLE). + +

+Principal component analysis and its various variants deal with the +problem of fitting a low-dimensional affine +subspace to a set of of +data points in a high-dimensional space. With its family of methods it +is one of the most used tools in data modeling, compression and +visualization. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs028.html b/doc/pub/week35/html/._week35-bs028.html new file mode 100644 index 000000000..8ecf956ec --- /dev/null +++ b/doc/pub/week35/html/._week35-bs028.html @@ -0,0 +1,468 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Preprocessing our data

+
+
+

+ +

+Before we proceed however, we will discuss how to preprocess our +data. Till now and in connection with our previous examples we have +not met so many cases where we are too sensitive to the scaling of our +data. Normally the data may need a rescaling and/or may be sensitive +to extreme values. Scaling the data renders our inputs much more +suitable for the algorithms we want to employ. + +

+For data sets gathered for real world applications, it is rather normal that +different features have very different units and +numerical scales. For example, a data set detailing health habits may include +features such as age in the range \( 0-80 \), and caloric intake of order \( 2000 \). +Many machine learning methods sensitive to the scales of the features and may perform poorly if they +are very different scales. Therefore, it is typical to scale +the features in a way to avoid such outlier values. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs029.html b/doc/pub/week35/html/._week35-bs029.html new file mode 100644 index 000000000..cbdc2688d --- /dev/null +++ b/doc/pub/week35/html/._week35-bs029.html @@ -0,0 +1,454 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Functionality in Scikit-Learn

+ +

+Scikit-Learn has several functions which allow us to rescale the +data, normally resulting in much better results in terms of various +accuracy scores. The StandardScaler function in Scikit-Learn +ensures that for each feature/predictor we study the mean value is +zero and the variance is one (every column in the design/feature +matrix). This scaling has the drawback that it does not ensure that +we have a particular maximum or minimum in our data set. Another +function included in Scikit-Learn is the MinMaxScaler which +ensures that all features are exactly between \( 0 \) and \( 1 \). The + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs030.html b/doc/pub/week35/html/._week35-bs030.html new file mode 100644 index 000000000..99229ca9c --- /dev/null +++ b/doc/pub/week35/html/._week35-bs030.html @@ -0,0 +1,470 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

More preprocessing

+ +

+

+
+

+The Normalizer scales each data +point such that the feature vector has a euclidean length of one. In other words, it +projects a data point on the circle (or sphere in the case of higher dimensions) with a +radius of 1. This means every data point is scaled by a different number (by the +inverse of it’s length). +This normalization is often used when only the direction (or angle) of the data matters, +not the length of the feature vector. + +

+The RobustScaler works similarly to the StandardScaler in that it +ensures statistical properties for each feature that guarantee that +they are on the same scale. However, the RobustScaler uses the median +and quartiles, instead of mean and variance. This makes the +RobustScaler ignore data points that are very different from the rest +(like measurement errors). These odd data points are also called +outliers, and might often lead to trouble for other scaling +techniques. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs031.html b/doc/pub/week35/html/._week35-bs031.html new file mode 100644 index 000000000..052a5b9d0 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs031.html @@ -0,0 +1,454 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Frequently used scaling functions

+ +

+Many features are often scaled using standardization to improve performance. In Scikit-Learn this is given by the StandardScaler function as discussed above. It is easy however to write your own. +Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature: + +$$ + x_j^{(i)} \rightarrow \frac{x_j^{(i)} - \overline{x}_j}{\sigma(x_j)}, +$$ + +where \( \overline{x}_j \) and \( \sigma(x_j) \) are the mean and standard deviation, respectively, of the feature \( x_j \). +This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don't wish to calculate it, it is then common to simply set it to one. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs032.html b/doc/pub/week35/html/._week35-bs032.html new file mode 100644 index 000000000..b9011ece2 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs032.html @@ -0,0 +1,479 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Example of own Standard scaling

+ +

+Let us consider the following vanilla example where we use both +Scikit-Learn and write our own function as well. We produce a +simple test design matrix with random numbers. Each column could then +represent a specific feature whose mean value is subracted. + +

+ + +

import sklearn.linear_model as skl
+from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import  train_test_split
+from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
+import numpy as np
+import pandas as pd
+from IPython.display import display
+np.random.seed(100)
+# setting up a 10 x 5 matrix
+rows = 10
+cols = 5
+X = np.random.randn(rows,cols)
+XPandas = pd.DataFrame(X)
+display(XPandas)
+print(XPandas.mean())
+print(XPandas.std())
+XPandas = (XPandas -XPandas.mean())
+display(XPandas)
+#  This option does not include the standard deviation
+scaler = StandardScaler(with_std=False)
+scaler.fit(X)
+Xscaled = scaler.transform(X)
+display(XPandas-Xscaled)
+
+

+Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs033.html b/doc/pub/week35/html/._week35-bs033.html new file mode 100644 index 000000000..2f641852d --- /dev/null +++ b/doc/pub/week35/html/._week35-bs033.html @@ -0,0 +1,455 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Min-Max Scaling

+ +

+Another commonly used scaling method is min-max scaling. This is very +useful for when we want the features to lie in a certain interval. To +scale the feature \( x_j \) to the interval \( [a, b] \), we can apply the +transformation + +$$ +x_j^{(i)} \rightarrow (b-a)\frac{x_j^{(i)} - \min(x_j)}{\max(x_j) - \min(x_j)} - a +$$ + +where \( \min(x_j) \) and \( \max(x_j) \) return the minimum and maximum value of \( x_j \) over the data set, respectively. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs034.html b/doc/pub/week35/html/._week35-bs034.html new file mode 100644 index 000000000..436998e5e --- /dev/null +++ b/doc/pub/week35/html/._week35-bs034.html @@ -0,0 +1,503 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Testing the Means Squared Error as function of Complexity

+One of +the aims is to reproduce Figure 2.11 of Hastie et al. +We will also use Ridge and Lasso regression. + +

+Our data is defined by \( x\in [-3,3] \) with a total of for example \( 100 \) data points. +

+ + +

np.random.seed()
+n = 100
+maxdegree = 14
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+
+

+where \( y \) is the function we want to fit with a given polynomial. + +

+Write a first code which sets up a design matrix \( X \) defined by a fifth-order polynomial. Scale your data and split it in training and test data. +

+ + +

import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+
+
+np.random.seed(2018)
+n = 50
+maxdegree = 5
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+TestError = np.zeros(maxdegree)
+TrainError = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(x_train)
+x_train_scaled = scaler.transform(x_train)
+x_test_scaled = scaler.transform(x_test)
+
+for degree in range(maxdegree):
+    model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
+    clf = model.fit(x_train_scaled,y_train)
+    y_fit = clf.predict(x_train_scaled)
+    y_pred = clf.predict(x_test_scaled) 
+    polydegree[degree] = degree
+    TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+    TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )
+
+plt.plot(polydegree, TestError, label='Test Error')
+plt.plot(polydegree, TrainError, label='Train Error')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs035.html b/doc/pub/week35/html/._week35-bs035.html new file mode 100644 index 000000000..b8ab58356 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs035.html @@ -0,0 +1,539 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

More preprocessing examples, Franke function and regression

+ +

+ + +

# Common imports
+import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import sklearn.linear_model as skl
+from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import  train_test_split
+from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+
+def FrankeFunction(x,y):
+	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+	return term1 + term2 + term3 + term4
+
+
+def create_X(x, y, n ):
+	if len(x.shape) > 1:
+		x = np.ravel(x)
+		y = np.ravel(y)
+
+	N = len(x)
+	l = int((n+1)*(n+2)/2)		# Number of elements in beta
+	X = np.ones((N,l))
+
+	for i in range(1,n+1):
+		q = int((i)*(i+1)/2)
+		for k in range(i+1):
+			X[:,q+k] = (x**(i-k))*(y**k)
+
+	return X
+
+
+# Making meshgrid of datapoints and compute Franke's function
+n = 5
+N = 1000
+x = np.sort(np.random.uniform(0, 1, N))
+y = np.sort(np.random.uniform(0, 1, N))
+z = FrankeFunction(x, y)
+X = create_X(x, y, n=n)    
+# split in training and test data
+X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
+
+
+clf = skl.LinearRegression().fit(X_train, y_train)
+
+# The mean squared error and R2 score
+print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test)))
+print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
+
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+print("Feature min values before scaling:\n {}".format(X_train.min(axis=0)))
+print("Feature max values before scaling:\n {}".format(X_train.max(axis=0)))
+
+print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0)))
+print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0)))
+
+clf = skl.LinearRegression().fit(X_train_scaled, y_train)
+
+
+print("MSE after  scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))
+print("R2 score for  scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs036.html b/doc/pub/week35/html/._week35-bs036.html new file mode 100644 index 000000000..4c11d7b80 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs036.html @@ -0,0 +1,479 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Mathematical Interpretation of Ordinary Least Squares

+ +

+What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD). + +

+We have shown that in ordinary least squares the optimal parameters \( \beta \) are given by + +$$ +\hat{\boldsymbol{\beta}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

+The hat over \( \boldsymbol{\beta} \) means we have the optimal parameters after minimization of the cost function. + +

+This means that our best model is defined as + +$$ +\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

+We now define a matrix +$$ +\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T. +$$ + +

+We can rewrite +$$ +\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{A}\boldsymbol{y}. +$$ + +

+The matrix \( \boldsymbol{A} \) has the important property that \( \boldsymbol{A}^2=\boldsymbol{A} \). This is the definition of a projection matrix. +We can then interpret our optimal model \( \tilde{\boldsymbol{y}} \) as being represented by an orthogonal projection of \( \boldsymbol{y} \) onto a space defined by the column vectors of \( \boldsymbol{X} \). In our case here the matrix \( \boldsymbol{A} \) is a square matrix. If it is a general rectangular matrix we have an oblique projection matrix. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs037.html b/doc/pub/week35/html/._week35-bs037.html new file mode 100644 index 000000000..5e37eeae0 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs037.html @@ -0,0 +1,452 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Residual Error

+ +

+We have defined the residual error as +$$ +\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=\left[\boldsymbol{I}-\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\right]\boldsymbol{y}. +$$ + +

+The residual errors are then the projections of \( \boldsymbol{y} \) onto the orthogonal component of the space defined by the column vectors of \( \boldsymbol{X} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs038.html b/doc/pub/week35/html/._week35-bs038.html new file mode 100644 index 000000000..c2f147ceb --- /dev/null +++ b/doc/pub/week35/html/._week35-bs038.html @@ -0,0 +1,464 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple case

+ +

+If the matrix \( \boldsymbol{X} \) is an orthogonal (or unitary in case of complex values) matrix, we have + +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{X}\boldsymbol{X}^T = \boldsymbol{I}. +$$ + +

+In this case the matrix \( \boldsymbol{A} \) becomes +$$ +\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T)=\boldsymbol{I}, +$$ + +and we have the obvious case +$$ +\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=0. +$$ + +

+This serves also as a useful test of our codes. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs039.html b/doc/pub/week35/html/._week35-bs039.html new file mode 100644 index 000000000..043422126 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs039.html @@ -0,0 +1,488 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The singular value decomposition

+ +

+

+
+

+ +

+The examples we have looked at so far are cases where we normally can +invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion where we fit of various functions leads to +row vectors of the design matrix which are essentially orthogonal due +to the polynomial character of our model. Obtaining the inverse of the +design matrix is then often done via a so-called LU, QR or Cholesky +decomposition. + +

+As we will also see in the first project, +this may +however not the be case in general and a standard matrix inversion +algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below. + +

+There is however a way to circumvent this problem and also +gain some insights about the ordinary least squares approach, and +later shrinkage methods like Ridge and Lasso regressions. + +

+This is given by the Singular Value Decomposition (SVD) algorithm, +perhaps the most powerful linear algebra algorithm. The SVD provides +a numerically stable matrix decomposition that is used in a large +swath oc applications and the decomposition is always stable +numerically. + +

+In machine learning it plays a central role in dealing with for +example design matrices that may be near singular or singular. +Furthermore, as we will see here, the singular values can be related +to the covariance matrix (and thereby the correlation matrix) and in +turn the variance of a given quantity. It plays also an important role +in the principal component analysis where high-dimensional data can be +reduced to the statistically relevant features. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs040.html b/doc/pub/week35/html/._week35-bs040.html new file mode 100644 index 000000000..23ce5aec9 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs040.html @@ -0,0 +1,489 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Linear Regression Problems

+ +

+One of the typical problems we encounter with linear regression, in particular +when the matrix \( \boldsymbol{X} \) (our so-called design matrix) is high-dimensional, +are problems with near singular or singular matrices. The column vectors of \( \boldsymbol{X} \) +may be linearly dependent, normally referred to as super-collinearity. +This means that the matrix may be rank deficient and it is basically impossible to +to model the data using linear regression. As an example, consider the matrix +$$ +\begin{align*} +\mathbf{X} & = \left[ +\begin{array}{rrr} +1 & -1 & 2 +\\ +1 & 0 & 1 +\\ +1 & 2 & -1 +\\ +1 & 1 & 0 +\end{array} \right] +\end{align*} +$$ + +

+The columns of \( \boldsymbol{X} \) are linearly dependent. We see this easily since the +the first column is the row-wise sum of the other two columns. The rank (more correct, +the column rank) of a matrix is the dimension of the space spanned by the +column vectors. Hence, the rank of \( \mathbf{X} \) is equal to the number +of linearly independent columns. In this particular case the matrix has rank 2. + +

+Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies +that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this +$$ +\begin{align*} +\boldsymbol{X} & = \left[ +\begin{array}{rr} +1 & -1 +\\ +1 & -1 +\end{array} \right]. +\end{align*} +$$ + +We see easily that \( \mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0 \). Hence, \( \mathbf{X} \) is singular and its inverse is undefined. +This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least an eigenvalue which is zero. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs041.html b/doc/pub/week35/html/._week35-bs041.html new file mode 100644 index 000000000..fc1b01818 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs041.html @@ -0,0 +1,466 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Fixing the singularity

+ +

+If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem +$$ +\begin{align} +\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y}, +\tag{1} +\end{align} +$$ + +has linearly dependent column vectors, we will not be able to compute the inverse +of \( \boldsymbol{X}^T\boldsymbol{X} \) and we cannot find the parameters (estimators) \( \beta_i \). +The estimators are only well-defined if \( (\boldsymbol{X}^{T}\boldsymbol{X})^{-1} \) exits. +This is more likely to happen when the matrix \( \boldsymbol{X} \) is high-dimensional. In this case it is likely to encounter a situation where +the regression parameters \( \beta_i \) cannot be estimated. + +

+A cheap ad hoc approach is simply to add a small diagonal component to the matrix to invert, that is we change +$$ +\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I}, +$$ + +where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge regression this is actually what we end up evaluating. The parameter \( \lambda \) is called a hyperparameter. More about this later. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs042.html b/doc/pub/week35/html/._week35-bs042.html new file mode 100644 index 000000000..4813ccc42 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs042.html @@ -0,0 +1,477 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Basic math of the SVD

+ +

+From standard linear algebra we know that a square matrix \( \boldsymbol{X} \) can be diagonalized if and only it is +a so-called normal matrix, that is if \( \boldsymbol{X}\in {\mathbb{R}}^{n\times n} \) +we have \( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) or if \( \boldsymbol{X}\in {\mathbb{C}}^{n\times n} \) we have \( \boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X} \). +The matrix has then a set of eigenpairs + +$$ +(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n), +$$ + +and the eigenvalues are given by the diagonal matrix +$$ +\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n). +$$ + +The matrix \( \boldsymbol{X} \) can be written in terms of an orthogonal/unitary transformation \( \boldsymbol{U} \) +$$ +\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +$$ + +with \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I} \) or \( \boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I} \). + +

+Not all square matrices are diagonalizable. A matrix like the one discussed above +$$ +\boldsymbol{X} = \begin{bmatrix} +1& -1 \\ +1& -1\\ +\end{bmatrix} +$$ + +is not diagonalizable, it is a so-called defective matrix. It is easy to see that the condition +\( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) is not fulfilled. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs043.html b/doc/pub/week35/html/._week35-bs043.html new file mode 100644 index 000000000..5b321c1dd --- /dev/null +++ b/doc/pub/week35/html/._week35-bs043.html @@ -0,0 +1,490 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The SVD, a Fantastic Algorithm

+ +

+However, and this is the strength of the SVD algorithm, any general +matrix \( \boldsymbol{X} \) can be decomposed in terms of a diagonal matrix and +two orthogonal/unitary matrices. The Singular Value Decompostion +(SVD) theorem +states that a general \( m\times n \) matrix \( \boldsymbol{X} \) can be written in +terms of a diagonal matrix \( \boldsymbol{\Sigma} \) of dimensionality \( m\times n \) +and two orthognal matrices \( \boldsymbol{U} \) and \( \boldsymbol{V} \), where the first has +dimensionality \( m \times m \) and the last dimensionality \( n\times n \). +We have then + +$$ +\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T +$$ + +

+As an example, the above defective matrix can be decomposed as + +$$ +\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +$$ + +

+with eigenvalues \( \sigma_1=2 \) and \( \sigma_2=0 \). +The SVD exits always! + +

+The SVD +decomposition (singular values) gives eigenvalues +\( \sigma_i\geq\sigma_{i+1} \) for all \( i \) and for dimensions larger than \( i=p \), the +eigenvalues (singular values) are zero. + +

+In the general case, where our design matrix \( \boldsymbol{X} \) has dimension +\( n\times p \), the matrix is thus decomposed into an \( n\times n \) +orthogonal matrix \( \boldsymbol{U} \), a \( p\times p \) orthogonal matrix \( \boldsymbol{V} \) +and a diagonal matrix \( \boldsymbol{\Sigma} \) with \( r=\mathrm{min}(n,p) \) +singular values \( \sigma_i\geq 0 \) on the main diagonal and zeros filling +the rest of the matrix. There are at most \( p \) singular values +assuming that \( n > p \). In our regression examples for the nuclear +masses and the equation of state this is indeed the case, while for +the Ising model we have \( p > n \). These are often cases that lead to +near singular or singular matrices. + +

+The columns of \( \boldsymbol{U} \) are called the left singular vectors while the columns of \( \boldsymbol{V} \) are the right singular vectors. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs044.html b/doc/pub/week35/html/._week35-bs044.html new file mode 100644 index 000000000..52f9c4fad --- /dev/null +++ b/doc/pub/week35/html/._week35-bs044.html @@ -0,0 +1,463 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Economy-size SVD

+ +

+If we assume that \( n > p \), then our matrix \( \boldsymbol{U} \) has dimension \( n +\times n \). The last \( n-p \) columns of \( \boldsymbol{U} \) become however +irrelevant in our calculations since they are multiplied with the +zeros in \( \boldsymbol{\Sigma} \). + +

+The economy-size decomposition removes extra rows or columns of zeros +from the diagonal matrix of singular values, \( \boldsymbol{\Sigma} \), along with the columns +in either \( \boldsymbol{U} \) or \( \boldsymbol{V} \) that multiply those zeros in the expression. +Removing these zeros and columns can improve execution time +and reduce storage requirements without compromising the accuracy of +the decomposition. + +

+If \( n > p \), we keep only the first \( p \) columns of \( \boldsymbol{U} \) and \( \boldsymbol{\Sigma} \) has dimension \( p\times p \). +If \( p > n \), then only the first \( n \) columns of \( \boldsymbol{V} \) are computed and \( \boldsymbol{\Sigma} \) has dimension \( n\times n \). +The \( n=p \) case is obvious, we retain the full SVD. +In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs045.html b/doc/pub/week35/html/._week35-bs045.html new file mode 100644 index 000000000..79a976c09 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs045.html @@ -0,0 +1,486 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Codes for the SVD

+ +

+ + +

import numpy as np
+# SVD inversion
+def SVD(A):
+    ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
+    SVD is numerically more stable than the inversion algorithms provided by
+    numpy and scipy.linalg at the cost of being slower.
+    '''
+    U, S, VT = np.linalg.svd(A,full_matrices=True)
+    print('test U')
+    print( (np.transpose(U) @ U - U @np.transpose(U)))
+    print('test VT')
+    print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))
+    print(U)
+    print(S)
+    print(VT)
+
+    D = np.zeros((len(U),len(VT)))
+    for i in range(0,len(VT)):
+        D[i,i]=S[i]
+    return U @ D @ VT
+
+
+X = np.array([ [1.0,-1.0], [1.0,-1.0]])
+#X = np.array([[1, 2], [3, 4], [5, 6]])
+
+print(X)
+C = SVD(X)
+# Print the difference between the original matrix and the SVD one
+print(C-X)
+
+

+The matrix \( \boldsymbol{X} \) has columns that are linearly dependent. The first +column is the row-wise sum of the other two columns. The rank of a +matrix (the column rank) is the dimension of space spanned by the +column vectors. The rank of the matrix is the number of linearly +independent columns, in this case just \( 2 \). We see this from the +singular values when running the above code. Running the standard +inversion algorithm for matrix inversion with \( \boldsymbol{X}^T\boldsymbol{X} \) results +in the program terminating due to a singular matrix. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs046.html b/doc/pub/week35/html/._week35-bs046.html new file mode 100644 index 000000000..0c35a1028 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs046.html @@ -0,0 +1,459 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Note about SVD Calculations

+ +

+The \( U \), \( S \), and \( V \) matrices returned from the svd() function +cannot be multiplied directly. + +

+As you can see from the code, the \( S \) vector must be converted into a +diagonal matrix. This may cause a problem as the size of the matrices +do not fit the rules of matrix multiplication, where the number of +columns in a matrix must match the number of rows in the subsequent +matrix. + +

+If you wish to include the zero singular values, you will need to +resize the matrices and set up a diagonal matrix as done in the above +example + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs047.html b/doc/pub/week35/html/._week35-bs047.html new file mode 100644 index 000000000..a82021dfd --- /dev/null +++ b/doc/pub/week35/html/._week35-bs047.html @@ -0,0 +1,449 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Friday September 3

+ +

+Video of Lecture from 2020 and handwritten notes + +

+More material will be added here, see handwritten notes also. Note that this material will be cleaned up after the lecture of Friday September 3. See the handwritten notes from Friday's lecture at https://github.com/CompPhysics/MachineLearning/tree/master/doc/HandWrittenNotes/2021. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs048.html b/doc/pub/week35/html/._week35-bs048.html new file mode 100644 index 000000000..ecfb4eb31 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs048.html @@ -0,0 +1,480 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Mathematics of the SVD and implications

+ +

+Let us take a closer look at the mathematics of the SVD and the various implications for machine learning studies. + +

+Our starting point is our design matrix \( \boldsymbol{X} \) of dimension \( n\times p \) +$$ +\boldsymbol{X}=\begin{bmatrix} +x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ +x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ +x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ +\dots & \dots & \dots & \dots \dots & \dots \\ +x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ +x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ +\end{bmatrix}. +$$ + +

+We can SVD decompose our matrix as +$$ +\boldsymbol{X}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +$$ + +where \( \boldsymbol{U} \) is an orthogonal matrix of dimension \( n\times n \), meaning that \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{U}^T\boldsymbol{U}=\boldsymbol{I}_n \). Here \( \boldsymbol{I}_n \) is the unit matrix of dimension \( n \times n \). + +

+Similarly, \( \boldsymbol{V} \) is an orthogonal matrix of dimension \( p\times p \), meaning that \( \boldsymbol{V}\boldsymbol{V}^T=\boldsymbol{V}^T\boldsymbol{V}=\boldsymbol{I}_p \). Here \( \boldsymbol{I}_n \) is the unit matrix of dimension \( p \times p \). + +

+Finally \( \boldsymbol{\Sigma} \) contains the singular values \( \sigma_i \). This matrix has dimension \( n\times p \) and the singular values \( \sigma_i \) are all positive. The non-zero values are ordered in descending order, that is + +$$ +\sigma_0 > \sigma_1 > \sigma_2 > \dots > \sigma_{p-1} > 0. +$$ + +

+All values beyond \( p-1 \) are all zero. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs049.html b/doc/pub/week35/html/._week35-bs049.html new file mode 100644 index 000000000..d5a9bf5cb --- /dev/null +++ b/doc/pub/week35/html/._week35-bs049.html @@ -0,0 +1,501 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Example Matrix

+ +

+As an example, consider the following \( 3\times 2 \) example for the matrix \( \boldsymbol{\Sigma} \) + +$$ +\boldsymbol{\Sigma}= +\begin{bmatrix} +2& 0 \\ +0 & 1 \\ +0 & 0 \\ +\end{bmatrix} +$$ + +

+The singular values are \( \sigma_0=2 \) and \( \sigma_1=1 \). It is common to rewrite the matrix \( \boldsymbol{\Sigma} \) as + +$$ +\boldsymbol{\Sigma}= +\begin{bmatrix} +\boldsymbol{\tilde{\Sigma}}\\ +\boldsymbol{0}\\ +\end{bmatrix}, +$$ + +

+where +$$ +\boldsymbol{\tilde{\Sigma}}= +\begin{bmatrix} +2& 0 \\ +0 & 1 \\ +\end{bmatrix}, +$$ + +contains only the singular values. Note also (and we will use this below) that + +$$ +\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}= +\begin{bmatrix} +4& 0 \\ +0 & 1 \\ +\end{bmatrix}, +$$ + +which is a \( 2\times 2 \) matrix while +$$ +\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T= +\begin{bmatrix} +4& 0 & 0\\ +0 & 1 & 0\\ +0 & 0 & 0\\ +\end{bmatrix}, +$$ + +

+is a \( 3\times 3 \) matrix. The last row and column of this last matrix +contain only zeros. This will have important consequences for our SVD +decomposition of the design matrix. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs050.html b/doc/pub/week35/html/._week35-bs050.html new file mode 100644 index 000000000..f274153b0 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs050.html @@ -0,0 +1,495 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Setting up the Matrix to be inverted

+ +

+The matrix that may cause problems for us is \( \boldsymbol{X}^T\boldsymbol{X} \). Using the SVD we can rewrite this matrix as + +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +$$ + +and using the orthogonality of the matrix \( \boldsymbol{U} \) we have + +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. +$$ + +We define \( \boldsymbol{\Sigma}^T\boldsymbol{\Sigma}=\tilde{\boldsymbol{\Sigma}}^2 \) which is a diagonal matrix containing only the singular values squared. It has dimensionality \( p \times p \). + +

+This means, using the orthogonality of \( \boldsymbol{V} \), that we get + +$$ +\boldsymbol{X}^T\boldsymbol{X}=\tilde{\boldsymbol{\Sigma}}^2. +$$ + +

+We can now insert the result for the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) into our equation for ordinary least squares where + +$$ +\tilde{y}_{\mathrm{OLS}}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +and using our SVD decomposition of \( \boldsymbol{X} \) we have + +$$ +\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\tilde{\boldsymbol{\Sigma}}^{-2}\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{y}, +$$ + +which gives us, using the orthogonality of the matrices \( \boldsymbol{U} \) and \( \boldsymbol{V} \), + +$$ +\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_i\boldsymbol{u}^T_j\boldsymbol{y}, +$$ + +

+Note here that when we perform the multiplication of the various matrices, the orthogonal vectors of the matrix \( \boldsymbol{U} \) +$$ +\boldsymbol{U}=[\boldsymbol{u}_0,\boldsymbol{u}_1,\dots,\boldsymbol{u}_{n-1}], +$$ + +that belong to \( i>p-1 \), result in only zeros when we perform the multiplications. This means that the sum above has non-zero elements only up to \( i=p-1 \). This corresponds also to the number of singular values (these are all non-zero). + +

+It means that the ordinary least square model (with the optimal parameters) \( \boldsymbol{\tilde{y}} \), corresponds to an orthogonal transformation of the output (or target) vector \( \boldsymbol{y} \) by the vectors of the matrix \( \boldsymbol{U} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs051.html b/doc/pub/week35/html/._week35-bs051.html new file mode 100644 index 000000000..1333ac3c8 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs051.html @@ -0,0 +1,491 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Further properties (important for our analyses later)

+ +

+Let us study again \( \boldsymbol{X}^T\boldsymbol{X} \) in terms of our SVD, +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. +$$ + +

+If we now multiply from the right with \( \boldsymbol{V} \) (using the orthogonality of \( \boldsymbol{V} \)) we get +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}. +$$ + +This means the vectors \( \boldsymbol{v}_i \) of the orthogonal matrix \( \boldsymbol{V} \) are the eigenvectors of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) +with eigenvalues given by the singular values squared, that is +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2. +$$ + +

+Similarly, if we use the SVD decomposition for the matrix \( \boldsymbol{X}\boldsymbol{X}^T \), we have +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T\boldsymbol{U}^T. +$$ + +

+If we now multiply from the right with \( \boldsymbol{U} \) (using the orthogonality of \( \boldsymbol{U} \)) we get +$$ +\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{U}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T. +$$ + +This means the vectors \( \boldsymbol{u}_i \) of the orthogonal matrix \( \boldsymbol{U} \) are the eigenvectors of the matrix \( \boldsymbol{X}\boldsymbol{X}^T \) +with eigenvalues given by the singular values squared, that is +$$ +\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{u}_i=\boldsymbol{u}_i\sigma_i^2. +$$ + +

+Important note: we have defined our design matrix \( \boldsymbol{X} \) to be an +\( n\times p \) matrix. In most supervised learning cases we have that \( n +\ge p \), and quite often we have \( n >> p \). For linear algebra based methods like ordinary least squares or Ridge regression, this leads to a matrix \( \boldsymbol{X}^T\boldsymbol{X} \) which is small and thereby easier to handle from a computational point of view (in terms of number of floating point operations). + +

+In our lectures, the number of columns will +always refer to the number of features in our data set, while the +number of rows represents the number of data inputs. Note that in +other texts you may find the opposite notation. This has consequences +for the definition of for example the covariance matrix and its relation to the SVD. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs052.html b/doc/pub/week35/html/._week35-bs052.html new file mode 100644 index 000000000..b81ca182d --- /dev/null +++ b/doc/pub/week35/html/._week35-bs052.html @@ -0,0 +1,470 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Meet the Covariance Matrix

+ +

+Before we move on to a discussion of Ridge and Lasso regression, we want to show an important example of the above. + +

+We have already noted that the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) in ordinary +least squares is proportional to the second derivative of the cost +function, that is we have + +$$ +\frac{\partial^2 C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}^T\partial \boldsymbol{\beta}} =\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}. +$$ + +This quantity defines was what is called the Hessian matrix (the second derivative of a function we want to optimize). + +

+The Hessian matrix plays an important role and is defined in this course as + +$$ +\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}. +$$ + +

+The Hessian matrix for ordinary least squares is also proportional to +the covariance matrix. This means also that we can use the SVD to find +the eigenvalues of the covariance matrix and the Hessian matrix in +terms of the singular values. Let us develop these arguments, as they will play an important role in our machine learning studies. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs053.html b/doc/pub/week35/html/._week35-bs053.html new file mode 100644 index 000000000..26eab7d24 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs053.html @@ -0,0 +1,483 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Introducing the Covariance and Correlation functions

+ +

+Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about +the definition of the covariance and the correlation function. These are quantities that play a central role in machine learning methods. + +

+Suppose we have defined two vectors +\( \hat{x} \) and \( \hat{y} \) with \( n \) elements each. The covariance matrix \( \boldsymbol{C} \) is defined as +$$ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\ + \end{bmatrix}, +$$ + +where for example +$$ +\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). +$$ + +With this definition and recalling that the variance is defined as +$$ +\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2, +$$ + +we can rewrite the covariance matrix as +$$ +\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\ + \end{bmatrix}. +$$ + +

+Note: we have used \( 1/n \) in the above definitions of the sample variance and covariance. We assume then that we can calculate the exact mean value. +What you will find in essentially all statistics texts are equations +with a factor \( 1/(n-1) \). This is called Bessel's correction. This +method corrects the bias in the estimation of the population variance +and covariance. It also partially corrects the bias in the estimation +of the population standard deviation. If you use a library like +Scikit-Learn or nunmpy's function calculate the covariance, this +quantity will be computed with a factor \( 1/(n-1) \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs054.html b/doc/pub/week35/html/._week35-bs054.html new file mode 100644 index 000000000..af9600ba6 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs054.html @@ -0,0 +1,469 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Covariance and Correlation Matrix

+ +

+The covariance takes values between zero and infinity and may thus +lead to problems with loss of numerical precision for particularly +large values. It is common to scale the covariance matrix by +introducing instead the correlation matrix defined via the so-called +correlation function + +$$ +\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}. +$$ + +

+The correlation function is then given by values \( \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] +\in [-1,1] \). This avoids eventual problems with too large values. We +can then define the correlation matrix for the two vectors \( \boldsymbol{x} \) +and \( \boldsymbol{y} \) as + +$$ +\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\ + \mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\ + \end{bmatrix}, +$$ + +

+In the above example this is the function we constructed using pandas. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs055.html b/doc/pub/week35/html/._week35-bs055.html new file mode 100644 index 000000000..d669aa053 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs055.html @@ -0,0 +1,499 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Function and Design/Feature Matrix

+ +

+In our derivation of the various regression algorithms like Ordinary Least Squares or Ridge regression +we defined the design/feature matrix \( \boldsymbol{X} \) as + +$$ +\boldsymbol{X}=\begin{bmatrix} +x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\ +x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\ +x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\ +\dots & \dots & \dots & \dots \dots & \dots \\ +x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\ +x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\ +\end{bmatrix}, +$$ + +with \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \), with the predictors/features \( p \) refering to the column numbers and the +entries \( n \) being the row elements. +We can rewrite the design/feature matrix in terms of its column vectors as +$$ +\boldsymbol{X}=\begin{bmatrix} \boldsymbol{x}_0 & \boldsymbol{x}_1 & \boldsymbol{x}_2 & \dots & \dots & \boldsymbol{x}_{p-1}\end{bmatrix}, +$$ + +with a given vector +$$ +\boldsymbol{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}. +$$ + +

+With these definitions, we can now rewrite our \( 2\times 2 \) +correlation/covariance matrix in terms of a moe general design/feature +matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \). This leads to a \( p\times p \) +covariance matrix for the vectors \( \boldsymbol{x}_i \) with \( i=0,1,\dots,p-1 \) + +$$ +\boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix} +\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\ +\end{bmatrix}, +$$ + +and the correlation matrix +$$ +\boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix} +1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\ +\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\dots & \dots & \dots & \dots & \dots & \dots \\ +\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\ +\end{bmatrix}, +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs056.html b/doc/pub/week35/html/._week35-bs056.html new file mode 100644 index 000000000..e0bbf6598 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs056.html @@ -0,0 +1,479 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Covariance Matrix Examples

+ +

+The Numpy function np.cov calculates the covariance elements using +the factor \( 1/(n-1) \) instead of \( 1/n \) since it assumes we do not have +the exact mean values. The following simple function uses the +np.vstack function which takes each vector of dimension \( 1\times n \) +and produces a \( 2\times n \) matrix \( \boldsymbol{W} \) + +

+Note that this assumes you have the features as the rows, and the inputs as columns, that is +$$ +\boldsymbol{W} = \begin{bmatrix} x_0 & x_1 & x_2 & \dots & x_{n-2} & x_{n-1} \\ + y_0 & y_1 & y_2 & \dots & y_{n-2} & y_{n-1} \\ + \end{bmatrix}, +$$ + +

+which in turn is converted into into the \( 2\times 2 \) covariance matrix +\( \boldsymbol{C} \) via the Numpy function np.cov(). We note that we can also calculate +the mean value of each set of samples \( \boldsymbol{x} \) etc using the Numpy +function np.mean(x). We can also extract the eigenvalues of the +covariance matrix through the np.linalg.eig() function. + +

+ + +

# Importing various packages
+import numpy as np
+n = 100
+x = np.random.normal(size=n)
+print(np.mean(x))
+y = 4+3*x+np.random.normal(size=n)
+print(np.mean(y))
+W = np.vstack((x, y))
+C = np.cov(W)
+print(C)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs057.html b/doc/pub/week35/html/._week35-bs057.html new file mode 100644 index 000000000..1357f43e3 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs057.html @@ -0,0 +1,483 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Matrix

+ +

+The previous example can be converted into the correlation matrix by +simply scaling the matrix elements with the variances. We should also +subtract the mean values for each column. This leads to the following +code which sets up the correlations matrix for the previous example in +a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the \( 2\times 2 \) correlation matrix (since we have only two vectors). + +

+ + +

import numpy as np
+n = 100
+# define two vectors                                                                                           
+x = np.random.random(size=n)
+y = 4+3*x+np.random.normal(size=n)
+#scaling the x and y vectors                                                                                   
+x = x - np.mean(x)
+y = y - np.mean(y)
+variance_x = np.sum(x@x)/n
+variance_y = np.sum(y@y)/n
+print(variance_x)
+print(variance_y)
+cov_xy = np.sum(x@y)/n
+cov_xx = np.sum(x@x)/n
+cov_yy = np.sum(y@y)/n
+C = np.zeros((2,2))
+C[0,0]= cov_xx/variance_x
+C[1,1]= cov_yy/variance_y
+C[0,1]= cov_xy/np.sqrt(variance_y*variance_x)
+C[1,0]= C[0,1]
+print(C)
+
+

+We see that the matrix elements along the diagonal are one as they +should be and that the matrix is symmetric. Furthermore, diagonalizing +this matrix we easily see that it is a positive definite matrix. + +

+The above procedure with numpy can be made more compact if we use pandas. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs058.html b/doc/pub/week35/html/._week35-bs058.html new file mode 100644 index 000000000..ebeaa6b31 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs058.html @@ -0,0 +1,466 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Matrix with Pandas

+ +

+We whow here how we can set up the correlation matrix using pandas, as done in this simple code +

+ + +

import numpy as np
+import pandas as pd
+n = 10
+x = np.random.normal(size=n)
+x = x - np.mean(x)
+y = 4+3*x+np.random.normal(size=n)
+y = y - np.mean(y)
+# Note that we transpose the matrix in order to stay with our ordering n x p
+X = (np.vstack((x, y))).T
+print(X)
+Xpd = pd.DataFrame(X)
+print(Xpd)
+correlation_matrix = Xpd.corr()
+print(correlation_matrix)
+
+

+We expand this model to the Franke function discussed above. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs059.html b/doc/pub/week35/html/._week35-bs059.html new file mode 100644 index 000000000..45b089c86 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs059.html @@ -0,0 +1,501 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Correlation Matrix with Pandas and the Franke function

+ +

+ + +

# Common imports
+import numpy as np
+import pandas as pd
+
+
+def FrankeFunction(x,y):
+	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+	return term1 + term2 + term3 + term4
+
+
+def create_X(x, y, n ):
+	if len(x.shape) > 1:
+		x = np.ravel(x)
+		y = np.ravel(y)
+
+	N = len(x)
+	l = int((n+1)*(n+2)/2)		# Number of elements in beta
+	X = np.ones((N,l))
+
+	for i in range(1,n+1):
+		q = int((i)*(i+1)/2)
+		for k in range(i+1):
+			X[:,q+k] = (x**(i-k))*(y**k)
+
+	return X
+
+
+# Making meshgrid of datapoints and compute Franke's function
+n = 4
+N = 100
+x = np.sort(np.random.uniform(0, 1, N))
+y = np.sort(np.random.uniform(0, 1, N))
+z = FrankeFunction(x, y)
+X = create_X(x, y, n=n)    
+
+Xpd = pd.DataFrame(X)
+# subtract the mean values and set up the covariance matrix
+Xpd = Xpd - Xpd.mean()
+covariance_matrix = Xpd.cov()
+print(covariance_matrix)
+
+

+We note here that the covariance is zero for the first rows and +columns since all matrix elements in the design matrix were set to one +(we are fitting the function in terms of a polynomial of degree \( n \)). + +

+This means that the variance for these elements will be zero and will +cause problems when we set up the correlation matrix. We can simply +drop these elements and construct a correlation +matrix without these elements. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs060.html b/doc/pub/week35/html/._week35-bs060.html new file mode 100644 index 000000000..91c082eb0 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs060.html @@ -0,0 +1,479 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Rewriting the Covariance and/or Correlation Matrix

+ +

+We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix \( \boldsymbol{X} \) as +$$ +\boldsymbol{C}[\boldsymbol{x}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}= \mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}]. +$$ + +

+To see this let us simply look at a design matrix \( \boldsymbol{X}\in {\mathbb{R}}^{2\times 2} \) +$$ +\boldsymbol{X}=\begin{bmatrix} +x_{00} & x_{01}\\ +x_{10} & x_{11}\\ +\end{bmatrix}=\begin{bmatrix} +\boldsymbol{x}_{0} & \boldsymbol{x}_{1}\\ +\end{bmatrix}. +$$ + +

+If we then compute the expectation value (note the \( 1/n \) factor instead of \( 1/(n-1) \)) +$$ +\mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}=\frac{1}{n}\begin{bmatrix} +x_{00}^2+x_{10}^2 & x_{00}x_{01}+x_{10}x_{11}\\ +x_{01}x_{00}+x_{11}x_{10} & x_{01}^2+x_{11}^2\\ +\end{bmatrix}, +$$ + +which is just +$$ +\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]=\begin{bmatrix} \mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] \\ + \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] \\ + \end{bmatrix}, +$$ + +where we wrote $$\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]$$ to indicate that this is the covariance of the vectors \( \boldsymbol{x} \) of the design/feature matrix \( \boldsymbol{X} \). + +

+It is easy to generalize this to a matrix \( \boldsymbol{X}\in {\mathbb{R}}^{n\times p} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs061.html b/doc/pub/week35/html/._week35-bs061.html new file mode 100644 index 000000000..cd0033439 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs061.html @@ -0,0 +1,472 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Linking with the SVD

+ +

+We saw earlier that +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. +$$ + +Since the matrices here have dimension \( p\times p \), with \( p \) corresponding to the singular values, we defined earlier the matrix +$$ +\boldsymbol{\Sigma}^T\boldsymbol{\Sigma} = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix}, +$$ + +where the tilde-matrix \( \tilde{\boldsymbol{\Sigma}} \) is a matrix of dimension \( p\times p \) containing only the singular values \( \sigma_i \), that is + +$$ +\tilde{\boldsymbol{\Sigma}}=\begin{bmatrix} \sigma_0 & 0 & 0 & \dots & 0 & 0 \\ + 0 & \sigma_1 & 0 & \dots & 0 & 0 \\ + 0 & 0 & \sigma_2 & \dots & 0 & 0 \\ + 0 & 0 & 0 & \dots & \sigma_{p-2} & 0 \\ + 0 & 0 & 0 & \dots & 0 & \sigma_{p-1} \\ +\end{bmatrix}, +$$ + +meaning we can write +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2\boldsymbol{V}^T. +$$ + +Multiplying from the right with \( \boldsymbol{V} \) (using the orthogonality of \( \boldsymbol{V} \)) we get +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs062.html b/doc/pub/week35/html/._week35-bs062.html new file mode 100644 index 000000000..7cda55c54 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs062.html @@ -0,0 +1,477 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

What does it mean?

+ +

+This means the vectors \( \boldsymbol{v}_i \) of the orthogonal matrix \( \boldsymbol{V} \) +are the eigenvectors of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) with eigenvalues +given by the singular values squared, that is + +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2. +$$ + +

+In other words, each non-zero singular value of \( \boldsymbol{X} \) is a positive +square root of an eigenvalue of \( \boldsymbol{X}^T\boldsymbol{X} \). It means also that +the columns of \( \boldsymbol{V} \) are the eigenvectors of +\( \boldsymbol{X}^T\boldsymbol{X} \). Since we have ordered the singular values of +\( \boldsymbol{X} \) in a descending order, it means that the column vectors +\( \boldsymbol{v}_i \) are hierarchically ordered by how much correlation they +encode from the columns of \( \boldsymbol{X} \). + +

+Note that these are also the eigenvectors and eigenvalues of the +Hessian matrix. + +

+If we now recall the definition of the covariance matrix (not using +Bessel's correction) we have + +$$ +\boldsymbol{C}[\boldsymbol{X}]=\frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}, +$$ + +

+meaning that every squared non-singular value of \( \boldsymbol{X} \) divided by \( n \) ( +the number of samples) are the eigenvalues of the covariance +matrix. Every singular value of \( \boldsymbol{X} \) is thus a positive square +root of an eigenvalue of \( \boldsymbol{X}^T\boldsymbol{X} \). If the matrix \( \boldsymbol{X} \) is +self-adjoint, the singular values of \( \boldsymbol{X} \) are equal to the +absolute value of the eigenvalues of \( \boldsymbol{X} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs063.html b/doc/pub/week35/html/._week35-bs063.html new file mode 100644 index 000000000..ef73e2446 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs063.html @@ -0,0 +1,472 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

And finally \( \boldsymbol{X}\boldsymbol{X}^T \)

+ +

+For \( \boldsymbol{X}\boldsymbol{X}^T \) we found + +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{U}^T. +$$ + +Since the matrices here have dimension \( n\times n \), we have +$$ +\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \boldsymbol{0}\\ \end{bmatrix}=\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}, +$$ + +leading to +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}\boldsymbol{U}^T. +$$ + +

+Multiplying with \( \boldsymbol{U} \) from the right gives us the eigenvalue problem +$$ +(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U}=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}. +$$ + +

+It means that the eigenvalues of \( \boldsymbol{X}\boldsymbol{X}^T \) are again given by +the non-zero singular values plus now a series of zeros. The column +vectors of \( \boldsymbol{U} \) are the eigenvectors of \( \boldsymbol{X}\boldsymbol{X}^T \) and +measure how much correlations are contained in the rows of \( \boldsymbol{X} \). + +

+Since we will mainly be interested in the correlations among the features +of our data (the columns of \( \boldsymbol{X} \), the quantity of interest for us are the non-zero singular +values and the column vectors of \( \boldsymbol{V} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs064.html b/doc/pub/week35/html/._week35-bs064.html new file mode 100644 index 000000000..22ae7ccbf --- /dev/null +++ b/doc/pub/week35/html/._week35-bs064.html @@ -0,0 +1,490 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Ridge and LASSO Regression

+ +

+Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is +our optimization problem is +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}. +$$ + +or we can state it as +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2, +$$ + +where we have used the definition of a norm-2 vector, that is +$$ +\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}. +$$ + +

+By minimizing the above equation with respect to the parameters +\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the +parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by +defining a new cost function to be optimized, that is + +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2 +$$ + +

+which leads to the Ridge regression minimization problem where we +require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is +a finite number larger than zero. By defining + +$$ +C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1, +$$ + +

+we have a new optimization equation +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1 +$$ + +which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. + +

+Here we have defined the norm-1 as +$$ +\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs065.html b/doc/pub/week35/html/._week35-bs065.html new file mode 100644 index 000000000..aa769f0bb --- /dev/null +++ b/doc/pub/week35/html/._week35-bs065.html @@ -0,0 +1,494 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Deriving the Ridge Regression Equations

+ +

+Using the matrix-vector expression for Ridge regression and dropping the parameter \( 1/n \) in front of the standard means squared error equation, we have + +$$ +C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta}, +$$ + +and +taking the derivatives with respect to \( \boldsymbol{\beta} \) we obtain then +a slightly modified matrix inversion problem which for finite values +of \( \lambda \) does not suffer from singularity problems. We obtain +the optimal parameters +$$ +\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +

+with \( \boldsymbol{I} \) being a \( p\times p \) identity matrix with the constraint that + +$$ +\sum_{i=0}^{p-1} \beta_i^2 \leq t, +$$ + +

+with \( t \) a finite positive number. + +

+When we compare this with the ordinary least squares result we have +$$ +\hat{\boldsymbol{\beta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). + +

+We see that Ridge regression is nothing but the standard OLS with a +modified diagonal term added to \( \boldsymbol{X}^T\boldsymbol{X} \). The consequences, in +particular for our discussion of the bias-variance tradeoff are rather +interesting. We will see that for specific values of \( \lambda \), we may +even reduce the variance of the optimal parameters \( \boldsymbol{\beta} \). These topics and other related ones, will be discussed after the more linear algebra oriented analysis here. + +

+Using our insights about the SVD of the design matrix \( \boldsymbol{X} \) +We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix \( \boldsymbol{U} \) as +$$ +\tilde{\boldsymbol{y}}_{\mathrm{OLS}}=\boldsymbol{X}\boldsymbol{\beta} =\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}. +$$ + +

+For Ridge regression this becomes + +$$ +\tilde{\boldsymbol{y}}_{\mathrm{Ridge}}=\boldsymbol{X}\boldsymbol{\beta}_{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{\Sigma}^2\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y}, +$$ + +

+with the vectors \( \boldsymbol{u}_j \) being the columns of \( \boldsymbol{U} \) from the SVD of the matrix \( \boldsymbol{X} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs066.html b/doc/pub/week35/html/._week35-bs066.html new file mode 100644 index 000000000..6fd75b053 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs066.html @@ -0,0 +1,452 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Interpreting the Ridge results

+ +

+Since \( \lambda \geq 0 \), it means that compared to OLS, we have + +$$ +\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1. +$$ + +

+Ridge regression finds the coordinates of \( \boldsymbol{y} \) with respect to the +orthonormal basis \( \boldsymbol{U} \), it then shrinks the coordinates by +\( \frac{\sigma_j^2}{\sigma_j^2+\lambda} \). Recall that the SVD has +eigenvalues ordered in a descending way, that is \( \sigma_i \geq +\sigma_{i+1} \). + +

+For small eigenvalues \( \sigma_i \) it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs067.html b/doc/pub/week35/html/._week35-bs067.html new file mode 100644 index 000000000..0dd6aafb2 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs067.html @@ -0,0 +1,466 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

More interpretations

+ +

+For the sake of simplicity, let us assume that the design matrix is orthonormal, that is + +$$ +\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}. +$$ + +

+In this case the standard OLS results in +$$ +\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y}, +$$ + +

+and + +$$ +\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}}, +$$ + +

+that is the Ridge estimator scales the OLS estimator by the inverse of a factor \( 1+\lambda \), and +the Ridge estimator converges to zero when the hyperparameter goes to +infinity. + +

+We will come back to more interpreations after we have gone through some of the statistical analysis part. + +

+For more discussions of Ridge and Lasso regression, Wessel van Wieringen's article is highly recommended. +Similarly, Mehta et al's article is also recommended. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs068.html b/doc/pub/week35/html/._week35-bs068.html new file mode 100644 index 000000000..a07d68a26 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs068.html @@ -0,0 +1,459 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Deriving the Lasso Regression Equations

+ +

+Using the matrix-vector expression for Lasso regression and dropping the parameter \( 1/n \) in front of the standard means squared error equation, we have the following cost function + +$$ +C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_1, +$$ + +

+Taking the derivative with respect to \( \boldsymbol{\beta} \) and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty) +$$ +\frac{d \vert \beta\vert}{d \boldsymbol{\beta}}=\mathrm{sgn}(\boldsymbol{\beta})=\left\{\begin{array}{cc} 1 & \beta > 0 \\ 0 & \beta =0\\-1 & \beta < 0, \end{array}\right. +$$ + +we have that the derivative of the cost function is + +$$ +\frac{\partial C(\boldsymbol{X},\boldsymbol{\beta})}{\partial \boldsymbol{\beta}}=-2\boldsymbol{X}^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})+\lambda sgn(\boldsymbol{\beta})=0, +$$ + +and reordering we have +$$ +\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta})+\lambda sgn(\boldsymbol{\beta})=2\boldsymbol{X}^T(\boldsymbol{y}. +$$ + +This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package CVXOPT. We will discuss this later. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs069.html b/doc/pub/week35/html/._week35-bs069.html new file mode 100644 index 000000000..0068e4598 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs069.html @@ -0,0 +1,741 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Exercises for week 36, September 6-10

+ +

+The exercises here are meant to prepare you for work with project 1. The first exercise is a follow-up of exercise 2 from week 35 August 30-September 3). + +

+ + +

Exercise 1: Adding Ridge and Lasso Regression

+ +

+This exercise is a continuation of exercise 2 from exercise set 1 +(week 35, August 30-September 3). We will use the same function to +generate our data set, still staying with a simple function \( y(x) \) +which we want to fit using linear regression, but now extending the +analysis to include the Ridge and the Lasso regression methods. + +

+We will thus again generate our own dataset for a function \( y(x) \) where +\( x \in [0,1] \) and defined by random numbers computed with the uniform +distribution. The function \( y \) is a quadratic polynomial in \( x \) with +added stochastic noise according to the normal distribution \( \cal{N}(0,1) \). + +

+The following simple Python instructions define our \( x \) and \( y \) values (with 100 data points). +

+ + +

x = np.random.rand(100)
+y = 2.0+5*x*x+0.1*np.random.randn(100)
+
+

+Write your own code for the Ridge method (see chapter 3.4 of Hastie et al., equations (3.43) and (3.44)) and compute the parametrization for different values of \( \lambda \). Compare and analyze your results with those from exercise 3. Study the dependence on \( \lambda \) while also varying the strength of the noise in your expression for \( y(x) \). + +

+The code here allows you to perform your own Ridge calculation and +perform calculations for various values of the regularization +parameter \( \lambda \). This program can easily be extended upon. + +

+ + +

import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(3155)
+
+x = np.random.rand(100)
+y = 2.0+5*x*x+0.1*np.random.randn(100)
+
+# number of features p (here degree of polynomial
+p = 3
+#  The design matrix now as function of a given polynomial
+X = np.zeros((len(x),p))
+X[:,0] = 1.0
+X[:,1] = x
+X[:,2] = x*x
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+# matrix inversion to find beta
+OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
+print(OLSbeta)
+# and then make the prediction
+ytildeOLS = X_train @ OLSbeta
+print("Training R2 for OLS")
+print(R2(y_train,ytildeOLS))
+print("Training MSE for OLS")
+print(MSE(y_train,ytildeOLS))
+ypredictOLS = X_test @ OLSbeta
+print("Test R2 for OLS")
+print(R2(y_test,ypredictOLS))
+print("Test MSE OLS")
+print(MSE(y_test,ypredictOLS))
+
+# Repeat now for Ridge regression and various values of the regularization parameter
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 20
+MSEPredict = np.zeros(nlambdas)
+MSETrain = np.zeros(nlambdas)
+lambdas = np.logspace(-4, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
+    # and then make the prediction
+    ytildeRidge = X_train @ Ridgebeta
+    ypredictRidge = X_test @ Ridgebeta
+    MSEPredict[i] = MSE(y_test,ypredictRidge)
+    MSETrain[i] = MSE(y_train,ytildeRidge)
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSETrain, label = 'MSE Ridge train')
+plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+

+Repeat the above but using the functionality of +Scikit-Learn. Compare your code with the results from +Scikit-Learn. Remember to run with the same random numbers for +generating \( x \) and \( y \). Observe also that when you compare with Scikit-Learn, you need to pay attention to how the intercept is dealt with. + +

+Finally, using Scikit-Learn or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as +$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n} +\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2, +$$ + +and the \( R^2 \) score function. +If \( \tilde{\hat{y}}_i \) is the predicted value of the \( i-th \) sample and \( y_i \) is the corresponding true value, then the score \( R^2 \) is defined as +$$ +R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2}, +$$ + +where we have defined the mean value of \( \hat{y} \) as +$$ +\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i. +$$ + +Discuss these quantities as functions of the variable \( \lambda \) in the Ridge and Lasso regression methods. + +

Exercise: Linear Regression for a two-dimensional function

+ +

+This is a longer exercise and the aim is to study in more detail various +regression methods, including the Ordinary Least Squares (OLS) method, +Ridge regression and finally Lasso regression. +This exercise forms a part of project 1. + +

+We will study how to fit polynomials to a specific +two-dimensional function called Franke's +function. This +is a function which has been widely used when testing various +interpolation and fitting algorithms. + +

+The Franke function, which is a weighted sum of four exponentials reads as follows +$$ +\begin{align*} +f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\ +&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }. +\end{align*} +$$ + +

+The function will be defined for \( x,y\in [0,1] \). Our first step will +be to perform an OLS regression analysis of this function, trying out +a polynomial fit with an \( x \) and \( y \) dependence of the form \( [x, y, +x^2, y^2, xy, \dots] \). We will fit a +function (for example a polynomial) of \( x \) and \( y \). Thereafter we +will repeat much of the same procedure using the Ridge and Lasso +regression methods, introducing thus a dependence on the bias +(penalty) \( \lambda \). + +

+The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it) +

+ + +

from mpl_toolkits.mplot3d import Axes3D
+import matplotlib.pyplot as plt
+from matplotlib import cm
+from matplotlib.ticker import LinearLocator, FormatStrFormatter
+import numpy as np
+from random import random, seed
+
+fig = plt.figure()
+ax = fig.gca(projection='3d')
+
+# Make data.
+x = np.arange(0, 1, 0.05)
+y = np.arange(0, 1, 0.05)
+x, y = np.meshgrid(x,y)
+
+
+def FrankeFunction(x,y):
+    term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+    term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+    term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+    term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+    return term1 + term2 + term3 + term4
+
+
+z = FrankeFunction(x, y)
+
+# Plot the surface.
+surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm,
+                       linewidth=0, antialiased=False)
+
+# Customize the z axis.
+ax.set_zlim(-0.10, 1.40)
+ax.zaxis.set_major_locator(LinearLocator(10))
+ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
+
+# Add a color bar which maps values to colors.
+fig.colorbar(surf, shrink=0.5, aspect=5)
+
+plt.show()
+
+

+We will generate our own dataset for a function +\( \mathrm{FrankeFunction}(x,y) \) with \( x,y \in [0,1] \). The function +\( f(x,y) \) is the Franke function. You should explore also the addition +an added stochastic noise to this function using the normal +distribution \( \cal{N}(0,1) \). + +

+Write your own code (using either a matrix inversion or a singular +value decomposition from e.g., numpy ) or use your code and perform a standard least square regression +analysis using polynomials in \( x \) and \( y \) up to fifth order. You can use scikit-learn as well. + +

+Evaluate the Mean Squared error (MSE) + +$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n} +\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2, +$$ + +

+and the \( R^2 \) score function. If \( \tilde{\hat{y}}_i \) is the predicted +value of the \( i-th \) sample and \( y_i \) is the corresponding true value, +then the score \( R^2 \) is defined as + +$$ +R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2}, +$$ + +

+where we have defined the mean value of \( \hat{y} \) as + +$$ +\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i. +$$ + +

+You should split your data in train and test and also consider scaling the data. + +

+To set up the design matrix, the following code can be used +

+ + +

def FrankeFunction(x,y):
+	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+	return term1 + term2 + term3 + term4
+
+
+def create_X(x, y, n ):
+	if len(x.shape) > 1:
+		x = np.ravel(x)
+		y = np.ravel(y)
+
+	N = len(x)
+	l = int((n+1)*(n+2)/2)		# Number of elements in beta
+	X = np.ones((N,l))
+
+	for i in range(1,n+1):
+		q = int((i)*(i+1)/2)
+		for k in range(i+1):
+			X[:,q+k] = (x**(i-k))*(y**k)
+
+	return X
+
+
+# Making meshgrid of datapoints and compute Franke's function
+n = 5
+N = 1000
+x = np.sort(np.random.uniform(0, 1, N))
+y = np.sort(np.random.uniform(0, 1, N))
+z = FrankeFunction(x, y)
+X = create_X(x, y, n=n)    
+
+

+Write then your own code for the Ridge method or use Scikit-Learn. +Perform the same analysis as you did for ordinary Least Squares (for the same polynomials) but now for different values of \( \lambda \). Compare and +analyze your results with those obtained with ordinary Least Squares. Study the +dependence on \( \lambda \). + +

+This part is essentially a repeat of the previous ones, but now +with Lasso regression. Write either your own code or +use the functionalities of Scikit-Learn (recommended). +Give a +critical discussion of the three methods and a judgement of which +model fits the data best. + +

+ + +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs070.html b/doc/pub/week35/html/._week35-bs070.html new file mode 100644 index 000000000..2cf53e509 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs070.html @@ -0,0 +1,494 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Mathematical Properties

+ +

+There are several interesting mathematical properties which will be +relevant when we are going to discuss the differences between say +ordinary least squares (OLS) and Ridge regression. + +

+We have from OLS that the parameters of the linear approximation are given by +$$ +\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

+The matrix to invert can be rewritten in terms of our SVD decomposition as + +$$ +\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T. +$$ + +Using the orthogonality properties of \( \boldsymbol{U} \) we have + +$$ +\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T = \boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T, +$$ + +with \( \boldsymbol{D} \) being a diagonal matrix with values along the diagonal given by the singular values squared. + +

+This means that +$$ +(\boldsymbol{X}^T\boldsymbol{X})\boldsymbol{V} = \boldsymbol{V}\boldsymbol{D}, +$$ + +that is the eigenvectors of \( (\boldsymbol{X}^T\boldsymbol{X}) \) are given by the columns of the right singular matrix of \( \boldsymbol{X} \) and the eigenvalues are the squared singular values. It is easy to show (show this) that +$$ +(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U} = \boldsymbol{U}\boldsymbol{D}, +$$ + +that is, the eigenvectors of \( (\boldsymbol{X}\boldsymbol{X})^T \) are the columns of the left singular matrix and the eigenvalues are the same. + +

+Going back to our OLS equation we have +$$ +\boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}. +$$ + +We will come back to this expression when we discuss Ridge regression. + +$$ \tilde{y}^{OLS}={\bf X}\hat{\beta}^{OLS}=\sum_{j=1}^p {\bf u}_j{\bf u}_j^T{\bf y}$$ and for Ridge we have  + +$$ \tilde{y}^{Ridge}={\bf X}\hat{\beta}^{Ridge}=\sum_{j=1}^p {\bf u}_j\frac{\sigma_j^2}{\sigma_j^2+\lambda}{\bf u}_j^T{\bf y}$$ .  + +

+It is indeed the economy-sized SVD, note the summation runs up tp $$p$$ only and not $$n$$.  + +

+Here we have that $${\bf X} = {\bf U}{\bf \Sigma}{\bf V}^T$$, with $$\Sigma$$ being an $$ n\times p$$ matrix and $${\bf V}$$ being a $$ p\times p$$ matrix. We also have assumed here that $$ n > p$$.  + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week35/html/._week35-bs071.html b/doc/pub/week35/html/._week35-bs071.html new file mode 100644 index 000000000..e2c2b2680 --- /dev/null +++ b/doc/pub/week35/html/._week35-bs071.html @@ -0,0 +1,745 @@ + + + + + + + + +Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Exercises for week 36, September 6-10

+ +

+The exercises here are meant to prepare you for work with project 1. The first exercise is a follow-up of exercise 2 from week 35 August 30-September 3). + +

+ + +

Exercise 1: Adding Ridge and Lasso Regression

+ +

+This exercise is a continuation of exercise 2 from exercise set 1 +(week 35, August 30-September 3). We will use the same function to +generate our data set, still staying with a simple function \( y(x) \) +which we want to fit using linear regression, but now extending the +analysis to include the Ridge and the Lasso regression methods. + +

+We will thus again generate our own dataset for a function \( y(x) \) where +\( x \in [0,1] \) and defined by random numbers computed with the uniform +distribution. The function \( y \) is a quadratic polynomial in \( x \) with +added stochastic noise according to the normal distribution \( \cal{N}(0,1) \). + +

+The following simple Python instructions define our \( x \) and \( y \) values (with 100 data points). +

+ + +

x = np.random.rand(100)
+y = 2.0+5*x*x+0.1*np.random.randn(100)
+
+

+Write your own code for the Ridge method (see chapter 3.4 of Hastie et al., equations (3.43) and (3.44)) and compute the parametrization for different values of \( \lambda \). Compare and analyze your results with those from exercise 3. Study the dependence on \( \lambda \) while also varying the strength of the noise in your expression for \( y(x) \). + +

+The code here allows you to perform your own Ridge calculation and +perform calculations for various values of the regularization +parameter \( \lambda \). This program can easily be extended upon. + +

+ + +

import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(3155)
+
+x = np.random.rand(100)
+y = 2.0+5*x*x+0.1*np.random.randn(100)
+
+# number of features p (here degree of polynomial
+p = 3
+#  The design matrix now as function of a given polynomial
+X = np.zeros((len(x),p))
+X[:,0] = 1.0
+X[:,1] = x
+X[:,2] = x*x
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+# matrix inversion to find beta
+OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
+print(OLSbeta)
+# and then make the prediction
+ytildeOLS = X_train @ OLSbeta
+print("Training R2 for OLS")
+print(R2(y_train,ytildeOLS))
+print("Training MSE for OLS")
+print(MSE(y_train,ytildeOLS))
+ypredictOLS = X_test @ OLSbeta
+print("Test R2 for OLS")
+print(R2(y_test,ypredictOLS))
+print("Test MSE OLS")
+print(MSE(y_test,ypredictOLS))
+
+# Repeat now for Ridge regression and various values of the regularization parameter
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 20
+MSEPredict = np.zeros(nlambdas)
+MSETrain = np.zeros(nlambdas)
+lambdas = np.logspace(-4, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
+    # and then make the prediction
+    ytildeRidge = X_train @ Ridgebeta
+    ypredictRidge = X_test @ Ridgebeta
+    MSEPredict[i] = MSE(y_test,ypredictRidge)
+    MSETrain[i] = MSE(y_train,ytildeRidge)
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSETrain, label = 'MSE Ridge train')
+plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+

+Repeat the above but using the functionality of +Scikit-Learn. Compare your code with the results from +Scikit-Learn. Remember to run with the same random numbers for +generating \( x \) and \( y \). Observe also that when you compare with Scikit-Learn, you need to pay attention to how the intercept is dealt with. + +

+Finally, using Scikit-Learn or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as +$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n} +\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2, +$$ + +and the \( R^2 \) score function. +If \( \tilde{\hat{y}}_i \) is the predicted value of the \( i-th \) sample and \( y_i \) is the corresponding true value, then the score \( R^2 \) is defined as +$$ +R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2}, +$$ + +where we have defined the mean value of \( \hat{y} \) as +$$ +\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i. +$$ + +Discuss these quantities as functions of the variable \( \lambda \) in the Ridge and Lasso regression methods. + +

Exercise: Linear Regression for a two-dimensional function

+ +

+This is a longer exercise and the aim is to study in more detail various +regression methods, including the Ordinary Least Squares (OLS) method, +Ridge regression and finally Lasso regression. +This exercise forms a part of project 1. + +

+We will study how to fit polynomials to a specific +two-dimensional function called Franke's +function. This +is a function which has been widely used when testing various +interpolation and fitting algorithms. + +

+The Franke function, which is a weighted sum of four exponentials reads as follows +$$ +\begin{align*} +f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\ +&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }. +\end{align*} +$$ + +

+The function will be defined for \( x,y\in [0,1] \). Our first step will +be to perform an OLS regression analysis of this function, trying out +a polynomial fit with an \( x \) and \( y \) dependence of the form \( [x, y, +x^2, y^2, xy, \dots] \). We will fit a +function (for example a polynomial) of \( x \) and \( y \). Thereafter we +will repeat much of the same procedure using the Ridge and Lasso +regression methods, introducing thus a dependence on the bias +(penalty) \( \lambda \). + +

+The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it) +

+ + +

from mpl_toolkits.mplot3d import Axes3D
+import matplotlib.pyplot as plt
+from matplotlib import cm
+from matplotlib.ticker import LinearLocator, FormatStrFormatter
+import numpy as np
+from random import random, seed
+
+fig = plt.figure()
+ax = fig.gca(projection='3d')
+
+# Make data.
+x = np.arange(0, 1, 0.05)
+y = np.arange(0, 1, 0.05)
+x, y = np.meshgrid(x,y)
+
+
+def FrankeFunction(x,y):
+    term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+    term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+    term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+    term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+    return term1 + term2 + term3 + term4
+
+
+z = FrankeFunction(x, y)
+
+# Plot the surface.
+surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm,
+                       linewidth=0, antialiased=False)
+
+# Customize the z axis.
+ax.set_zlim(-0.10, 1.40)
+ax.zaxis.set_major_locator(LinearLocator(10))
+ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
+
+# Add a color bar which maps values to colors.
+fig.colorbar(surf, shrink=0.5, aspect=5)
+
+plt.show()
+
+

+We will generate our own dataset for a function +\( \mathrm{FrankeFunction}(x,y) \) with \( x,y \in [0,1] \). The function +\( f(x,y) \) is the Franke function. You should explore also the addition +an added stochastic noise to this function using the normal +distribution \( \cal{N}(0,1) \). + +

+Write your own code (using either a matrix inversion or a singular +value decomposition from e.g., numpy ) or use your code and perform a standard least square regression +analysis using polynomials in \( x \) and \( y \) up to fifth order. You can use scikit-learn as well. + +

+Evaluate the Mean Squared error (MSE) + +$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n} +\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2, +$$ + +

+and the \( R^2 \) score function. If \( \tilde{\hat{y}}_i \) is the predicted +value of the \( i-th \) sample and \( y_i \) is the corresponding true value, +then the score \( R^2 \) is defined as + +$$ +R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2}, +$$ + +

+where we have defined the mean value of \( \hat{y} \) as + +$$ +\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i. +$$ + +

+You should split your data in train and test and also consider scaling the data. + +

+To set up the design matrix, the following code can be used +

+ + +

def FrankeFunction(x,y):
+	term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
+	term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
+	term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
+	term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
+	return term1 + term2 + term3 + term4
+
+
+def create_X(x, y, n ):
+	if len(x.shape) > 1:
+		x = np.ravel(x)
+		y = np.ravel(y)
+
+	N = len(x)
+	l = int((n+1)*(n+2)/2)		# Number of elements in beta
+	X = np.ones((N,l))
+
+	for i in range(1,n+1):
+		q = int((i)*(i+1)/2)
+		for k in range(i+1):
+			X[:,q+k] = (x**(i-k))*(y**k)
+
+	return X
+
+
+# Making meshgrid of datapoints and compute Franke's function
+n = 5
+N = 1000
+x = np.sort(np.random.uniform(0, 1, N))
+y = np.sort(np.random.uniform(0, 1, N))
+z = FrankeFunction(x, y)
+X = create_X(x, y, n=n)    
+
+

+Write then your own code for the Ridge method or use Scikit-Learn. +Perform the same analysis as you did for ordinary Least Squares (for the same polynomials) but now for different values of \( \lambda \). Compare and +analyze your results with those obtained with ordinary Least Squares. Study the +dependence on \( \lambda \). + +

+This part is essentially a repeat of the previous ones, but now +with Lasso regression. Write either your own code or +use the functionalities of Scikit-Learn (recommended). +Give a +critical discussion of the three methods and a judgement of which +model fits the data best. + +

+ + +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs000.html b/doc/pub/week36/html/._week36-bs000.html new file mode 100644 index 000000000..b7fdad551 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs000.html @@ -0,0 +1,452 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + + + +
+

Week 36: Statistical interpretation of Linear Regression and Resampling techniques

+ +

+ + +

+Morten Hjorth-Jensen [1, 2] +
+ +

+ + +

[1] Department of Physics, University of Oslo
+
[2] Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University
+
+

+

Sep 10, 2021

+
+

+ + +

Read »

+ + +
+ +

+ +

+ + +
+ + + + + + + +
+ © 1999-2021, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs001.html b/doc/pub/week36/html/._week36-bs001.html new file mode 100644 index 000000000..ea3fc68dc --- /dev/null +++ b/doc/pub/week36/html/._week36-bs001.html @@ -0,0 +1,440 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Plans for week 36

+ +
    +
  • Thursday: Summary from last week on SVD, Statistics, probability theory and linear regression. Video of Lecture.
    • +
  • Friday: Linear Regression and links with Statistics, Resampling methods and presentation of first project.
    • +
+ +Recommended Reading: + +
    +
  1. Lectures on Regression
    2. +
  2. Bishop 1.1, 1.2, 2.1, 2.2, 2.3 and 3.1
    3. +
  3. Hastie et al chapter 3
    4. +
+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs002.html b/doc/pub/week36/html/._week36-bs002.html new file mode 100644 index 000000000..5744fea8c --- /dev/null +++ b/doc/pub/week36/html/._week36-bs002.html @@ -0,0 +1,429 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Thursday September 9

+ +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs003.html b/doc/pub/week36/html/._week36-bs003.html new file mode 100644 index 000000000..a621bd94e --- /dev/null +++ b/doc/pub/week36/html/._week36-bs003.html @@ -0,0 +1,430 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Summary from last Week and Examples

+ +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs004.html b/doc/pub/week36/html/._week36-bs004.html new file mode 100644 index 000000000..3e568c489 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs004.html @@ -0,0 +1,463 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Linear Regression and the SVD

+ +

+We used the SVD to analyse the matrix to invert in ordinary lineat regression +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T. +$$ + +Since the matrices here have dimension \( p\times p \), with \( p \) corresponding to the singular values, we defined last week the matrix +$$ +\boldsymbol{\Sigma}^T\boldsymbol{\Sigma} = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\end{bmatrix}, +$$ + +where the tilde-matrix \( \tilde{\boldsymbol{\Sigma}} \) is a matrix of dimension \( p\times p \) containing only the singular values \( \sigma_i \), that is + +$$ +\tilde{\boldsymbol{\Sigma}}=\begin{bmatrix} \sigma_0 & 0 & 0 & \dots & 0 & 0 \\ + 0 & \sigma_1 & 0 & \dots & 0 & 0 \\ + 0 & 0 & \sigma_2 & \dots & 0 & 0 \\ + 0 & 0 & 0 & \dots & \sigma_{p-2} & 0 \\ + 0 & 0 & 0 & \dots & 0 & \sigma_{p-1} \\ +\end{bmatrix}, +$$ + +meaning we can write +$$ +\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2\boldsymbol{V}^T. +$$ + +Multiplying from the right with \( \boldsymbol{V} \) (using the orthogonality of \( \boldsymbol{V} \)) we get +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs005.html b/doc/pub/week36/html/._week36-bs005.html new file mode 100644 index 000000000..307be260c --- /dev/null +++ b/doc/pub/week36/html/._week36-bs005.html @@ -0,0 +1,470 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

What does it mean?

+ +

+This means the vectors \( \boldsymbol{v}_i \) of the orthogonal matrix \( \boldsymbol{V} \) +are the eigenvectors of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) with eigenvalues +given by the singular values squared, that is + +$$ +\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2. +$$ + +

+In other words, each non-zero singular value of \( \boldsymbol{X} \) is a positive +square root of an eigenvalue of \( \boldsymbol{X}^T\boldsymbol{X} \). It means also that +the columns of \( \boldsymbol{V} \) are the eigenvectors of +\( \boldsymbol{X}^T\boldsymbol{X} \). Since we have ordered the singular values of +\( \boldsymbol{X} \) in a descending order, it means that the column vectors +\( \boldsymbol{v}_i \) are hierarchically ordered by how much correlation they +encode from the columns of \( \boldsymbol{X} \). + +

+Note that these are also the eigenvectors and eigenvalues of the +Hessian matrix. + +

+If we now recall the definition of the covariance matrix (not using +Bessel's correction) we have + +$$ +\boldsymbol{C}[\boldsymbol{X}]=\frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}, +$$ + +

+meaning that every squared non-singular value of \( \boldsymbol{X} \) divided by \( n \) ( +the number of samples) are the eigenvalues of the covariance +matrix. Every singular value of \( \boldsymbol{X} \) is thus a positive square +root of an eigenvalue of \( \boldsymbol{X}^T\boldsymbol{X} \). If the matrix \( \boldsymbol{X} \) is +self-adjoint, the singular values of \( \boldsymbol{X} \) are equal to the +absolute value of the eigenvalues of \( \boldsymbol{X} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs006.html b/doc/pub/week36/html/._week36-bs006.html new file mode 100644 index 000000000..036586223 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs006.html @@ -0,0 +1,467 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

And finally \( \boldsymbol{X}\boldsymbol{X}^T \)

+ +

+For \( \boldsymbol{X}\boldsymbol{X}^T \) we found + +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{U}^T. +$$ + +Since the matrices here have dimension \( n\times n \), we have +$$ +\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \boldsymbol{0}\\ \end{bmatrix}=\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}, +$$ + +leading to +$$ +\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}\boldsymbol{U}^T. +$$ + +

+Multiplying with \( \boldsymbol{U} \) from the right gives us the eigenvalue problem +$$ +(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U}=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}. +$$ + +

+It means that the eigenvalues of \( \boldsymbol{X}\boldsymbol{X}^T \) are again given by +the non-zero singular values plus now a series of zeros. The column +vectors of \( \boldsymbol{U} \) are the eigenvectors of \( \boldsymbol{X}\boldsymbol{X}^T \) and +measure how much correlations are contained in the rows of \( \boldsymbol{X} \). + +

+Since we will mainly be interested in the correlations among the features +of our data (the columns of \( \boldsymbol{X} \), the quantity of interest for us are the non-zero singular +values and the column vectors of \( \boldsymbol{V} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs007.html b/doc/pub/week36/html/._week36-bs007.html new file mode 100644 index 000000000..1961cdcce --- /dev/null +++ b/doc/pub/week36/html/._week36-bs007.html @@ -0,0 +1,478 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Code for SVD and Inversion of Matrices

+ +

+How do we use the SVD to invert a matrix \( \boldsymbol{X}^\boldsymbol{X} \) which is singular or near singular? +The simple answer is to use the linear algebra function for pseudoinvers, that is +

+ + +

Ainv = np.linlag.pinv(A)
+
+

+Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD. + +

+ + +

import numpy as np
+# SVD inversion
+def SVDinv(A):
+    ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
+    SVD is numerically more stable than the inversion algorithms provided by
+    numpy and scipy.linalg at the cost of being slower.
+    '''
+    U, s, VT = np.linalg.svd(A)
+    print('test U')
+    print( (np.transpose(U) @ U - U @np.transpose(U)))
+    print('test VT')
+    print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))
+
+
+    D = np.zeros((len(U),len(VT)))
+    D = np.diag(s)
+    UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)
+    return np.matmul(V,np.matmul(invD,UT))
+
+
+#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])
+# Non-singular square matrix
+X = np.array( [ [1,2,3],[2,4,5],[3,5,6]])
+print(X)
+A = np.transpose(X) @ X
+# Brute force inversion
+B = np.linalg.inv(A)  # here we could use np.linalg.pinv(A)
+C = SVDinv(A)
+print(np.abs(B-C))
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs008.html b/doc/pub/week36/html/._week36-bs008.html new file mode 100644 index 000000000..d6c9ccd4a --- /dev/null +++ b/doc/pub/week36/html/._week36-bs008.html @@ -0,0 +1,484 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Inverse of Rectangular Matrix

+ +

+Although our matrix to invert \( \boldsymbol{X}^T\boldsymbol{X} \) is a square matrix, our matrix may be singular. + +

+The pseudoinverse is the generalization of the matrix inverse for square matrices to +rectangular matrices where the number of rows and columns are not equal. + +

+It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse. +It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices. + +

+Using the SVD we can obtain the pseudoinverse of a matrix \( \boldsymbol{A} \) (labeled here as \( \boldsymbol{A}_{\mathrm{PI}} \) +$$ +\boldsymbol{A}_{\mathrm{PI}}= \boldsymbol{V}\boldsymbol{D}_{\mathrm{PI}}\boldsymbol{U}^T, +$$ + +where \( \boldsymbol{D}_{\mathrm{PI}} \) can be calculated by creating a diagonal matrix from \( \boldsymbol{Sigma} \) where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD. + +

+ + +

import numpy as np
+# SVD inversion
+def SVDinv(A):
+    U, s, VT = np.linalg.svd(A)
+    # reciprocals of singular values of s
+    d = 1.0 / s
+    # create m x n D matrix
+    D = np.zeros(A.shape)
+    # populate D with n x n diagonal matrix
+    D[:A.shape[1], :A.shape[1]] = np.diag(d)
+    UT = np.transpose(U)
+    V = np.transpose(VT)
+    return np.matmul(V,np.matmul(D.T,UT))
+
+
+A = np.array([ [0.3, 0.4], [0.5, 0.6], [0.7, 0.8],[0.9, 1.0]])
+print(A)
+# Brute force inversion of super-collinear matrix
+B = np.linalg.pinv(A)
+print(B)
+# Compare our own algorithm with pinv
+C = SVDinv(A)
+print(np.abs(C-B))
+
+

+As you can see from this example, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by Numpy. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs009.html b/doc/pub/week36/html/._week36-bs009.html new file mode 100644 index 000000000..15240a2b2 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs009.html @@ -0,0 +1,454 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Ridge and LASSO Regression

+ +

+Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is +our optimization problem is +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}. +$$ + +or we can state it as +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2, +$$ + +where we have used the definition of a norm-2 vector, that is +$$ +\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs010.html b/doc/pub/week36/html/._week36-bs010.html new file mode 100644 index 000000000..001ae8a49 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs010.html @@ -0,0 +1,472 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

From OLS to Ridge and Lasso

+ +

+By minimizing the above equation with respect to the parameters +\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the +parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by +defining a new cost function to be optimized, that is + +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2 +$$ + +

+which leads to the Ridge regression minimization problem where we +require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is +a finite number larger than zero. By defining + +$$ +C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1, +$$ + +

+we have a new optimization equation +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1 +$$ + +which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. + +

+Here we have defined the norm-1 as +$$ +\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs011.html b/doc/pub/week36/html/._week36-bs011.html new file mode 100644 index 000000000..d22ff157b --- /dev/null +++ b/doc/pub/week36/html/._week36-bs011.html @@ -0,0 +1,478 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Deriving the Ridge Regression Equations

+ +

+Using the matrix-vector expression for Ridge regression and dropping the parameter \( 1/n \) in front of the standard means squared error equation, we have + +$$ +C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta}, +$$ + +and +taking the derivatives with respect to \( \boldsymbol{\beta} \) we obtain then +a slightly modified matrix inversion problem which for finite values +of \( \lambda \) does not suffer from singularity problems. We obtain +the optimal parameters +$$ +\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +

+with \( \boldsymbol{I} \) being a \( p\times p \) identity matrix with the constraint that + +$$ +\sum_{i=0}^{p-1} \beta_i^2 \leq t, +$$ + +

+with \( t \) a finite positive number. + +

+When we compare this with the ordinary least squares result we have +$$ +\hat{\boldsymbol{\beta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$ + +which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). + +

+We see that Ridge regression is nothing but the standard OLS with a +modified diagonal term added to \( \boldsymbol{X}^T\boldsymbol{X} \). The consequences, in +particular for our discussion of the bias-variance tradeoff are rather +interesting. We will see that for specific values of \( \lambda \), we may +even reduce the variance of the optimal parameters \( \boldsymbol{\beta} \). These topics and other related ones, will be discussed after the more linear algebra oriented analysis here. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs012.html b/doc/pub/week36/html/._week36-bs012.html new file mode 100644 index 000000000..5920f562c --- /dev/null +++ b/doc/pub/week36/html/._week36-bs012.html @@ -0,0 +1,454 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

SVD analysis

+ +

+Using our insights about the SVD of the design matrix \( \boldsymbol{X} \) +We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix \( \boldsymbol{U} \) as +$$ +\tilde{\boldsymbol{y}}_{\mathrm{OLS}}=\boldsymbol{X}\boldsymbol{\beta} =\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}. +$$ + +

+For Ridge regression this becomes + +$$ +\tilde{\boldsymbol{y}}_{\mathrm{Ridge}}=\boldsymbol{X}\boldsymbol{\beta}_{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{\Sigma}^2\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y}, +$$ + +

+with the vectors \( \boldsymbol{u}_j \) being the columns of \( \boldsymbol{U} \) from the SVD of the matrix \( \boldsymbol{X} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs013.html b/doc/pub/week36/html/._week36-bs013.html new file mode 100644 index 000000000..1aa1ec6f9 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs013.html @@ -0,0 +1,454 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Interpreting the Ridge results

+ +

+Since \( \lambda \geq 0 \), it means that compared to OLS, we have + +$$ +\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1. +$$ + +

+Ridge regression finds the coordinates of \( \boldsymbol{y} \) with respect to the +orthonormal basis \( \boldsymbol{U} \), it then shrinks the coordinates by +\( \frac{\sigma_j^2}{\sigma_j^2+\lambda} \). Recall that the SVD has +eigenvalues ordered in a descending way, that is \( \sigma_i \geq +\sigma_{i+1} \). + +

+For small eigenvalues \( \sigma_i \) it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs014.html b/doc/pub/week36/html/._week36-bs014.html new file mode 100644 index 000000000..53dc2f2cb --- /dev/null +++ b/doc/pub/week36/html/._week36-bs014.html @@ -0,0 +1,469 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

More interpretations

+ +

+For the sake of simplicity, let us assume that the design matrix is orthonormal, that is + +$$ +\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}. +$$ + +

+In this case the standard OLS results in +$$ +\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y}, +$$ + +

+and + +$$ +\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}}, +$$ + +

+that is the Ridge estimator scales the OLS estimator by the inverse of a factor \( 1+\lambda \), and +the Ridge estimator converges to zero when the hyperparameter goes to +infinity. + +

+We will come back to more interpreations after we have gone through some of the statistical analysis part. + +

+For more discussions of Ridge and Lasso regression, Wessel van Wieringen's article is highly recommended. +Similarly, Mehta et al's article is also recommended. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs015.html b/doc/pub/week36/html/._week36-bs015.html new file mode 100644 index 000000000..0066c9cac --- /dev/null +++ b/doc/pub/week36/html/._week36-bs015.html @@ -0,0 +1,463 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Deriving the Lasso Regression Equations

+ +

+Using the matrix-vector expression for Lasso regression and dropping the parameter \( 1/n \) in front of the standard mean squared error equation, we have the following cost function + +$$ +C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_1, +$$ + +

+Taking the derivative with respect to \( \boldsymbol{\beta} \) and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty) +$$ +\frac{d \vert \beta\vert}{d \boldsymbol{\beta}}=\mathrm{sgn}(\boldsymbol{\beta})=\left\{\begin{array}{cc} 1 & \beta > 0 \\ 0 & \beta =0\\-1 & \beta < 0, \end{array}\right. +$$ + +we have that the derivative of the cost function is + +$$ +\frac{\partial C(\boldsymbol{X},\boldsymbol{\beta})}{\partial \boldsymbol{\beta}}=-2\boldsymbol{X}^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})+\lambda sgn(\boldsymbol{\beta})=0, +$$ + +and reordering we have +$$ +\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta}+\lambda sgn(\boldsymbol{\beta})=2\boldsymbol{X}^T\boldsymbol{y}. +$$ + +This equation does not lead to a nice analytical equation as in Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package CVXOPT. We will discuss this later. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs016.html b/doc/pub/week36/html/._week36-bs016.html new file mode 100644 index 000000000..e54034c6c --- /dev/null +++ b/doc/pub/week36/html/._week36-bs016.html @@ -0,0 +1,452 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression

+ +

+Let us assume that our design matrix is given by unit (identity) matrix, that is a square diagonal matrix with ones only along the +diagonal. In this case we have an equal number of rows and columns \( n=p \). + +

+Our model approximation is just \( \tilde{\boldsymbol{y}}=\boldsymbol{\beta} \) and the mean squared error and thereby the cost function for ordinary least sqquares (OLS) is then (we drop the term \( 1/n \)) +$$ +C(\boldsymbol{\beta})=\sum_{i=0}^{p-1}(y_i-\beta_i)^2, +$$ + +and minimizing we have that +$$ +\hat{\beta}_i^{\mathrm{OLS}} = y_i. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs017.html b/doc/pub/week36/html/._week36-bs017.html new file mode 100644 index 000000000..b8a875cda --- /dev/null +++ b/doc/pub/week36/html/._week36-bs017.html @@ -0,0 +1,448 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Ridge Regression

+ +

+For Ridge regression our cost function is +$$ +C(\boldsymbol{\beta})=\sum_{i=0}^{p-1}(y_i-\beta_i)^2+\lambda\sum_{i=0}^{p-1}\beta_i^2, +$$ + +and minimizing we have that +$$ +\hat{\beta}_i^{\mathrm{Ridge}} = \frac{y_i}{1+\lambda}. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs018.html b/doc/pub/week36/html/._week36-bs018.html new file mode 100644 index 000000000..69ea86e69 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs018.html @@ -0,0 +1,458 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Lasso Regression

+ +

+For Lasso regression our cost function is +$$ +C(\boldsymbol{\beta})=\sum_{i=0}^{p-1}(y_i-\beta_i)^2+\lambda\sum_{i=0}^{p-1}\vert\beta_i\vert=\sum_{i=0}^{p-1}(y_i-\beta_i)^2+\lambda\sum_{i=0}^{p-1}\sqrt{\beta_i^2}, +$$ + +and minimizing we have that +$$ +-2\sum_{i=0}^{p-1}(y_i-\beta_i)+\lambda \sum_{i=0}^{p-1}\frac{(\beta_i)}{\vert\beta_i\vert}=0, +$$ + +which leads to +$$ +\hat{\boldsymbol{\beta}}_i^{\mathrm{Lasso}} = \left\{\begin{array}{ccc}y_i-\frac{\lambda}{2} &\mathrm{if} & y_i> \frac{\lambda}{2}\\ + y_i+\frac{\lambda}{2} &\mathrm{if} & y_i < -\frac{\lambda}{2}\\ + 0 &\mathrm{if} & \vert y_i\vert\le \frac{\lambda}{2}\end{array}\right.\\. +$$ + +

+Plotting these results (figure in handwritten notes for week 36) shows clearly that Lasso regression suppresses (sets to zero) values of \( \beta_i \) for specific values of \( \lambda \). Ridge regression reduces on the other hand the values of \( \beta_i \) as function of \( \lambda \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs019.html b/doc/pub/week36/html/._week36-bs019.html new file mode 100644 index 000000000..a7c2710a2 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs019.html @@ -0,0 +1,451 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Yet another Example

+ +

+Let us assume we have a data set with outputs/targets given by the vector + +$$ +\boldsymbol{y}=\begin{bmatrix}4 \\ 2 \\3\end{bmatrix}, +$$ + +and our inputs as a \( 3\times 2 \) design matrix +$$ +\boldsymbol{X}=\begin{bmatrix}2 & 0\\ 0 & 1 \\ 0 & 0\end{bmatrix}, +$$ + +meaning that we have two features and two unknown parameters \( \beta_0 \) and \( \beta_1 \) to be determined either by ordinary least squares, Ridge or Lasso regression. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs020.html b/doc/pub/week36/html/._week36-bs020.html new file mode 100644 index 000000000..466079c18 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs020.html @@ -0,0 +1,453 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The OLS case

+ +

+For ordinary least squares (OLS) we know that the optimal solution is + +$$ +\hat{\boldsymbol{\beta}}^{\mathrm{OLS}}=\left( \boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +Inserting the above values we obtain that + +$$ +\hat{\boldsymbol{\beta}}^{\mathrm{OLS}}=\begin{bmatrix}2 \\ 2\end{bmatrix}, +$$ + +

+The code which implements this simpler case is presented after the discussion of Ridge and Lasso. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs021.html b/doc/pub/week36/html/._week36-bs021.html new file mode 100644 index 000000000..56ee2ccc6 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs021.html @@ -0,0 +1,457 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The Ridge case

+ +

+For Ridge regression we have + +$$ +\hat{\boldsymbol{\beta}}^{\mathrm{Ridge}}=\left( \boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +Inserting the above values we obtain that + +$$ +\hat{\boldsymbol{\beta}}^{\mathrm{Ridge}}=\begin{bmatrix}\frac{8}{4+\lambda} \\ \frac{2}{1+\lambda}\end{bmatrix}, +$$ + +

+There is normally a constraint on the value of \( \vert\vert \boldsymbol{\beta}\vert\vert_2 \) via the parameter \( \lambda \). +Let us for simplicity assume that \( \beta_0^2+\beta_1^2=1 \) as constraint. This will allow us to find an expression for the optimal values of \( \beta \) and \( \lambda \). + +

+To see this, let us write the cost function for Ridge regression. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs022.html b/doc/pub/week36/html/._week36-bs022.html new file mode 100644 index 000000000..4a39d201e --- /dev/null +++ b/doc/pub/week36/html/._week36-bs022.html @@ -0,0 +1,466 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Writing the Cost Function

+ +

+We define the MSE without the \( 1/n \) factor and have then, using that +$$ +\boldsymbol{X}\boldsymbol{\beta}=\begin{bmatrix} 2\beta_0 \\ \beta_1 \\0 \end{bmatrix}, +$$ + + +$$ +C(\boldsymbol{\beta})=(4-2\beta_0)^2+(2-\beta_1)^2+\lambda(\beta_0^2+\beta_1^2), +$$ + +and taking the derivative with respect to \( \beta_0 \) we get +$$ +\beta_0=\frac{8}{4+\lambda}, +$$ + +and for \( \beta_1 \) we obtain +$$ +\beta_1=\frac{2}{1+\lambda}, +$$ + +

+Using the constraint for \( \beta_0^2+\beta_1^2=1 \) we can constrain \( \lambda \) by solving +$$ +\left(\frac{8}{4+\lambda}\right)^2+\left(\frac{2}{1+\lambda}\right)^2=1, +$$ + +which gives \( \lambda=4.571 \) and \( \beta_0=0.933 \) and \( \beta_1=0.359 \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs023.html b/doc/pub/week36/html/._week36-bs023.html new file mode 100644 index 000000000..989fd1bb7 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs023.html @@ -0,0 +1,462 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Lasso case

+ +

+For Lasso we need now, keeping a constraint on \( \vert\beta_0\vert+\vert\beta_1\vert=1 \), to take the derivative of the absolute values of \( \beta_0 \) +and \( \beta_1 \). This gives us the following derivatives of the cost function +$$ +C(\boldsymbol{\beta})=(4-2\beta_0)^2+(2-\beta_1)^2+\lambda(\vert\beta_0\vert+\vert\beta_1\vert), +$$ + + +$$ +\frac{\partial C(\boldsymbol{\beta})}{\partial \beta_0}=-4(4-2\beta_0)+\lambda\mathrm{sgn}(\beta_0)=0, +$$ + +and +$$ +\frac{\partial C(\boldsymbol{\beta})}{\partial \beta_1}=-2(2-\beta_1)+\lambda\mathrm{sgn}(\beta_1)=0. +$$ + +We have now four cases to solve besides the trivial cases \( \beta_0 \) and/or \( \beta_1 \) are zero, namely + +

    +
  1. \( \beta_0 > 0 \) and \( \beta_1 > 0 \),
    2. +
  2. \( \beta_0 > 0 \) and \( \beta_1 < 0 \),
    3. +
  3. \( \beta_0 < 0 \) and \( \beta_1 > 0 \),
    4. +
  4. \( \beta_0 < 0 \) and \( \beta_1 < 0 \).
    5. +
+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs024.html b/doc/pub/week36/html/._week36-bs024.html new file mode 100644 index 000000000..506d4f0f6 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs024.html @@ -0,0 +1,462 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The first Case

+ +

+If we consider the first case, we have then +$$ +-4(4-2\beta_0)+\lambda=0, +$$ + +and +$$ +-2(2-\beta_1)+\lambda=0. +$$ + +which yields + +$$ +\beta_0=\frac{16+\lambda}{8}, +$$ + +and +$$ +\beta_1=\frac{4+\lambda}{2}. +$$ + +

+Using the constraint on \( \beta_0 \) and \( \beta_1 \) we can then find the optimal value of \( \lambda \) for the different cases. We leave this as an exercise to you. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs025.html b/doc/pub/week36/html/._week36-bs025.html new file mode 100644 index 000000000..79c514d1e --- /dev/null +++ b/doc/pub/week36/html/._week36-bs025.html @@ -0,0 +1,499 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple code for solving the above problem

+ +

+Here we set up the OLS, Ridge and Lasso functionality in order to study the above example. Note that here we have opted for a set of values of \( \lambda \), meaning that we need to perform a search in order to find the optimal values. + +

+First we study and compare the OLS and Ridge results. The next code compares all three methods. + +

+ + +

import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+
+X = np.array( [ [ 2, 0], [0, 1], [0,0]])
+y = np.array( [4, 2, 3])
+
+
+# matrix inversion to find beta
+OLSbeta = np.linalg.inv(X.T @ X) @ X.T @ y
+print(OLSbeta)
+# and then make the prediction
+ytildeOLS = X @ OLSbeta
+print("Training MSE for OLS")
+print(MSE(y,ytildeOLS))
+ypredictOLS = X @ OLSbeta
+
+# Repeat now for Ridge regression and various values of the regularization parameter
+I = np.eye(2,2)
+# Decide which values of lambda to use
+nlambdas = 100
+MSEPredict = np.zeros(nlambdas)
+lambdas = np.logspace(-4, 4, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y
+#    print(Ridgebeta)
+    # and then make the prediction
+    ypredictRidge = X @ Ridgebeta
+    MSEPredict[i] = MSE(y,ypredictRidge)
+#    print(MSEPredict[i])
+    # Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Train')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+

+We see here that we reach a plateau. What is actually happening? + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs026.html b/doc/pub/week36/html/._week36-bs026.html new file mode 100644 index 000000000..60a60bc4e --- /dev/null +++ b/doc/pub/week36/html/._week36-bs026.html @@ -0,0 +1,497 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

With Lasso Regression

+ +

+ + +

import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn import linear_model
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+
+X = np.array( [ [ 2, 0], [0, 1], [0,0]])
+y = np.array( [4, 2, 3])
+
+
+# matrix inversion to find beta
+OLSbeta = np.linalg.inv(X.T @ X) @ X.T @ y
+print(OLSbeta)
+# and then make the prediction
+ytildeOLS = X @ OLSbeta
+print("Training MSE for OLS")
+print(MSE(y,ytildeOLS))
+ypredictOLS = X @ OLSbeta
+
+# Repeat now for Ridge regression and various values of the regularization parameter
+I = np.eye(2,2)
+# Decide which values of lambda to use
+nlambdas = 100
+MSERidgePredict = np.zeros(nlambdas)
+MSELassoPredict = np.zeros(nlambdas)
+lambdas = np.logspace(-4, 4, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y
+    print(Ridgebeta)
+    # and then make the prediction
+    ypredictRidge = X @ Ridgebeta
+    MSERidgePredict[i] = MSE(y,ypredictRidge)
+    RegLasso = linear_model.Lasso(lmb)
+    RegLasso.fit(X,y)
+    ypredictLasso = RegLasso.predict(X)
+    print(RegLasso.coef_)
+    MSELassoPredict[i] = MSE(y,ypredictLasso)
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSERidgePredict, 'r--', label = 'MSE Ridge Train')
+plt.plot(np.log10(lambdas), MSELassoPredict, 'r--', label = 'MSE Lasso Train')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs027.html b/doc/pub/week36/html/._week36-bs027.html new file mode 100644 index 000000000..afc97f721 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs027.html @@ -0,0 +1,519 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Another Example, now with a polynomial fit

+ +

+ + +

import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(3155)
+
+x = np.random.rand(100)
+y = 2.0+5*x*x+0.1*np.random.randn(100)
+
+# number of features p (here degree of polynomial
+p = 3
+#  The design matrix now as function of a given polynomial
+X = np.zeros((len(x),p))
+X[:,0] = 1.0
+X[:,1] = x
+X[:,2] = x*x
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+# matrix inversion to find beta
+OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
+print(OLSbeta)
+# and then make the prediction
+ytildeOLS = X_train @ OLSbeta
+print("Training MSE for OLS")
+print(MSE(y_train,ytildeOLS))
+ypredictOLS = X_test @ OLSbeta
+print("Test MSE OLS")
+print(MSE(y_test,ypredictOLS))
+
+# Repeat now for Lasso and Ridge regression and various values of the regularization parameter
+I = np.eye(p,p)
+# Decide which values of lambda to use
+nlambdas = 100
+MSEPredict = np.zeros(nlambdas)
+MSETrain = np.zeros(nlambdas)
+MSELassoPredict = np.zeros(nlambdas)
+MSELassoTrain = np.zeros(nlambdas)
+lambdas = np.logspace(-4, 4, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train
+    # include lasso using Scikit-Learn
+    RegLasso = linear_model.Lasso(lmb)
+    RegLasso.fit(X_train,y_train)
+    # and then make the prediction
+    ytildeRidge = X_train @ Ridgebeta
+    ypredictRidge = X_test @ Ridgebeta
+    ytildeLasso = RegLasso.predict(X_train)
+    ypredictLasso = RegLasso.predict(X_test)
+    MSEPredict[i] = MSE(y_test,ypredictRidge)
+    MSETrain[i] = MSE(y_train,ytildeRidge)
+    MSELassoPredict[i] = MSE(y_test,ypredictLasso)
+    MSELassoTrain[i] = MSE(y_train,ytildeLasso)
+
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSETrain, label = 'MSE Ridge train')
+plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Test')
+plt.plot(np.log10(lambdas), MSELassoTrain, label = 'MSE Lasso train')
+plt.plot(np.log10(lambdas), MSELassoPredict, 'r--', label = 'MSE Lasso Test')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs028.html b/doc/pub/week36/html/._week36-bs028.html new file mode 100644 index 000000000..b0f98248e --- /dev/null +++ b/doc/pub/week36/html/._week36-bs028.html @@ -0,0 +1,437 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Friday September 10

+ +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs029.html b/doc/pub/week36/html/._week36-bs029.html new file mode 100644 index 000000000..ebc7550a1 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs029.html @@ -0,0 +1,478 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Linking the regression analysis with a statistical interpretation

+ +

+We will now couple the discussions of ordinary least squares, Ridge +and Lasso regression with a statistical interpretation, that is we +move from a linear algebra analysis to a statistical analysis. In +particular, we will focus on what the regularization terms can result +in. We will amongst other things show that the regularization +parameter can reduce considerably the variance of the parameters +\( \beta \). + +

+The +advantage of doing linear regression is that we actually end up with +analytical expressions for several statistical quantities. +Standard least squares and Ridge regression allow us to +derive quantities like the variance and other expectation values in a +rather straightforward way. + +

+It is assumed that \( \varepsilon_i +\sim \mathcal{N}(0, \sigma^2) \) and the \( \varepsilon_{i} \) are +independent, i.e.: +$$ +\begin{align*} +\mbox{Cov}(\varepsilon_{i_1}, +\varepsilon_{i_2}) & = \left\{ \begin{array}{lcc} \sigma^2 & \mbox{if} +& i_1 = i_2, \\ 0 & \mbox{if} & i_1 \not= i_2. \end{array} \right. +\end{align*} +$$ + +The randomness of \( \varepsilon_i \) implies that +\( \mathbf{y}_i \) is also a random variable. In particular, +\( \mathbf{y}_i \) is normally distributed, because \( \varepsilon_i \sim +\mathcal{N}(0, \sigma^2) \) and \( \mathbf{X}_{i,\ast} \, \boldsymbol{\beta} \) is a +non-random scalar. To specify the parameters of the distribution of +\( \mathbf{y}_i \) we need to calculate its first two moments. + +

+Recall that \( \boldsymbol{X} \) is a matrix of dimensionality \( n\times p \). The +notation above \( \mathbf{X}_{i,\ast} \) means that we are looking at the +row number \( i \) and perform a sum over all values \( p \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs030.html b/doc/pub/week36/html/._week36-bs030.html new file mode 100644 index 000000000..57d01ad1b --- /dev/null +++ b/doc/pub/week36/html/._week36-bs030.html @@ -0,0 +1,452 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Assumptions made

+ +

+The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off) +that there exists a function \( f(\boldsymbol{x}) \) and a normal distributed error \( \boldsymbol{\varepsilon}\sim \mathcal{N}(0, \sigma^2) \) +which describe our data +$$ +\boldsymbol{y} = f(\boldsymbol{x})+\boldsymbol{\varepsilon} +$$ + +

+We approximate this function with our model from the solution of the linear regression equations, that is our +function \( f \) is approximated by \( \boldsymbol{\tilde{y}} \) where we want to minimize \( (\boldsymbol{y}-\boldsymbol{\tilde{y}})^2 \), our MSE, with +$$ +\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta}. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs031.html b/doc/pub/week36/html/._week36-bs031.html new file mode 100644 index 000000000..6324200d9 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs031.html @@ -0,0 +1,467 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Expectation value and variance

+ +

+We can calculate the expectation value of \( \boldsymbol{y} \) for a given element \( i \) +$$ +\begin{align*} +\mathbb{E}(y_i) & = +\mathbb{E}(\mathbf{X}_{i, \ast} \, \boldsymbol{\beta}) + \mathbb{E}(\varepsilon_i) +\, \, \, = \, \, \, \mathbf{X}_{i, \ast} \, \beta, +\end{align*} +$$ + +while +its variance is +$$ +\begin{align*} \mbox{Var}(y_i) & = \mathbb{E} \{ [y_i +- \mathbb{E}(y_i)]^2 \} \, \, \, = \, \, \, \mathbb{E} ( y_i^2 ) - +[\mathbb{E}(y_i)]^2 \\ & = \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, +\beta + \varepsilon_i )^2] - ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 \\ & += \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 + 2 \varepsilon_i +\mathbf{X}_{i, \ast} \, \boldsymbol{\beta} + \varepsilon_i^2 ] - ( \mathbf{X}_{i, +\ast} \, \beta)^2 \\ & = ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 + 2 +\mathbb{E}(\varepsilon_i) \mathbf{X}_{i, \ast} \, \boldsymbol{\beta} + +\mathbb{E}(\varepsilon_i^2 ) - ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 +\\ & = \mathbb{E}(\varepsilon_i^2 ) \, \, \, = \, \, \, +\mbox{Var}(\varepsilon_i) \, \, \, = \, \, \, \sigma^2. +\end{align*} +$$ + +Hence, \( y_i \sim \mathcal{N}( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta}, \sigma^2) \), that is \( \boldsymbol{y} \) follows a normal distribution with +mean value \( \boldsymbol{X}\boldsymbol{\beta} \) and variance \( \sigma^2 \) (not be confused with the singular values of the SVD). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs032.html b/doc/pub/week36/html/._week36-bs032.html new file mode 100644 index 000000000..4ed0a91ab --- /dev/null +++ b/doc/pub/week36/html/._week36-bs032.html @@ -0,0 +1,516 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Expectation value and variance for \( \boldsymbol{\beta} \)

+ +

+With the OLS expressions for the parameters \( \boldsymbol{\beta} \) we can evaluate the expectation value +$$ +\mathbb{E}(\boldsymbol{\beta}) = \mathbb{E}[ (\mathbf{X}^{\top} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbb{E}[ \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1} \mathbf{X}^{T}\mathbf{X}\boldsymbol{\beta}=\boldsymbol{\beta}. +$$ + +This means that the estimator of the regression parameters is unbiased. + +

+We can also calculate the variance + +

+The variance of \( \boldsymbol{\beta} \) is +$$ +\begin{eqnarray*} +\mbox{Var}(\boldsymbol{\beta}) & = & \mathbb{E} \{ [\boldsymbol{\beta} - \mathbb{E}(\boldsymbol{\beta})] [\boldsymbol{\beta} - \mathbb{E}(\boldsymbol{\beta})]^{T} \} +\\ +& = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \boldsymbol{\beta}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \boldsymbol{\beta}]^{T} \} +\\ +% & = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}]^{T} \} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +% \\ +% & = & \mathbb{E} \{ (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} \, \mathbf{Y}^{T} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} \} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +% \\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \mathbb{E} \{ \mathbf{Y} \, \mathbf{Y}^{T} \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +\\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \{ \mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} \, \mathbf{X}^{T} + \sigma^2 \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +% \\ +% & = & (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^T \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T % \mathbf{X})^{-1} +% \\ +% & & + \, \, \sigma^2 \, (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T \mathbf{X})^{-1} - \boldsymbol{\beta} \boldsymbol{\beta}^T +\\ +& = & \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} + \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +\, \, \, = \, \, \, \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}, +\end{eqnarray*} +$$ + +

+where we have used that \( \mathbb{E} (\mathbf{Y} \mathbf{Y}^{T}) = +\mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} \, \mathbf{X}^{T} + +\sigma^2 \, \mathbf{I}_{nn} \). From \( \mbox{Var}(\boldsymbol{\beta}) = \sigma^2 +\, (\mathbf{X}^{T} \mathbf{X})^{-1} \), one obtains an estimate of the +variance of the estimate of the \( j \)-th regression coefficient: +\( \boldsymbol{\sigma}^2 (\boldsymbol{\beta}_j ) = \boldsymbol{\sigma}^2 [(\mathbf{X}^{T} \mathbf{X})^{-1}]_{jj} \). This may be used to +construct a confidence interval for the estimates. + +

+In a similar way, we can obtain analytical expressions for say the +expectation values of the parameters \( \boldsymbol{\beta} \) and their variance +when we employ Ridge regression, allowing us again to define a confidence interval. + +

+It is rather straightforward to show that +$$ +\mathbb{E} \big[ \boldsymbol{\beta}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\boldsymbol{\beta}^{\mathrm{OLS}}. +$$ + +We see clearly that +\( \mathbb{E} \big[ \boldsymbol{\beta}^{\mathrm{Ridge}} \big] \not= \boldsymbol{\beta}^{\mathrm{OLS}} \) for any \( \lambda > 0 \). We say then that the ridge estimator is biased. + +

+We can also compute the variance as + +$$ +\mbox{Var}[\boldsymbol{\beta}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T} \mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, +$$ + +and it is easy to see that if the parameter \( \lambda \) goes to infinity then the variance of Ridge parameters \( \boldsymbol{\beta} \) goes to zero. + +

+With this, we can compute the difference + +$$ +\mbox{Var}[\boldsymbol{\beta}^{\mathrm{OLS}}]-\mbox{Var}(\boldsymbol{\beta}^{\mathrm{Ridge}})=\sigma^2 [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}[ 2\lambda\mathbf{I} + \lambda^2 (\mathbf{X}^{T} \mathbf{X})^{-1} ] \{ [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}. +$$ + +The difference is non-negative definite since each component of the +matrix product is non-negative definite. +This means the variance we obtain with the standard OLS will always for \( \lambda > 0 \) be larger than the variance of \( \boldsymbol{\beta} \) obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs033.html b/doc/pub/week36/html/._week36-bs033.html new file mode 100644 index 000000000..0b8245a15 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs033.html @@ -0,0 +1,456 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Deriving OLS from a probability distribution

+ +

+Our basic assumption when we derived the OLS equations was to assume +that our output is determined by a given continuous function +\( f(\boldsymbol{x}) \) and a random noise \( \boldsymbol{\epsilon} \) given by the normal +distribution with zero mean value and an undetermined variance +\( \sigma^2 \). + +

+We found above that the outputs \( \boldsymbol{y} \) have a mean value given by +\( \boldsymbol{X}\hat{\boldsymbol{\beta}} \) and variance \( \sigma^2 \). Since the entries to +the design matrix are not stochastic variables, we can assume that the +probability distribution of our targets is also a normal distribution +but now with mean value \( \boldsymbol{X}\hat{\boldsymbol{\beta}} \). This means that a +single output \( y_i \) is given by the Gaussian distribution + +$$ +y_i\sim \mathcal{N}(\boldsymbol{X}_{i,*}\boldsymbol{\beta}, \sigma^2)=\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs034.html b/doc/pub/week36/html/._week36-bs034.html new file mode 100644 index 000000000..dd2b8edf3 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs034.html @@ -0,0 +1,469 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Independent and Identically Distrubuted (iid)

+ +

+We assume now that the various \( y_i \) values are stochastically distributed according to the above Gaussian distribution. +We define this distribution as +$$ +p(y_i, \boldsymbol{X}\vert\boldsymbol{\beta})=\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}, +$$ + +which reads as finding the likelihood of an event \( y_i \) with the input variables \( \boldsymbol{X} \) given the parameters (to be determined) \( \boldsymbol{\beta} \). + +

+Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event \( \boldsymbol{y} \) as the product of the single events, that is we have + +$$ +p(\boldsymbol{y},\boldsymbol{X}\vert\boldsymbol{\beta})=\prod_{i=0}^{n-1}\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}=\prod_{i=0}^{n-1}p(y_i,\boldsymbol{X}\vert\boldsymbol{\beta}). +$$ + +

+We will write this in a more compact form reserving \( \boldsymbol{D} \) for the domain of events, including the ouputs (targets) and the inputs. That is +in case we have a simple one-dimensional input and output case +$$ +\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\dots, (x_{n-1},y_{n-1})]. +$$ + +In the more general case the various inputs should be replaced by the possible features represented by the input data set \( \boldsymbol{X} \). +We can now rewrite the above probability as +$$ +p(\boldsymbol{D}\vert\boldsymbol{\beta})=\prod_{i=0}^{n-1}\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}. +$$ + +

+It is a conditional probability (see below) and reads as the likelihood of a domain of events \( \boldsymbol{D} \) given a set of parameters \( \boldsymbol{\beta} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs035.html b/doc/pub/week36/html/._week36-bs035.html new file mode 100644 index 000000000..10f2720e8 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs035.html @@ -0,0 +1,462 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Maximum Likelihood Estimation (MLE)

+ +

+In statistics, maximum likelihood estimation (MLE) is a method of +estimating the parameters of an assumed probability distribution, +given some observed data. This is achieved by maximizing a likelihood +function so that, under the assumed statistical model, the observed +data is the most probable. + +

+We will assume here that our events are given by the above Gaussian +distribution and we will determine the optimal parameters \( \beta \) by +maximizing the above PDF. However, computing the derivatives of a +product function is cumbersome and can easily lead to overflow and/or +underflowproblems, with potentials for loss of numerical precision. + +

+In practice, it is more convenient to maximize the logarithm of the +PDF because it is a monotonically increasing function of the argument. +Alternatively, and this will be our option, we will minimize the +negative of the logarithm since this is a monotonically decreasing +function. + +

+Note also that maximization/minimization of the logarithm of the PDF +is equivalent to the maximization/minimization of the function itself. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs036.html b/doc/pub/week36/html/._week36-bs036.html new file mode 100644 index 000000000..6de932992 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs036.html @@ -0,0 +1,464 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

A new Cost Function

+ +

+We could now define a new cost function to minimize, namely the negative logarithm of the above PDF + +$$ +C(\boldsymbol{\beta}=-\log{\prod_{i=0}^{n-1}p(y_i,\boldsymbol{X}\vert\boldsymbol{\beta})}=-\sum_{i=0}^{n-1}\log{p(y_i,\boldsymbol{X}\vert\boldsymbol{\beta})}, +$$ + +which becomes +$$ +C(\boldsymbol{\beta}=\frac{n}{2}\log{2\pi\sigma^2}+\frac{\vert\vert (\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\vert\vert_2^2}{2\sigma^2}. +$$ + +

+Taking the derivative of the new cost function with respect to the parameters \( \beta \) we recognize our familiar OLS equation, namely + +$$ +\boldsymbol{X}^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right) =0, +$$ + +which leads to the well-known OLS equation for the optimal paramters \( \beta \) +$$ +\hat{\boldsymbol{\beta}}^{\mathrm{OLS}}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}! +$$ + +

+Before we make a similar analysis for Ridge and Lasso regression, we need a short reminder on statistics. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs037.html b/doc/pub/week36/html/._week36-bs037.html new file mode 100644 index 000000000..c77983682 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs037.html @@ -0,0 +1,475 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

More basic Statistics and Bayes' theorem

+ +

+A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry. +Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics. + +

+Assume we have two domains of events \( X=[x_0,x_1,\dots,x_{n-1}] \) and \( Y=[y_0,y_1,\dots,y_{n-1}] \). + +

+We define also the likelihood for \( X \) and \( Y \) as \( p(X) \) and \( p(Y) \) respectively. +The likelihood of a specific event \( x_i \) (or \( y_i \)) is then written as \( p(X=x_i) \) or just \( p(x_i)=p_i \). + +

+

+
+

+$$ +p(X \cup Y)= p(X)+p(Y)-p(X \cap Y). +$$ +

+
+ + +

+

+
+

+$$ +p(X \cup Y)= p(X,Y)= p(X\vert Y)p(Y)=p(Y\vert X)p(X), +$$ + +where we read \( p(X\vert Y) \) as the likelihood of obtaining \( X \) given \( Y \). +

+
+ + +

+If we have independent events then \( p(X,Y)=p(X)p(Y) \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs038.html b/doc/pub/week36/html/._week36-bs038.html new file mode 100644 index 000000000..0a701fcea --- /dev/null +++ b/doc/pub/week36/html/._week36-bs038.html @@ -0,0 +1,449 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Marginal Probability

+ +

+The marginal probability is defined in terms of only one of the set of variables \( X,Y \). For a discrete probability we have +

+
+

+$$ +p(X)=\sum_{i=0}^{n-1}p(X,Y=y_i)=\sum_{i=0}^{n-1}p(X\vert Y=y_i)p(Y=y_i)=\sum_{i=0}^{n-1}p(X\vert y_i)p(y_i). +$$ +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs039.html b/doc/pub/week36/html/._week36-bs039.html new file mode 100644 index 000000000..211ad7a42 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs039.html @@ -0,0 +1,449 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Conditional Probability

+ +

+The conditional probability, if \( p(Y) > 0 \), is +

+
+

+$$ +p(X\vert Y)= \frac{p(X,Y)}{p(Y)}=\frac{p(X,Y)}{\sum_{i=0}^{n-1}p(Y\vert X=x_i)p(x_i)}. +$$ +

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs040.html b/doc/pub/week36/html/._week36-bs040.html new file mode 100644 index 000000000..a860725d7 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs040.html @@ -0,0 +1,451 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Bayes' Theorem

+ +

+If we combine the conditional probability with the marginal probability and the standard product rule, we have +$$ +p(X\vert Y)= \frac{p(X,Y)}{p(Y)}, +$$ + +which we can rewrite as + +$$ +p(X\vert Y)= \frac{p(X,Y)}{\sum_{i=0}^{n-1}p(Y\vert X=x_i)p(x_i)}=\frac{p(Y\vert X)p(X)}{\sum_{i=0}^{n-1}p(Y\vert X=x_i)p(x_i)}, +$$ + +which is Bayes' theorem. It allows us to evaluate the uncertainty in in \( X \) after we have observed \( Y \). We can easily interchange \( X \) with \( Y \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs041.html b/doc/pub/week36/html/._week36-bs041.html new file mode 100644 index 000000000..5a059e0c2 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs041.html @@ -0,0 +1,449 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Interpretations of Bayes' Theorem

+ +

+The quantity \( p(Y\vert X) \) on the right-hand side of the theorem is +evaluated for the observed data \( Y \) and can be viewed as a function of +the parameter space represented by \( X \). This function is not +necesseraly normalized and is normally called the likelihood function. + +

+The function \( p(X) \) on the right hand side is called the prior while the function on the left hand side is the called the posterior probability. The denominator on the right hand side serves as a normalization factor for the posterior distribution. + +

+Let us try to illustrate Bayes' theorem through an example. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs042.html b/doc/pub/week36/html/._week36-bs042.html new file mode 100644 index 000000000..846214847 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs042.html @@ -0,0 +1,459 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Example of Usage of Bayes' theorem

+ +

+Let us suppose that you are undergoing a series of mammography scans in +order to rule out possible breast cancer cases. We define the +sensitivity for a positive event by the variable \( X \). It takes binary +values with \( X=1 \) representing a positive event and \( X=0 \) being a +negative event. We reserve \( Y \) as a classification parameter for +either a negative or a positive breast cancer confirmation. (Short note on wordings: positive here means having breast cancer, although none of us would consider this being a positive thing). + +

+We let \( Y=1 \) represent the the case of having breast cancer and \( Y=0 \) as not. + +

+Let us assume that if you have breast cancer, the test will be positive with a probability of \( 0.8 \), that is we have + +$$ +p(X=1\vert Y=1) =0.8. +$$ + +

+This obviously sounds scary since many would conclude that if the test is positive, there is a likelihood of \( 80\% \) for having cancer. +It is however not correct, as the following Bayesian analysis shows. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs043.html b/doc/pub/week36/html/._week36-bs043.html new file mode 100644 index 000000000..0d7f06734 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs043.html @@ -0,0 +1,460 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Doing it correctly

+ +

+If we look at various national surveys on breast cancer, the general likelihood of developing breast cancer is a very small number. +Let us assume that the prior probability in the population as a whole is + +$$ +p(Y=1) =0.004. +$$ + +

+We need also to account for the fact that the test may produce a false positive result (false alarm). Let us here assume that we have +$$ +p(X=1\vert Y=0) =0.1. +$$ + +

+Using Bayes' theorem we can then find the posterior probability that the person has breast cancer in case of a positive test, that is we can compute + +$$ +p(Y=1\vert X=1)=\frac{p(X=1\vert Y=1)p(Y=1)}{p(X=1\vert Y=1)p(Y=1)+p(X=1\vert Y=0)p(Y=0)}=\frac{0.8\times 0.004}{0.8\times 0.004+0.1\times 0.996}=0.031. +$$ + +That is, in case of a positive test, there is only a \( 3\% \) chance of having breast cancer! + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs044.html b/doc/pub/week36/html/._week36-bs044.html new file mode 100644 index 000000000..61595c542 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs044.html @@ -0,0 +1,446 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Bayes' Theorem and Ridge and Lasso Regression

+ +

+Hitherto we have discussed Ridge and Lasso regression in terms of a +linear analysis. This may to many of you feel rather technical and +perhaps not that intuitive. The question is whether we can develop a +more intuitive way of understanding what Ridge and Lasso express. + +

+Before we proceed let us perform a Ridge, Lasso and OLS analysis of a polynomial fit. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs045.html b/doc/pub/week36/html/._week36-bs045.html new file mode 100644 index 000000000..869df30c0 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs045.html @@ -0,0 +1,517 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Test Function for what happens with OLS, Ridge and Lasso

+ +

+We will play around with a study of the values for the optimal +parameters \( \boldsymbol{\beta} \) using OLS, Ridge and Lasso regression. For +OLS, you will notice as function of the noise and polynomial degree, +that the parameters \( \beta \) will fluctuate from order to order in the +polynomial fit and that for larger and larger polynomial degrees of freedom, the parameters will tend to increase in value for OLS. + +

+For Ridge and Lasso regression, the higher order parameters will typically be reduced, providing thereby less fluctuations from one order to another one. + +

+ + +

import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+from sklearn import linear_model
+
+def R2(y_data, y_model):
+    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+    n = np.size(y_model)
+    return np.sum((y_data-y_model)**2)/n
+
+# Make data set.
+n = 10000
+x = np.random.rand(n)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.randn(n)
+
+Maxpolydegree = 5
+X = np.zeros((len(x),Maxpolydegree))
+X[:,0] = 1.0
+
+for polydegree in range(1, Maxpolydegree):
+    for degree in range(polydegree):
+        X[:,degree] = x**(degree)
+
+
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+# matrix inversion to find beta
+OLSbeta = np.linalg.pinv(X_train.T @ X_train) @ X_train.T @ y_train
+print(OLSbeta)
+ypredictOLS = X_test @ OLSbeta
+print("Test MSE OLS")
+print(MSE(y_test,ypredictOLS))
+# Repeat now for Lasso and Ridge regression and various values of the regularization parameter using Scikit-Learn
+# Decide which values of lambda to use
+nlambdas = 4
+MSERidgePredict = np.zeros(nlambdas)
+MSELassoPredict = np.zeros(nlambdas)
+lambdas = np.logspace(-3, 1, nlambdas)
+for i in range(nlambdas):
+    lmb = lambdas[i]
+    # Make the fit using Ridge and Lasso
+    RegRidge = linear_model.Ridge(lmb,fit_intercept=False)
+    RegRidge.fit(X_train,y_train)
+    RegLasso = linear_model.Lasso(lmb,fit_intercept=False)
+    RegLasso.fit(X_train,y_train)
+    # and then make the prediction
+    ypredictRidge = RegRidge.predict(X_test)
+    ypredictLasso = RegLasso.predict(X_test)
+    # Compute the MSE and print it
+    MSERidgePredict[i] = MSE(y_test,ypredictRidge)
+    MSELassoPredict[i] = MSE(y_test,ypredictLasso)
+    print(lmb,RegRidge.coef_)
+    print(lmb,RegLasso.coef_)
+# Now plot the results
+plt.figure()
+plt.plot(np.log10(lambdas), MSERidgePredict, 'b', label = 'MSE Ridge Test')
+plt.plot(np.log10(lambdas), MSELassoPredict, 'r', label = 'MSE Lasso Test')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+

+How can we understand this? + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs046.html b/doc/pub/week36/html/._week36-bs046.html new file mode 100644 index 000000000..64a43cd8d --- /dev/null +++ b/doc/pub/week36/html/._week36-bs046.html @@ -0,0 +1,467 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Invoking Bayes' theorem

+ +

+Using Bayes' theorem we can gain a better intuition about Ridge and Lasso regression. + +

+For ordinary least squares we postulated that the maximum likelihood for the doamin of events \( \boldsymbol{D} \) (one-dimensional case) +$$ +\boldsymbol{D}=[(x_0,y_0), (x_1,y_1),\dots, (x_{n-1},y_{n-1})], +$$ + +is given by +$$ +p(\boldsymbol{D}\vert\boldsymbol{\beta})=\prod_{i=0}^{n-1}\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}. +$$ + +

+In Bayes' theorem this function plays the role of the so-called likelihood. We could now ask the question what is the posterior probability of a parameter set \( \boldsymbol{\beta} \) given a domain of events \( \boldsymbol{D} \)? That is, how can we define the posterior probability + +$$ +p(\boldsymbol{\beta}\vert\boldsymbol{D}). +$$ + +

+Bayes' theorem comes to our rescue here since (omitting the normalization constant) +$$ +p(\boldsymbol{\beta}\vert\boldsymbol{D})\propto p(\boldsymbol{D}\vert\boldsymbol{\beta})p(\boldsymbol{\beta}). +$$ + +

+We have a model for \( p(\boldsymbol{D}\vert\boldsymbol{\beta}) \) but need one for the prior \( p(\boldsymbol{\beta} \)! + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs047.html b/doc/pub/week36/html/._week36-bs047.html new file mode 100644 index 000000000..32782bdbe --- /dev/null +++ b/doc/pub/week36/html/._week36-bs047.html @@ -0,0 +1,473 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Ridge and Bayes

+ +

+With the posterior probability defined by a likelihood which we have +already modeled and an unknown prior, we are now ready to make +additional models for the prior. + +

+We can, based on our discussions of the variance of \( \boldsymbol{\beta} \) and the mean value, assume that the prior for the values \( \boldsymbol{\beta} \) is given by a Gaussian with mean value zero and variance \( \tau^2 \), that is + +$$ +p(\boldsymbol{\beta})=\prod_{j=0}^{p-1}\exp{\left(-\frac{\beta_j^2}{2\tau^2}\right)}. +$$ + +

+Our posterior probability becomes then (omitting the normalization factor which is just a constant) +$$ +p(\boldsymbol{\beta\vert\boldsymbol{D})}=\prod_{i=0}^{n-1}\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}\prod_{j=0}^{p-1}\exp{\left(-\frac{\beta_j^2}{2\tau^2}\right)}. +$$ + +

+We can now optimize this quantity with respect to \( \boldsymbol{\beta} \). As we +did for OLS, this is most conveniently done by taking the negative +logarithm of the posterior probability. Doing so and leaving out the +constants terms that do not depend on \( \beta \), we have + +$$ +C(\boldsymbol{\beta})=\frac{\vert\vert (\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\vert\vert_2^2}{2\sigma^2}+\frac{1}{2\tau^2}\vert\vert\boldsymbol{\beta}\vert\vert_2^2, +$$ + +and replacing \( 1/2\tau^2 \) with \( \lambda \) we have + +$$ +C(\boldsymbol{\beta})=\frac{\vert\vert (\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\vert\vert_2^2}{2\sigma^2}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_2^2, +$$ + +which is our Ridge cost function! Nice, isn't it? + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs048.html b/doc/pub/week36/html/._week36-bs048.html new file mode 100644 index 000000000..764f0fe91 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs048.html @@ -0,0 +1,467 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Lasso and Bayes

+ +

+To derive the Lasso cost function, we simply replace the Gaussian prior with an exponential distribution (Laplace in this case) with zero mean value, that is + +$$ +p(\boldsymbol{\beta})=\prod_{j=0}^{p-1}\exp{\left(-\frac{\vert\beta_j\vert}{\tau}\right)}. +$$ + +

+Our posterior probability becomes then (omitting the normalization factor which is just a constant) +$$ +p(\boldsymbol{\beta}\vert\boldsymbol{D})=\prod_{i=0}^{n-1}\frac{1}{\sqrt{2\pi\sigma^2}}\exp{\left[-\frac{(y_i-\boldsymbol{X}_{i,*}\boldsymbol{\beta})^2}{2\sigma^2}\right]}\prod_{j=0}^{p-1}\exp{\left(-\frac{\vert\beta_j\vert}{\tau}\right)}. +$$ + +

+Taking the negative +logarithm of the posterior probability and leaving out the +constants terms that do not depend on \( \beta \), we have + +$$ +C(\boldsymbol{\beta}=\frac{\vert\vert (\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\vert\vert_2^2}{2\sigma^2}+\frac{1}{\tau}\vert\vert\boldsymbol{\beta}\vert\vert_1, +$$ + +and replacing \( 1/\tau \) with \( \lambda \) we have + +$$ +C(\boldsymbol{\beta}=\frac{\vert\vert (\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\vert\vert_2^2}{2\sigma^2}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_1, +$$ + +which is our Lasso cost function! + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs049.html b/doc/pub/week36/html/._week36-bs049.html new file mode 100644 index 000000000..1d1222f6f --- /dev/null +++ b/doc/pub/week36/html/._week36-bs049.html @@ -0,0 +1,449 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Why resampling methods

+ +

+Before we proceed, we need to rethink what we have been doing. In our +eager to fit the data, we have omitted several important elements in +our regression analysis. In what follows we will + +

    +
  1. look at statistical properties, including a discussion of mean values, variance and the so-called bias-variance tradeoff
    2. +
  2. introduce resampling techniques like cross-validation, bootstrapping and jackknife and more
    3. +
+ +This will allow us to link the standard linear algebra methods we have discussed above to a statistical interpretation of the methods. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs050.html b/doc/pub/week36/html/._week36-bs050.html new file mode 100644 index 000000000..a3b53a757 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs050.html @@ -0,0 +1,466 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods

+
+
+

+Resampling methods are an indispensable tool in modern +statistics. They involve repeatedly drawing samples from a training +set and refitting a model of interest on each sample in order to +obtain additional information about the fitted model. For example, in +order to estimate the variability of a linear regression fit, we can +repeatedly draw different samples from the training data, fit a linear +regression to each new sample, and then examine the extent to which +the resulting fits differ. Such an approach may allow us to obtain +information that would not be available from fitting the model only +once using the original training sample. + +

+Two resampling methods are often used in Machine Learning analyses, + +

    +
  1. The bootstrap method
    2. +
  2. and Cross-Validation
    3. +
+ +In addition there are several other methods such as the Jackknife and the Blocking methods. We will discuss in particular +cross-validation and the bootstrap method. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs051.html b/doc/pub/week36/html/._week36-bs051.html new file mode 100644 index 000000000..969840f2b --- /dev/null +++ b/doc/pub/week36/html/._week36-bs051.html @@ -0,0 +1,462 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling approaches can be computationally expensive

+
+
+

+ +

+Resampling approaches can be computationally expensive, because they +involve fitting the same statistical method multiple times using +different subsets of the training data. However, due to recent +advances in computing power, the computational requirements of +resampling methods generally are not prohibitive. In this chapter, we +discuss two of the most commonly used resampling methods, +cross-validation and the bootstrap. Both methods are important tools +in the practical application of many statistical learning +procedures. For example, cross-validation can be used to estimate the +test error associated with a given statistical learning method in +order to evaluate its performance, or to select the appropriate level +of flexibility. The process of evaluating a model’s performance is +known as model assessment, whereas the process of selecting the proper +level of flexibility for a model is known as model selection. The +bootstrap is widely used. + +

+

+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs052.html b/doc/pub/week36/html/._week36-bs052.html new file mode 100644 index 000000000..330ee3d0b --- /dev/null +++ b/doc/pub/week36/html/._week36-bs052.html @@ -0,0 +1,449 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Why resampling methods ?

+
+
+

+ +

    +
  • Our simulations can be treated as computer experiments. This is particularly the case for Monte Carlo methods
    • +
  • The results can be analysed with the same statistical tools as we would use analysing experimental data.
    • +
  • As in all experiments, we are looking for expectation values and an estimate of how accurate they are, i.e., possible sources for errors.
    • +
+
+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs053.html b/doc/pub/week36/html/._week36-bs053.html new file mode 100644 index 000000000..786f3898d --- /dev/null +++ b/doc/pub/week36/html/._week36-bs053.html @@ -0,0 +1,455 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Statistical analysis

+
+
+

+ +

    +
  • As in other experiments, many numerical experiments have two classes of errors:
    • + +
      +
    • Statistical errors
      • +
    • Systematical errors
      • +
    + +
  • Statistical errors can be estimated using standard tools from statistics
    • +
  • Systematical errors are method specific and must be treated differently from case to case.
    • +
+
+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs054.html b/doc/pub/week36/html/._week36-bs054.html new file mode 100644 index 000000000..0270c983b --- /dev/null +++ b/doc/pub/week36/html/._week36-bs054.html @@ -0,0 +1,461 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods

+ +

+With all these analytical equations for both the OLS and Ridge +regression, we will now outline how to assess a given model. This will +lead us to a discussion of the so-called bias-variance tradeoff (see +below) and so-called resampling methods. + +

+One of the quantities we have discussed as a way to measure errors is +the mean-squared error (MSE), mainly used for fitting of continuous +functions. Another choice is the absolute error. + +

+In the discussions below we will focus on the MSE and in particular since we will split the data into test and training data, +we discuss the + +

    +
  1. prediction error or simply the test error \( \mathrm{Err_{Test}} \), where we have a fixed training set and the test error is the MSE arising from the data reserved for testing. We discuss also the
    2. +
  2. training error \( \mathrm{Err_{Train}} \), which is the average loss over the training data.
    3. +
+ +As our model becomes more and more complex, more of the training data tends to used. The training may thence adapt to more complicated structures in the data. This may lead to a decrease in the bias (see below for code example) and a slight increase of the variance for the test error. +For a certain level of complexity the test error will reach minimum, before starting to increase again. The +training error reaches a saturation. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs055.html b/doc/pub/week36/html/._week36-bs055.html new file mode 100644 index 000000000..35296bc5a --- /dev/null +++ b/doc/pub/week36/html/._week36-bs055.html @@ -0,0 +1,455 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods: Jackknife and Bootstrap

+ +

+Two famous +resampling methods are the independent bootstrap and the jackknife. + +

+The jackknife is a special case of the independent bootstrap. Still, the jackknife was made +popular prior to the independent bootstrap. And as the popularity of +the independent bootstrap soared, new variants, such as the dependent bootstrap. + +

+The Jackknife and independent bootstrap work for +independent, identically distributed random variables. +If these conditions are not +satisfied, the methods will fail. Yet, it should be said that if the data are +independent, identically distributed, and we only want to estimate the +variance of \( \overline{X} \) (which often is the case), then there is no +need for bootstrapping. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs056.html b/doc/pub/week36/html/._week36-bs056.html new file mode 100644 index 000000000..1086dcbdf --- /dev/null +++ b/doc/pub/week36/html/._week36-bs056.html @@ -0,0 +1,451 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods: Jackknife

+ +

+The Jackknife works by making many replicas of the estimator \( \widehat{\theta} \). +The jackknife is a resampling method where we systematically leave out one observation from the vector of observed values \( \boldsymbol{x} = (x_1,x_2,\cdots,X_n) \). +Let \( \boldsymbol{x}_i \) denote the vector +$$ +\boldsymbol{x}_i = (x_1,x_2,\cdots,x_{i-1},x_{i+1},\cdots,x_n), +$$ + +

+which equals the vector \( \boldsymbol{x} \) with the exception that observation +number \( i \) is left out. Using this notation, define +\( \widehat{\theta}_i \) to be the estimator +\( \widehat{\theta} \) computed using \( \vec{X}_i \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs057.html b/doc/pub/week36/html/._week36-bs057.html new file mode 100644 index 000000000..94f4b59ba --- /dev/null +++ b/doc/pub/week36/html/._week36-bs057.html @@ -0,0 +1,468 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Jackknife code example

+

+ + +

from numpy import *
+from numpy.random import randint, randn
+from time import time
+
+def jackknife(data, stat):
+    n = len(data);t = zeros(n); inds = arange(n); t0 = time()
+    ## 'jackknifing' by leaving out an observation for each i                                                                                                                      
+    for i in range(n):
+        t[i] = stat(delete(data,i) )
+
+    # analysis                                                                                                                                                                     
+    print("Runtime: %g sec" % (time()-t0)); print("Jackknife Statistics :")
+    print("original           bias      std. error")
+    print("%8g %14g %15g" % (stat(data),(n-1)*mean(t)/n, (n*var(t))**.5))
+
+    return t
+
+
+# Returns mean of data samples                                                                                                                                                     
+def stat(data):
+    return mean(data)
+
+
+mu, sigma = 100, 15
+datapoints = 10000
+x = mu + sigma*random.randn(datapoints)
+# jackknife returns the data sample                                                                                                                                                
+t = jackknife(x, stat)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs058.html b/doc/pub/week36/html/._week36-bs058.html new file mode 100644 index 000000000..0fb2c1ef1 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs058.html @@ -0,0 +1,454 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods: Bootstrap

+
+
+

+Bootstrapping is a nonparametric approach to statistical inference +that substitutes computation for more traditional distributional +assumptions and asymptotic results. Bootstrapping offers a number of +advantages: + +

    +
  1. The bootstrap is quite general, although there are some cases in which it fails.
    2. +
  2. Because it does not require distributional assumptions (such as normally distributed errors), the bootstrap can provide more accurate inferences when the data are not well behaved or when the sample size is small.
    3. +
  3. It is possible to apply the bootstrap to statistics with sampling distributions that are difficult to derive, even asymptotically.
    4. +
  4. It is relatively simple to apply the bootstrap to complex data-collection plans (such as stratified and clustered samples).
    5. +
+
+
+ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs059.html b/doc/pub/week36/html/._week36-bs059.html new file mode 100644 index 000000000..c51a6797e --- /dev/null +++ b/doc/pub/week36/html/._week36-bs059.html @@ -0,0 +1,448 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods: Bootstrap background

+ +

+Since \( \widehat{\theta} = \widehat{\theta}(\boldsymbol{X}) \) is a function of random variables, +\( \widehat{\theta} \) itself must be a random variable. Thus it has +a pdf, call this function \( p(\boldsymbol{t}) \). The aim of the bootstrap is to +estimate \( p(\boldsymbol{t}) \) by the relative frequency of +\( \widehat{\theta} \). You can think of this as using a histogram +in the place of \( p(\boldsymbol{t}) \). If the relative frequency closely +resembles \( p(\vec{t}) \), then using numerics, it is straight forward to +estimate all the interesting parameters of \( p(\boldsymbol{t}) \) using point +estimators. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs060.html b/doc/pub/week36/html/._week36-bs060.html new file mode 100644 index 000000000..3bebbbb4c --- /dev/null +++ b/doc/pub/week36/html/._week36-bs060.html @@ -0,0 +1,454 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods: More Bootstrap background

+ +

+In the case that \( \widehat{\theta} \) has +more than one component, and the components are independent, we use the +same estimator on each component separately. If the probability +density function of \( X_i \), \( p(x) \), had been known, then it would have +been straight forward to do this by: + +

    +
  1. Drawing lots of numbers from \( p(x) \), suppose we call one such set of numbers \( (X_1^*, X_2^*, \cdots, X_n^*) \).
    2. +
  2. Then using these numbers, we could compute a replica of \( \widehat{\theta} \) called \( \widehat{\theta}^* \).
    3. +
+ +By repeated use of (1) and (2), many +estimates of \( \widehat{\theta} \) could have been obtained. The +idea is to use the relative frequency of \( \widehat{\theta}^* \) +(think of a histogram) as an estimate of \( p(\boldsymbol{t}) \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs061.html b/doc/pub/week36/html/._week36-bs061.html new file mode 100644 index 000000000..894ef3456 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs061.html @@ -0,0 +1,453 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods: Bootstrap approach

+ +

+But +unless there is enough information available about the process that +generated \( X_1,X_2,\cdots,X_n \), \( p(x) \) is in general +unknown. Therefore, Efron in 1979 asked the +question: What if we replace \( p(x) \) by the relative frequency +of the observation \( X_i \); if we draw observations in accordance with +the relative frequency of the observations, will we obtain the same +result in some asymptotic sense? The answer is yes. + +

+Instead of generating the histogram for the relative +frequency of the observation \( X_i \), just draw the values +\( (X_1^*,X_2^*,\cdots,X_n^*) \) with replacement from the vector +\( \boldsymbol{X} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs062.html b/doc/pub/week36/html/._week36-bs062.html new file mode 100644 index 000000000..a84ecddc2 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs062.html @@ -0,0 +1,457 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Resampling methods: Bootstrap steps

+ +

+The independent bootstrap works like this: + +

    +
  1. Draw with replacement \( n \) numbers for the observed variables \( \boldsymbol{x} = (x_1,x_2,\cdots,x_n) \).
    2. +
  2. Define a vector \( \boldsymbol{x}^* \) containing the values which were drawn from \( \boldsymbol{x} \).
    3. +
  3. Using the vector \( \boldsymbol{x}^* \) compute \( \widehat{\theta}^* \) by evaluating \( \widehat \theta \) under the observations \( \boldsymbol{x}^* \).
    4. +
  4. Repeat this process \( k \) times.
    5. +
+ +When you are done, you can draw a histogram of the relative frequency +of \( \widehat \theta^* \). This is your estimate of the probability +distribution \( p(t) \). Using this probability distribution you can +estimate any statistics thereof. In principle you never draw the +histogram of the relative frequency of \( \widehat{\theta}^* \). Instead +you use the estimators corresponding to the statistic of interest. For +example, if you are interested in estimating the variance of \( \widehat +\theta \), apply the etsimator \( \widehat \sigma^2 \) to the values +\( \widehat \theta ^* \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs063.html b/doc/pub/week36/html/._week36-bs063.html new file mode 100644 index 000000000..18d690f39 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs063.html @@ -0,0 +1,496 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Code example for the Bootstrap method

+ +

+The following code starts with a Gaussian distribution with mean value +\( \mu =100 \) and variance \( \sigma=15 \). We use this to generate the data +used in the bootstrap analysis. The bootstrap analysis returns a data +set after a given number of bootstrap operations (as many as we have +data points). This data set consists of estimated mean values for each +bootstrap operation. The histogram generated by the bootstrap method +shows that the distribution for these mean values is also a Gaussian, +centered around the mean value \( \mu=100 \) but with standard deviation +\( \sigma/\sqrt{n} \), where \( n \) is the number of bootstrap samples (in +this case the same as the number of original data points). The value +of the standard deviation is what we expect from the central limit +theorem. + +

+ + +

from numpy import *
+from numpy.random import randint, randn
+from time import time
+import matplotlib.mlab as mlab
+import matplotlib.pyplot as plt
+
+# Returns mean of bootstrap samples                                                                                                                                                
+def stat(data):
+    return mean(data)
+
+# Bootstrap algorithm
+def bootstrap(data, statistic, R):
+    t = zeros(R); n = len(data); inds = arange(n); t0 = time()
+    # non-parametric bootstrap         
+    for i in range(R):
+        t[i] = statistic(data[randint(0,n,n)])
+
+    # analysis    
+    print("Runtime: %g sec" % (time()-t0)); print("Bootstrap Statistics :")
+    print("original           bias      std. error")
+    print("%8g %8g %14g %15g" % (statistic(data), std(data),mean(t),std(t)))
+    return t
+
+
+mu, sigma = 100, 15
+datapoints = 10000
+x = mu + sigma*random.randn(datapoints)
+# bootstrap returns the data sample                                    
+t = bootstrap(x, stat, datapoints)
+# the histogram of the bootstrapped  data                                                                                                    
+n, binsboot, patches = plt.hist(t, 50, normed=1, facecolor='red', alpha=0.75)
+
+# add a 'best fit' line  
+y = mlab.normpdf( binsboot, mean(t), std(t))
+lt = plt.plot(binsboot, y, 'r--', linewidth=1)
+plt.xlabel('Smarts')
+plt.ylabel('Probability')
+plt.axis([99.5, 100.6, 0, 3.0])
+plt.grid(True)
+
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs064.html b/doc/pub/week36/html/._week36-bs064.html new file mode 100644 index 000000000..466ede96d --- /dev/null +++ b/doc/pub/week36/html/._week36-bs064.html @@ -0,0 +1,453 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Various steps in cross-validation

+ +

+When the repetitive splitting of the data set is done randomly, +samples may accidently end up in a fast majority of the splits in +either training or test set. Such samples may have an unbalanced +influence on either model building or prediction evaluation. To avoid +this \( k \)-fold cross-validation structures the data splitting. The +samples are divided into \( k \) more or less equally sized exhaustive and +mutually exclusive subsets. In turn (at each split) one of these +subsets plays the role of the test set while the union of the +remaining subsets constitutes the training set. Such a splitting +warrants a balanced representation of each sample in both training and +test set over the splits. Still the division into the \( k \) subsets +involves a degree of randomness. This may be fully excluded when +choosing \( k=n \). This particular case is referred to as leave-one-out +cross-validation (LOOCV). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs065.html b/doc/pub/week36/html/._week36-bs065.html new file mode 100644 index 000000000..d4060d61c --- /dev/null +++ b/doc/pub/week36/html/._week36-bs065.html @@ -0,0 +1,464 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

How to set up the cross-validation for Ridge and/or Lasso

+ +
    +
  • Define a range of interest for the penalty parameter.
    • +
  • Divide the data set into training and test set comprising samples \( \{1, \ldots, n\} \setminus i \) and \( \{ i \} \), respectively.
    • +
  • Fit the linear regression model by means of ridge estimation for each \( \lambda \) in the grid using the training set, and the corresponding estimate of the error variance \( \boldsymbol{\sigma}_{-i}^2(\lambda) \), as
    • +
+ +$$ +\begin{align*} +\boldsymbol{\beta}_{-i}(\lambda) & = ( \boldsymbol{X}_{-i, \ast}^{T} +\boldsymbol{X}_{-i, \ast} + \lambda \boldsymbol{I}_{pp})^{-1} +\boldsymbol{X}_{-i, \ast}^{T} \boldsymbol{y}_{-i} +\end{align*} +$$ + + +
    +
  • Evaluate the prediction performance of these models on the test set by \( \log\{L[y_i, \boldsymbol{X}_{i, \ast}; \boldsymbol{\beta}_{-i}(\lambda), \boldsymbol{\sigma}_{-i}^2(\lambda)]\} \). Or, by the prediction error \( |y_i - \boldsymbol{X}_{i, \ast} \boldsymbol{\beta}_{-i}(\lambda)| \), the relative error, the error squared or the R2 score function.
    • +
  • Repeat the first three steps such that each sample plays the role of the test set once.
    • +
  • Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as
    • +
+ +$$ +\begin{align*} +\frac{1}{n} \sum_{i = 1}^n \log\{L[y_i, \mathbf{X}_{i, \ast}; \boldsymbol{\beta}_{-i}(\lambda), \boldsymbol{\sigma}_{-i}^2(\lambda)]\}. +\end{align*} +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs066.html b/doc/pub/week36/html/._week36-bs066.html new file mode 100644 index 000000000..56c85ffe2 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs066.html @@ -0,0 +1,452 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Cross-validation in brief

+ +

+For the various values of \( k \) + +

    +
  1. shuffle the dataset randomly.
    2. +
  2. Split the dataset into \( k \) groups.
    3. +
  3. For each unique group: + +
      +
    1. Decide which group to use as set for test data
      2. +
    2. Take the remaining groups as a training data set
      3. +
    3. Fit a model on the training set and evaluate it on the test set
      4. +
    4. Retain the evaluation score and discard the model
      5. +
    + +
  4. Summarize the model using the sample of model evaluation scores
    5. +
+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs067.html b/doc/pub/week36/html/._week36-bs067.html new file mode 100644 index 000000000..e5a0641c8 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs067.html @@ -0,0 +1,529 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Code Example for Cross-validation and \( k \)-fold Cross-validation

+ +

+The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial. +

+ + +

import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(3155)
+
+# Generate the data.
+nsamples = 100
+x = np.random.randn(nsamples)
+y = 3*x**2 + np.random.randn(nsamples)
+
+## Cross-validation on Ridge regression using KFold only
+
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 6)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+
+# Perform the cross-validation to estimate MSE
+scores_KFold = np.zeros((nlambdas, k))
+
+i = 0
+for lmb in lambdas:
+    ridge = Ridge(alpha = lmb)
+    j = 0
+    for train_inds, test_inds in kfold.split(x):
+        xtrain = x[train_inds]
+        ytrain = y[train_inds]
+
+        xtest = x[test_inds]
+        ytest = y[test_inds]
+
+        Xtrain = poly.fit_transform(xtrain[:, np.newaxis])
+        ridge.fit(Xtrain, ytrain[:, np.newaxis])
+
+        Xtest = poly.fit_transform(xtest[:, np.newaxis])
+        ypred = ridge.predict(Xtest)
+
+        scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred)
+
+        j += 1
+    i += 1
+
+
+estimated_mse_KFold = np.mean(scores_KFold, axis = 1)
+
+## Cross-validation using cross_val_score from sklearn along with KFold
+
+# kfold is an instance initialized above as:
+# kfold = KFold(n_splits = k)
+
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+    ridge = Ridge(alpha = lmb)
+
+    X = poly.fit_transform(x[:, np.newaxis])
+    estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold)
+
+    # cross_val_score return an array containing the estimated negative mse for every fold.
+    # we have to the the mean of every array in order to get an estimate of the mse of the model
+    estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+
+    i += 1
+
+## Plot and compare the slightly different ways to perform cross-validation
+
+plt.figure()
+
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('mse')
+
+plt.legend()
+
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs068.html b/doc/pub/week36/html/._week36-bs068.html new file mode 100644 index 000000000..1a932097e --- /dev/null +++ b/doc/pub/week36/html/._week36-bs068.html @@ -0,0 +1,494 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The bias-variance tradeoff

+ +

+We will discuss the bias-variance tradeoff in the context of +continuous predictions such as regression. However, many of the +intuitions and ideas discussed here also carry over to classification +tasks. Consider a dataset \( \mathcal{L} \) consisting of the data +\( \mathbf{X}_\mathcal{L}=\{(y_j, \boldsymbol{x}_j), j=0\ldots n-1\} \). + +

+Let us assume that the true data is generated from a noisy model + +$$ +\boldsymbol{y}=f(\boldsymbol{x}) + \boldsymbol{\epsilon} +$$ + +

+where \( \epsilon \) is normally distributed with mean zero and standard deviation \( \sigma^2 \). + +

+In our derivation of the ordinary least squares method we defined then +an approximation to the function \( f \) in terms of the parameters +\( \boldsymbol{\beta} \) and the design matrix \( \boldsymbol{X} \) which embody our model, +that is \( \boldsymbol{\tilde{y}}=\boldsymbol{X}\boldsymbol{\beta} \). + +

+Thereafter we found the parameters \( \boldsymbol{\beta} \) by optimizing the means squared error via the so-called cost function +$$ +C(\boldsymbol{X},\boldsymbol{\beta}) =\frac{1}{n}\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2=\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]. +$$ + +

+We can rewrite this as +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\frac{1}{n}\sum_i(f_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2+\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2+\sigma^2. +$$ + +

+The three terms represent the square of the bias of the learning +method, which can be thought of as the error caused by the simplifying +assumptions built into the method. The second term represents the +variance of the chosen model and finally the last terms is variance of +the error \( \boldsymbol{\epsilon} \). + +

+To derive this equation, we need to recall that the variance of \( \boldsymbol{y} \) and \( \boldsymbol{\epsilon} \) are both equal to \( \sigma^2 \). The mean value of \( \boldsymbol{\epsilon} \) is by definition equal to zero. Furthermore, the function \( f \) is not a stochastics variable, idem for \( \boldsymbol{\tilde{y}} \). +We use a more compact notation in terms of the expectation value +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{f}+\boldsymbol{\epsilon}-\boldsymbol{\tilde{y}})^2\right], +$$ + +and adding and subtracting \( \mathbb{E}\left[\boldsymbol{\tilde{y}}\right] \) we get +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{f}+\boldsymbol{\epsilon}-\boldsymbol{\tilde{y}}+\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2\right], +$$ + +which, using the abovementioned expectation values can be rewritten as +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2\right]+\mathrm{Var}\left[\boldsymbol{\tilde{y}}\right]+\sigma^2, +$$ + +that is the rewriting in terms of the so-called bias, the variance of the model \( \boldsymbol{\tilde{y}} \) and the variance of \( \boldsymbol{\epsilon} \). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs069.html b/doc/pub/week36/html/._week36-bs069.html new file mode 100644 index 000000000..7a8735a65 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs069.html @@ -0,0 +1,489 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Example code for Bias-Variance tradeoff

+

+ + +

import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+
+np.random.seed(2018)
+
+n = 500
+n_boostraps = 100
+degree = 18  # A quite high value, just to show.
+noise = 0.1
+
+# Make data set.
+x = np.linspace(-1, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1, x.shape)
+
+# Hold out some test data that is never used in training.
+x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+# Combine x transformation and model into one operation.
+# Not neccesary, but convenient.
+model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
+
+# The following (m x n_bootstraps) matrix holds the column vectors y_pred
+# for each bootstrap iteration.
+y_pred = np.empty((y_test.shape[0], n_boostraps))
+for i in range(n_boostraps):
+    x_, y_ = resample(x_train, y_train)
+
+    # Evaluate the new model on the same test data each time.
+    y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()
+
+# Note: Expectations and variances taken w.r.t. different training
+# data sets, hence the axis=1. Subsequent means are taken across the test data
+# set in order to obtain a total value, but before this we have error/bias/variance
+# calculated per data point in the test set.
+# Note 2: The use of keepdims=True is important in the calculation of bias as this 
+# maintains the column vector form. Dropping this yields very unexpected results.
+error = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+bias = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+variance = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+print('Error:', error)
+print('Bias^2:', bias)
+print('Var:', variance)
+print('{} >= {} + {} = {}'.format(error, bias, variance, bias+variance))
+
+plt.plot(x[::5, :], y[::5, :], label='f(x)')
+plt.scatter(x_test, y_test, label='Data points')
+plt.scatter(x_test, np.mean(y_pred, axis=1), label='Pred')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs070.html b/doc/pub/week36/html/._week36-bs070.html new file mode 100644 index 000000000..737d17777 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs070.html @@ -0,0 +1,480 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Understanding what happens

+

+ + +

import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
+
+np.random.seed(2018)
+
+n = 40
+n_boostraps = 100
+maxdegree = 14
+
+
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+for degree in range(maxdegree):
+    model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
+    y_pred = np.empty((y_test.shape[0], n_boostraps))
+    for i in range(n_boostraps):
+        x_, y_ = resample(x_train, y_train)
+        y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()
+
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+    variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+    print('Polynomial degree:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs071.html b/doc/pub/week36/html/._week36-bs071.html new file mode 100644 index 000000000..c4d145d25 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs071.html @@ -0,0 +1,462 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Summing up

+ +

+The bias-variance tradeoff summarizes the fundamental tension in +machine learning, particularly supervised learning, between the +complexity of a model and the amount of training data needed to train +it. Since data is often limited, in practice it is often useful to +use a less-complex model with higher bias, that is a model whose asymptotic +performance is worse than another model because it is easier to +train and less sensitive to sampling noise arising from having a +finite-sized training dataset (smaller variance). + +

+The above equations tell us that in +order to minimize the expected test error, we need to select a +statistical learning method that simultaneously achieves low variance +and low bias. Note that variance is inherently a nonnegative quantity, +and squared bias is also nonnegative. Hence, we see that the expected +test MSE can never lie below \( Var(\epsilon) \), the irreducible error. + +

+What do we mean by the variance and bias of a statistical learning +method? The variance refers to the amount by which our model would change if we +estimated it using a different training data set. Since the training +data are used to fit the statistical learning method, different +training data sets will result in a different estimate. But ideally the +estimate for our model should not vary too much between training +sets. However, if a method has high variance then small changes in +the training data can result in large changes in the model. In general, more +flexible statistical methods have higher variance. + +

+You may also find this recent article of interest. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs072.html b/doc/pub/week36/html/._week36-bs072.html new file mode 100644 index 000000000..b6f2c5bff --- /dev/null +++ b/doc/pub/week36/html/._week36-bs072.html @@ -0,0 +1,503 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Another Example from Scikit-Learn's Repository

+

+ + +

"""
+============================
+Underfitting vs. Overfitting
+============================
+
+This example demonstrates the problems of underfitting and overfitting and
+how we can use linear regression with polynomial features to approximate
+nonlinear functions. The plot shows the function that we want to approximate,
+which is a part of the cosine function. In addition, the samples from the
+real function and the approximations of different models are displayed. The
+models have polynomial features of different degrees. We can see that a
+linear function (polynomial with degree 1) is not sufficient to fit the
+training samples. This is called **underfitting**. A polynomial of degree 4
+approximates the true function almost perfectly. However, for higher degrees
+the model will **overfit** the training data, i.e. it learns the noise of the
+training data.
+We evaluate quantitatively **overfitting** / **underfitting** by using
+cross-validation. We calculate the mean squared error (MSE) on the validation
+set, the higher, the less likely the model generalizes correctly from the
+training data.
+"""
+
+print(__doc__)
+
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import cross_val_score
+
+
+def true_fun(X):
+    return np.cos(1.5 * np.pi * X)
+
+np.random.seed(0)
+
+n_samples = 30
+degrees = [1, 4, 15]
+
+X = np.sort(np.random.rand(n_samples))
+y = true_fun(X) + np.random.randn(n_samples) * 0.1
+
+plt.figure(figsize=(14, 5))
+for i in range(len(degrees)):
+    ax = plt.subplot(1, len(degrees), i + 1)
+    plt.setp(ax, xticks=(), yticks=())
+
+    polynomial_features = PolynomialFeatures(degree=degrees[i],
+                                             include_bias=False)
+    linear_regression = LinearRegression()
+    pipeline = Pipeline([("polynomial_features", polynomial_features),
+                         ("linear_regression", linear_regression)])
+    pipeline.fit(X[:, np.newaxis], y)
+
+    # Evaluate the models using crossvalidation
+    scores = cross_val_score(pipeline, X[:, np.newaxis], y,
+                             scoring="neg_mean_squared_error", cv=10)
+
+    X_test = np.linspace(0, 1, 100)
+    plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model")
+    plt.plot(X_test, true_fun(X_test), label="True function")
+    plt.scatter(X, y, edgecolor='b', s=20, label="Samples")
+    plt.xlabel("x")
+    plt.ylabel("y")
+    plt.xlim((0, 1))
+    plt.ylim((-2, 2))
+    plt.legend(loc="best")
+    plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format(
+        degrees[i], -scores.mean(), scores.std()))
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs073.html b/doc/pub/week36/html/._week36-bs073.html new file mode 100644 index 000000000..da8f6d8d9 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs073.html @@ -0,0 +1,510 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

More examples on bootstrap and cross-validation and errors

+ +

+ + +

# Common imports
+import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.model_selection import train_test_split
+from sklearn.utils import resample
+from sklearn.metrics import mean_squared_error
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("EoS.csv"),'r')
+
+# Read the EoS data as  csv file and organize the data into two arrays with density and energies
+EoS = pd.read_csv(infile, names=('Density', 'Energy'))
+EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')
+EoS = EoS.dropna()
+Energies = EoS['Energy']
+Density = EoS['Density']
+#  The design matrix now as function of various polytrops
+
+Maxpolydegree = 30
+X = np.zeros((len(Density),Maxpolydegree))
+X[:,0] = 1.0
+testerror = np.zeros(Maxpolydegree)
+trainingerror = np.zeros(Maxpolydegree)
+polynomial = np.zeros(Maxpolydegree)
+
+trials = 100
+for polydegree in range(1, Maxpolydegree):
+    polynomial[polydegree] = polydegree
+    for degree in range(polydegree):
+        X[:,degree] = Density**(degree/3.0)
+
+# loop over trials in order to estimate the expectation value of the MSE
+    testerror[polydegree] = 0.0
+    trainingerror[polydegree] = 0.0
+    for samples in range(trials):
+        x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)
+        model = LinearRegression(fit_intercept=True).fit(x_train, y_train)
+        ypred = model.predict(x_train)
+        ytilde = model.predict(x_test)
+        testerror[polydegree] += mean_squared_error(y_test, ytilde)
+        trainingerror[polydegree] += mean_squared_error(y_train, ypred) 
+
+    testerror[polydegree] /= trials
+    trainingerror[polydegree] /= trials
+    print("Degree of polynomial: %3d"% polynomial[polydegree])
+    print("Mean squared error on training data: %.8f" % trainingerror[polydegree])
+    print("Mean squared error on test data: %.8f" % testerror[polydegree])
+
+plt.plot(polynomial, np.log10(trainingerror), label='Training Error')
+plt.plot(polynomial, np.log10(testerror), label='Test Error')
+plt.xlabel('Polynomial degree')
+plt.ylabel('log10[MSE]')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs074.html b/doc/pub/week36/html/._week36-bs074.html new file mode 100644 index 000000000..549141174 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs074.html @@ -0,0 +1,498 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The same example but now with cross-validation

+ +

+ + +

# Common imports
+import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.metrics import mean_squared_error
+from sklearn.model_selection import KFold
+from sklearn.model_selection import cross_val_score
+
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("EoS.csv"),'r')
+
+# Read the EoS data as  csv file and organize the data into two arrays with density and energies
+EoS = pd.read_csv(infile, names=('Density', 'Energy'))
+EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')
+EoS = EoS.dropna()
+Energies = EoS['Energy']
+Density = EoS['Density']
+#  The design matrix now as function of various polytrops
+
+Maxpolydegree = 30
+X = np.zeros((len(Density),Maxpolydegree))
+X[:,0] = 1.0
+estimated_mse_sklearn = np.zeros(Maxpolydegree)
+polynomial = np.zeros(Maxpolydegree)
+k =5
+kfold = KFold(n_splits = k)
+
+for polydegree in range(1, Maxpolydegree):
+    polynomial[polydegree] = polydegree
+    for degree in range(polydegree):
+        X[:,degree] = Density**(degree/3.0)
+        OLS = LinearRegression()
+# loop over trials in order to estimate the expectation value of the MSE
+    estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold)
+#[:, np.newaxis]
+    estimated_mse_sklearn[polydegree] = np.mean(-estimated_mse_folds)
+
+plt.plot(polynomial, np.log10(estimated_mse_sklearn), label='Test Error')
+plt.xlabel('Polynomial degree')
+plt.ylabel('log10[MSE]')
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/html/._week36-bs075.html b/doc/pub/week36/html/._week36-bs075.html new file mode 100644 index 000000000..b6a328b71 --- /dev/null +++ b/doc/pub/week36/html/._week36-bs075.html @@ -0,0 +1,464 @@ + + + + + + + + +Week 36: Statistical interpretation of Linear Regression and Resampling techniques + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Week 36: Statistical interpretation of Linear Regression and Resampling techniques +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Cross-validation with Ridge

+

+ + +

import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+    ridge = Ridge(alpha = lmb)
+    estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+    estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+    i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+

+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/week36/ipynb/.ipynb_checkpoints/week36-checkpoint.ipynb b/doc/pub/week36/ipynb/.ipynb_checkpoints/week36-checkpoint.ipynb new file mode 100644 index 000000000..c8927a608 --- /dev/null +++ b/doc/pub/week36/ipynb/.ipynb_checkpoints/week36-checkpoint.ipynb @@ -0,0 +1,3108 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "# Week 36: Statistical interpretation of Linear Regression and Resampling techniques\n", + "\n", + " \n", + "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", + "\n", + "Date: **Sep 9, 2021**\n", + "\n", + "Copyright 1999-2021, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n", + "\n", + "\n", + "\n", + "\n", + "## Plans for week 36\n", + "\n", + "* Thursday: Summary from last week on SVD, Statistics, probability theory and linear regression\n", + "\n", + "* Friday: Linear Regression and links with Statistics, Resampling methods and presentation of first project.\n", + "\n", + "Recommended Reading:\n", + "1. Lectures on Regression\n", + "\n", + "2. Bishop 1.1, 1.2, 2.1, 2.2, 2.3 and 3.1\n", + "\n", + "3. Hastie et al chapter 3\n", + "\n", + "## Thursday September 9\n", + "\n", + "\n", + "## Summary from last Week and Examples\n", + "\n", + "## Linear Regression and the SVD\n", + "\n", + "We used the SVD to analyse the matrix to invert in ordinary lineat regression" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T=\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{V}^T.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Since the matrices here have dimension $p\\times p$, with $p$ corresponding to the singular values, we defined last week the matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma} = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\\\ \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where the tilde-matrix $\\tilde{\\boldsymbol{\\Sigma}}$ is a matrix of dimension $p\\times p$ containing only the singular values $\\sigma_i$, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\tilde{\\boldsymbol{\\Sigma}}=\\begin{bmatrix} \\sigma_0 & 0 & 0 & \\dots & 0 & 0 \\\\\n", + " 0 & \\sigma_1 & 0 & \\dots & 0 & 0 \\\\\n", + "\t\t\t\t 0 & 0 & \\sigma_2 & \\dots & 0 & 0 \\\\\n", + "\t\t\t\t 0 & 0 & 0 & \\dots & \\sigma_{p-2} & 0 \\\\\n", + "\t\t\t\t 0 & 0 & 0 & \\dots & 0 & \\sigma_{p-1} \\\\\n", + "\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "meaning we can write" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{X}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2\\boldsymbol{V}^T.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Multiplying from the right with $\\boldsymbol{V}$ (using the orthogonality of $\\boldsymbol{V}$) we get" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{V}=\\boldsymbol{V}\\tilde{\\boldsymbol{\\Sigma}}^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## What does it mean?\n", + "\n", + "This means the vectors $\\boldsymbol{v}_i$ of the orthogonal matrix $\\boldsymbol{V}$\n", + "are the eigenvectors of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$ with eigenvalues\n", + "given by the singular values squared, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)\\boldsymbol{v}_i=\\boldsymbol{v}_i\\sigma_i^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In other words, each non-zero singular value of $\\boldsymbol{X}$ is a positive\n", + "square root of an eigenvalue of $\\boldsymbol{X}^T\\boldsymbol{X}$. It means also that\n", + "the columns of $\\boldsymbol{V}$ are the eigenvectors of\n", + "$\\boldsymbol{X}^T\\boldsymbol{X}$. Since we have ordered the singular values of\n", + "$\\boldsymbol{X}$ in a descending order, it means that the column vectors\n", + "$\\boldsymbol{v}_i$ are hierarchically ordered by how much correlation they\n", + "encode from the columns of $\\boldsymbol{X}$. \n", + "\n", + "\n", + "Note that these are also the eigenvectors and eigenvalues of the\n", + "Hessian matrix.\n", + "\n", + "If we now recall the definition of the covariance matrix (not using\n", + "Bessel's correction) we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{C}[\\boldsymbol{X}]=\\frac{1}{n}\\boldsymbol{X}^T\\boldsymbol{X},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "meaning that every squared non-singular value of $\\boldsymbol{X}$ divided by $n$ (\n", + "the number of samples) are the eigenvalues of the covariance\n", + "matrix. Every singular value of $\\boldsymbol{X}$ is thus a positive square\n", + "root of an eigenvalue of $\\boldsymbol{X}^T\\boldsymbol{X}$. If the matrix $\\boldsymbol{X}$ is\n", + "self-adjoint, the singular values of $\\boldsymbol{X}$ are equal to the\n", + "absolute value of the eigenvalues of $\\boldsymbol{X}$.\n", + "\n", + "## And finally $\\boldsymbol{X}\\boldsymbol{X}^T$\n", + "\n", + "For $\\boldsymbol{X}\\boldsymbol{X}^T$ we found" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}\\boldsymbol{V}^T\\boldsymbol{V}\\boldsymbol{\\Sigma}^T\\boldsymbol{U}^T=\\boldsymbol{U}\\boldsymbol{\\Sigma}^T\\boldsymbol{\\Sigma}\\boldsymbol{U}^T.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Since the matrices here have dimension $n\\times n$, we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\Sigma}\\boldsymbol{\\Sigma}^T = \\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\\\ \\boldsymbol{0}\\\\ \\end{bmatrix}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} \\boldsymbol{0}\\\\ \\end{bmatrix}=\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "leading to" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}\\boldsymbol{X}^T=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}\\boldsymbol{U}^T.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Multiplying with $\\boldsymbol{U}$ from the right gives us the eigenvalue problem" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "(\\boldsymbol{X}\\boldsymbol{X}^T)\\boldsymbol{U}=\\boldsymbol{U}\\begin{bmatrix} \\tilde{\\boldsymbol{\\Sigma}} & \\boldsymbol{0} \\\\ \\boldsymbol{0} & \\boldsymbol{0}\\\\ \\end{bmatrix}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It means that the eigenvalues of $\\boldsymbol{X}\\boldsymbol{X}^T$ are again given by\n", + "the non-zero singular values plus now a series of zeros. The column\n", + "vectors of $\\boldsymbol{U}$ are the eigenvectors of $\\boldsymbol{X}\\boldsymbol{X}^T$ and\n", + "measure how much correlations are contained in the rows of $\\boldsymbol{X}$.\n", + "\n", + "Since we will mainly be interested in the correlations among the features\n", + "of our data (the columns of $\\boldsymbol{X}$, the quantity of interest for us are the non-zero singular\n", + "values and the column vectors of $\\boldsymbol{V}$.\n", + "\n", + "\n", + "## Ridge and LASSO Regression\n", + "\n", + "Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is \n", + "our optimization problem is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "or we can state it as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we have used the definition of a norm-2 vector, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## From OLS to Ridge and Lasso\n", + "\n", + "By minimizing the above equation with respect to the parameters\n", + "$\\boldsymbol{\\beta}$ we could then obtain an analytical expression for the\n", + "parameters $\\boldsymbol{\\beta}$. We can add a regularization parameter $\\lambda$ by\n", + "defining a new cost function to be optimized, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which leads to the Ridge regression minimization problem where we\n", + "require that $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\\le t$, where $t$ is\n", + "a finite number larger than zero. By defining" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have a new optimization equation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n", + "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n", + "\n", + "Here we have defined the norm-1 as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deriving the Ridge Regression Equations\n", + "\n", + "Using the matrix-vector expression for Ridge regression and dropping the parameter $1/n$ in front of the standard means squared error equation, we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\boldsymbol{\\beta}^T\\boldsymbol{\\beta},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and \n", + "taking the derivatives with respect to $\\boldsymbol{\\beta}$ we obtain then\n", + "a slightly modified matrix inversion problem which for finite values\n", + "of $\\lambda$ does not suffer from singularity problems. We obtain\n", + "the optimal parameters" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $\\boldsymbol{I}$ being a $p\\times p$ identity matrix with the constraint that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\sum_{i=0}^{p-1} \\beta_i^2 \\leq t,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with $t$ a finite positive number. \n", + "\n", + "When we compare this with the ordinary least squares result we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{OLS}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$.\n", + "\n", + "\n", + "We see that Ridge regression is nothing but the standard OLS with a\n", + "modified diagonal term added to $\\boldsymbol{X}^T\\boldsymbol{X}$. The consequences, in\n", + "particular for our discussion of the bias-variance tradeoff are rather\n", + "interesting. We will see that for specific values of $\\lambda$, we may\n", + "even reduce the variance of the optimal parameters $\\boldsymbol{\\beta}$. These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.\n", + "\n", + "## SVD analysis\n", + "\n", + "Using our insights about the SVD of the design matrix $\\boldsymbol{X}$ \n", + "We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix $\\boldsymbol{U}$ as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\tilde{\\boldsymbol{y}}_{\\mathrm{OLS}}=\\boldsymbol{X}\\boldsymbol{\\beta} =\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For Ridge regression this becomes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\tilde{\\boldsymbol{y}}_{\\mathrm{Ridge}}=\\boldsymbol{X}\\boldsymbol{\\beta}_{\\mathrm{Ridge}} = \\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{\\Sigma}^2\\boldsymbol{V}^T+\\lambda\\boldsymbol{I} \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\sum_{j=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{y},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "with the vectors $\\boldsymbol{u}_j$ being the columns of $\\boldsymbol{U}$ from the SVD of the matrix $\\boldsymbol{X}$. \n", + "\n", + "## Interpreting the Ridge results\n", + "\n", + "Since $\\lambda \\geq 0$, it means that compared to OLS, we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda} \\leq 1.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ridge regression finds the coordinates of $\\boldsymbol{y}$ with respect to the\n", + "orthonormal basis $\\boldsymbol{U}$, it then shrinks the coordinates by\n", + "$\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}$. Recall that the SVD has\n", + "eigenvalues ordered in a descending way, that is $\\sigma_i \\geq\n", + "\\sigma_{i+1}$.\n", + "\n", + "For small eigenvalues $\\sigma_i$ it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.\n", + "\n", + "\n", + "## More interpretations\n", + "\n", + "For the sake of simplicity, let us assume that the design matrix is orthonormal, that is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{X}=(\\boldsymbol{X}^T\\boldsymbol{X})^{-1} =\\boldsymbol{I}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this case the standard OLS results in" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta}^{\\mathrm{OLS}} = \\boldsymbol{X}^T\\boldsymbol{y}=\\sum_{i=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\boldsymbol{y},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{I}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\left(1+\\lambda\\right)^{-1}\\boldsymbol{\\beta}^{\\mathrm{OLS}},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\\lambda$, and\n", + "the Ridge estimator converges to zero when the hyperparameter goes to\n", + "infinity.\n", + "\n", + "We will come back to more interpreations after we have gone through some of the statistical analysis part. \n", + "\n", + "For more discussions of Ridge and Lasso regression, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n", + "Similarly, [Mehta et al's article](https://arxiv.org/abs/1803.08823) is also recommended.\n", + "\n", + "## Deriving the Lasso Regression Equations\n", + "\n", + "Using the matrix-vector expression for Lasso regression and dropping the parameter $1/n$ in front of the standard mean squared error equation, we have the following **cost** function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Taking the derivative with respect to $\\boldsymbol{\\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{d \\vert \\beta\\vert}{d \\boldsymbol{\\beta}}=\\mathrm{sgn}(\\boldsymbol{\\beta})=\\left\\{\\begin{array}{cc} 1 & \\beta > 0 \\\\ 0 & \\beta =0\\\\-1 & \\beta < 0, \\end{array}\\right.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have that the derivative of the cost function is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{X},\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}=-2\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})+\\lambda sgn(\\boldsymbol{\\beta})=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and reordering we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta}+\\lambda sgn(\\boldsymbol{\\beta})=2\\boldsymbol{X}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This equation does not lead to a nice analytical equation as in Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package [CVXOPT](https://cvxopt.org/). We will discuss this later. \n", + "\n", + "\n", + "\n", + "\n", + "## Simple example to illustrate Ordinary Least Squares, Ridge and Lasso Regression\n", + "\n", + "Let us assume that our design matrix is given by unit (identity) matrix, that is a square diagonal matrix with ones only along the\n", + "diagonal. In this case we have an equal number of rows and columns $n=p$.\n", + "\n", + "Our model approximation is just $\\tilde{\\boldsymbol{y}}=\\boldsymbol{\\beta}$ and the mean squared error and thereby the cost function for ordinary least sqquares (OLS) is then (we drop the term $1/n$)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and minimizing we have that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\beta}_i^{\\mathrm{OLS}} = y_i.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Ridge Regression\n", + "\n", + "For Ridge regression our cost function is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2+\\lambda\\sum_{i=0}^{p-1}\\beta_i^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and minimizing we have that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\beta}_i^{\\mathrm{Ridge}} = \\frac{y_i}{1+\\lambda}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Lasso Regression\n", + "\n", + "For Lasso regression our cost function is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2+\\lambda\\sum_{i=0}^{p-1}\\vert\\beta_i\\vert=\\sum_{i=0}^{p-1}(y_i-\\beta_i)^2+\\lambda\\sum_{i=0}^{p-1}\\sqrt{\\beta_i^2},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and minimizing we have that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "-2\\sum_{i=0}^{p-1}(y_i-\\beta_i)+\\lambda \\sum_{i=0}^{p-1}\\frac{(\\beta_i)}{\\vert\\beta_i\\vert}=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which leads to" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}_i^{\\mathrm{Lasso}} = \\left\\{\\begin{array}{ccc}y_i-\\frac{\\lambda}{2} &\\mathrm{if} & y_i> \\frac{\\lambda}{2}\\\\\n", + " y_i+\\frac{\\lambda}{2} &\\mathrm{if} & y_i< -\\frac{\\lambda}{2}\\\\\n", + "\t\t\t\t\t\t\t 0 &\\mathrm{if} & \\vert y_i\\vert\\le \\frac{\\lambda}{2}\\end{array}\\right.\\\\.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Plotting these results (figure in handwritten notes for week 36) shows clearly that Lasso regression suppresses (sets to zero) values of $\\beta_i$ for specific values of $\\lambda$. Ridge regression reduces on the other hand the values of $\\beta_i$ as function of $\\lambda$.\n", + "\n", + "We will now couple the discussions of ordinary least squares, Ridge and Lasso regression with a statistical interpretation, that is we move from a linear algebra analysis to a statistical analysis. In particular, we will focus on what the regularization terms can result in.\n", + "We will amongst other things show that the regularization parameter can reduce considerably the variance of the parameters $\\beta$.\n", + "\n", + "\n", + "## Yet another Example\n", + "\n", + "Let us assume we have a data set with outputs/targets given by the vector" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y}=\\begin{bmatrix}4 \\\\ 2 \\\\3\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and our inputs as a $3\\times 2$ design matrix" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}=\\begin{bmatrix}2 & 0\\\\ 0 & 1 \\\\ 0 & 0\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "meaning that we have two features and two unknown parameters $\\beta_0$ and $\\beta_1$ to be determined either by ordinary least squares, Ridge or Lasso regression.\n", + "\n", + "## The OLS case\n", + "\n", + "For ordinary least squares (OLS) we know that the optimal solution is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left( \\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Inserting the above values we obtain that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\begin{bmatrix}2 \\\\ 2\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The code which implements this simpler case is presented after the discussion of Ridge and Lasso.\n", + "\n", + "## The Ridge case\n", + "\n", + "For Ridge regression we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}=\\left( \\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Inserting the above values we obtain that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{Ridge}}=\\begin{bmatrix}\\frac{8}{4+\\lambda} \\\\ \\frac{2}{1+\\lambda}\\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There is normally a constraint on the value of $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2$ via the parameter $\\lambda$.\n", + "Let us for simplicity assume that $\\beta_0^2+\\beta_1^2=1$ as constraint. This will allow us to find an expression for the optimal values of $\\beta$ and $\\lambda$.\n", + "\n", + "To see this, let us write the cost function for Ridge regression. \n", + "\n", + "\n", + "## Writing the Cost Function\n", + "\n", + "We define the MSE without the $1/n$ factor and have then, using that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}\\boldsymbol{\\beta}=\\begin{bmatrix} 2\\beta_0 \\\\ \\beta_1 \\\\0 \\end{bmatrix},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=(4-2\\beta_0)^2+(2-\\beta_1)^2+\\lambda(\\beta_0^2+\\beta_1^2),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and taking the derivative with respect to $\\beta_0$ we get" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_0=\\frac{8}{4+\\lambda},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and for $\\beta_1$ we obtain" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_1=\\frac{2}{1+\\lambda},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the constraint for $\\beta_0^2+\\beta_1^2=1$ we can constrain $\\lambda$ by solving" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\left(\\frac{8}{4+\\lambda}\\right)^2+\\left(\\frac{2}{1+\\lambda}\\right)^2=1,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which gives $\\lambda=4.571$ and $\\beta_0=0.933$ and $\\beta_1=0.359$.\n", + "\n", + "## Lasso case\n", + "\n", + "For Lasso we need now, keeping a constraint on $\\vert\\beta_0\\vert+\\vert\\beta_1\\vert=1$, to take the derivative of the absolute values of $\\beta_0$\n", + "and $\\beta_1$. This gives us the following derivatives of the cost function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta})=(4-2\\beta_0)^2+(2-\\beta_1)^2+\\lambda(\\vert\\beta_0\\vert+\\vert\\beta_1\\vert),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_0}=-4(4-2\\beta_0)+\\lambda\\mathrm{sgn}(\\beta_0)=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\frac{\\partial C(\\boldsymbol{\\beta})}{\\partial \\beta_1}=-2(2-\\beta_1)+\\lambda\\mathrm{sgn}(\\beta_1)=0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have now four cases to solve besides the trivial cases $\\beta_0$ and/or $\\beta_1$ are zero, namely\n", + "1. $\\beta_0 > 0$ and $\\beta_1 > 0$,\n", + "\n", + "2. $\\beta_0 > 0$ and $\\beta_1 < 0$,\n", + "\n", + "3. $\\beta_0 < 0$ and $\\beta_1 > 0$,\n", + "\n", + "4. $\\beta_0 < 0$ and $\\beta_1 < 0$.\n", + "\n", + "## The first Case\n", + "\n", + "If we consider the first case, we have then" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "-4(4-2\\beta_0)+\\lambda=0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "-2(2-\\beta_1)+\\lambda=0.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which yields" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_0=\\frac{16+\\lambda}{8},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\beta_1=\\frac{4+\\lambda}{2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the constraint on $\\beta_0$ and $\\beta_1$ we can then find the optimal value of $\\lambda$ for the different cases. We leave this as an exercise to you.\n", + "\n", + "## Simple code for solving the above problem\n", + "\n", + "Here we set up the OLS, Ridge and Lasso functionality in order to study the above example. Note that here we have opted for a set of values of $\\lambda$, meaning that we need to perform a search in order to find the optimal values.\n", + "\n", + "First we study and compare the OLS and Ridge results. The next code compares all three methods." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2. 2.]\n", + "Training MSE for OLS\n", + "3.0\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjmElEQVR4nO3deXxV1bn/8c/DJKMgECoyCFalCE4YQVBRqzggRUUriMNFRPQ6oNepWr16by23WLy/q9IWKsWCBZEKgspUaRWHFkSiiEGcsFgmMaBEUIRAnt8f6wQSCAkJ2dln+L5fr/PKOWftc/YXAk9W1l5nLXN3REQk/dSIO4CIiERDBV5EJE2pwIuIpCkVeBGRNKUCLyKSpmrFHaC45s2be7t27eKOISKSMnJycja4e1ZpbUlV4Nu1a8fixYvjjiEikjLM7PN9tWmIRkQkTanAi4ikKRV4EZE0lVRj8KUpKChg9erVfP/993FHkQqoW7curVu3pnbt2nFHEclYSV/gV69eTaNGjWjXrh1mFncc2Q/uzsaNG1m9ejXt27ePO45Ixkr6IZrvv/+eZs2aqbinEDOjWbNm+q1LJGZJX+ABFfcUpO+ZSPySfohGRCRW330H334L338PW7fCtm1w0EFw9NGh/c03YdMmKCjYfTv0UDj77NA+YQLk58POnbtvHTrAxReH9l/9Cq6/Hpo3r/LoKvD7wcy48sormThxIgA7duygZcuWdOvWjZkzZ7J+/Xquu+46Vq1aRUFBAe3atWP27NmsXLmSjh070qFDh13vdccdd3DNNdeUeP8zzzyTdevWUbduXerUqcPYsWM54YQTAOjduzfPPPMMTZo0KfGa//qv/6Jhw4bcddddlf5zbdy4kbMT/wi/+OILatasSVZW+EDcokWLqFOnTrnvMX/+fOrUqUOPHj0qnUMkUl9/DWvWQF5euG3YAGbw7/8e2h96CP7xD/jmG9i8GbZsgfbt4bXXQvtZZ8GiRSXfs0cP+Pvfw/0bboAPPijZft55uwv8f/4nrFpVsv2yy3YX+F//OtxXgY9HgwYNyM3NZevWrdSrV4958+bRqlWrXe0PPvggvXr14rbbbgNg6dKlu9p++MMfsmTJknLPMWnSJLKzs/njH//I3Xffzbx58wCYPXt21f5himnWrNmubJX9gTF//nwaNmyoAi/xWb8eli2DTz+Fzz6Dzz8PPeai/ztDh8LUqSVf84Mf7C7wX30VeuiHHAJt20LDhqHAF/mP/wg/FOrW3X1r0WJ3+zPPwPbtULv27lujRrvb3303fK1Zc/etVrHS+9VX4QdOBFTg91Pv3r2ZNWsWl112GZMnT+aKK67gjTfeAGDdunWce+65u4497rjjKn2e7t27M3LkyF2Pi5ZvaN68OcOHD2fChAm0aNGCNm3acNJJJwHw9ttvc91111GjRg169erFnDlzyM3NZefOndx7773Mnz+fbdu2cfPNN3PDDTeUmyEnJ4c77riDLVu20Lx5c8aPH0/Lli154oknGDNmDLVq1eKYY45hxIgRjBkzhpo1azJx4kRGjRrF6aefXuk/u0iZvv02FMvFi2HpUhg9OgyVjBgBjz0WjqlTB9q0gcMPhx07QiG95ZbQY87KCoW5eXNo2nT3+44aVfZ5Bwwou/3448tub9as7PYIr1elXoE/88y9n7v8crjppjBW1rv33u2DBoXbhg3hG13c/Pn7ddoBAwbwi1/8gj59+rB06VIGDx68q8DffPPN9O/fn9/85jecc845XHvttRx22GEArFixYtdwC1BuEZw7dy4XF/3qVkxOTg7PPvssS5YsYceOHXTp0mVXgb/22msZO3Ys3bt359577931mnHjxtG4cWPefvtttm3bxqmnnsq5555b5tTFgoICbr31Vl544QWysrKYMmUK999/P0899RQjRozgn//8JwcddBCbNm2iSZMm3HjjjQc8VCRSptmz4b77IDcXCgvDcz/4AaxdG3raQ4bAT34CRx0FrVpBjT3mjpxxRvVnThKpV+Bjctxxx7Fy5UomT55M7z1+iJx33nl89tlnzJ07lzlz5nDiiSeSm5sL7P8QzZVXXsn27dvZsmVLqce/8cYbXHLJJdSvXx+Avn37ArBp0yY2b95M9+7dARg4cCAzZ84E4OWXX2bp0qVMTfx6mp+fzyeffFJmgf/oo4/Izc2lV69eAOzcuZOWLVvu+ju48sorufjii0v9ISRyQNxhyRJ46SWYNStcfPzxj8NwR4sW8MADcPLJkJ0dLmIW6dQp3GQvqVfgy+px169fdnvz5vvdYy9N3759ueuuu5g/fz4bN24s0da0aVMGDhzIwIED6dOnD6+//vquHvb+mDRpEieddBJ33303t956K88//3ylcxZxd0aNGsV5551Xodd06tSJBQsW7NU2a9YsXn/9dV566SWGDx/O+++/f8AZRdiyBR59NIxlf/JJGLLo2jUMsQCcfjokrklJxaTEPPhkMXjwYB566CGOPfbYEs+/8sorfPfddwBs3ryZFStW0LZt2wq/v5nx8MMPs3DhQj788MMSbT179mTGjBls3bqVzZs389JLLwHQpEkTGjVqxFtvvQXAs88+u+s15513HqNHj6agoACAjz/+mG+//bbMDB06dCAvL29XgS8oKGDZsmUUFhayatUqzjrrLB555BHy8/PZsmULjRo1YvPmzRX+s0qG27EjFHMI4+hjxkDr1vDkk/DFF7BwIRS7riWVk3o9+Bi1bt2aYcOG7fV8Tk4Ot9xyC7Vq1aKwsJAhQ4Zw8skns3Llyr3G4AcPHlzqexSpV68ed955JyNHjmTcuHG7nu/SpQv9+/fn+OOPp0WLFpx88sm72saNG8f1119PjRo1OOOMM2jcuDEAQ4YMYeXKlXTp0gV3JysrixkzZpT5Z6xTpw5Tp05l2LBh5Ofns2PHDm6//XaOPvporrrqKvLz83F3hg0bRpMmTfjJT37CZZddxgsvvKCLrFK+rVth7FgYOTKMlX/2WZh1smIFNGgQd7q0Y+4ed4ZdsrOzfc8NP5YvX07Hjh1jSpQatmzZQsOGDQEYMWIE69at4/HHH485lb53Usy2baGXPmJE6KH37Al33gkXXhimDUqlmVmOu2eX1qYefBqYNWsWv/rVr9ixYweHH34448ePjzuSSEkvvwy33x5mwU2eXPpsOKlyKvBpoH///vTv3z/uGCIlff55+AToT38KffrAggVwyilxp8ooKXGRNZmGkWT/6HuWwdzhT3+C444LHzL6/vswM0bFvdolfYGvW7cuGzduVMFIIUXrwdetWzfuKFLdvv46fPLzmmvg2GPDbBj9O4hN0g/RtG7dmtWrV5OXlxd3FKmAoh2dJIN8+y106wb//Cf8z//APffoAmrMkr7A165dW7sCiaSCBg3Cwl49eoSbxC7pC7yIJLnRo8NwzGmngdYkSipJPwYvIkns0UfDQn9PPRV3EilFpAXezG4zs1wzW2Zmt0d5LhGpZr/8Jdx9d1jN9fe/jzuNlCKyAm9mnYHrga7A8UAfMzsyqvOJSDV6+OGwU9HVV8OkSWG5AUk6UfbgOwJvuft37r4DeA3oF+H5RKQ6FBbCe++FqZDjx5fcnUiSSpTfmVxguJk1A7YCvYHFex5kZkOBoUClVmAUkWrkHhYJmzIlFPo9N9eQpBLZd8fdlwOPAC8Dc4ElwM5SjnvS3bPdPbtow2cRSUI5OWH64+rVYX67hmWSXqQ/ft19nLuf5O49ga+Bj6M8n4hEZMMGuPhiWLdOH15KIZEOnplZC3f/0szaEsbftRiFSKopLISrroK8vLBgWGILR0l+UV8dmZYYgy8Abnb3TRGfT0Sq2vDh8Je/hKmQJ54YdxqpgEgLvLtrex+RVLZ9O0yfHnrw118fdxqpIM1vEpF9q1MH/v73MExjFncaqSDNcRKR0k2YAPn5UK+e9ktNUSrwIrK3OXNg0CD47W/jTiIHQAVeRErKzw/L/h5zTNgYW1KWxuBFpKR77oG1a2HaNDjooLjTyAFQD15Edlu8GMaOhdtvh65d404jB0gFXkR2a9o0TIl86KG4k0gV0BCNiOx2xBHw9NNxp5Aqoh68iMB338GQIbBiRdxJpAqpwIsI/PrXMG5cuLgqaUMFXiTTrV8fCvzll8PpWl0knajAi2S6Rx4Ja8788pdxJ5EqpgIvksnWroXRo8PeqkcdFXcaqWKaRSOSyerWhZtvhptuijuJREAFXiSTNW0Kjz4adwqJiIZoRDLV//0f/PWvcaeQCKnAi2SitWvhZz+D55+PO4lESAVeJBM98QTs3Al33RV3EomQCrxIptm8GcaMgUsvDUsTSNpSgRfJNOPGhTXftdZ72ou0wJvZf5jZMjPLNbPJZlY3yvOJyH5o0gSuuAK6dYs7iUQssgJvZq2AYUC2u3cGagIDojqfiOynQYPgmWfiTiHVIOohmlpAPTOrBdQHtJKRSFzcYfp02LYt7iRSTSIr8O6+BngU+BewDsh395ejOp+IlGPBAujXDyZOjDuJVJMoh2gOAS4C2gOHAQ3M7KpSjhtqZovNbHFeXl5UcURk9Gho1Aj69487iVSTKIdozgH+6e557l4APA/02PMgd3/S3bPdPTsrKyvCOCIZbMMGeO45uOYaaNgw7jRSTaIs8P8CTjGz+mZmwNnA8gjPJyL7MmFCGHu/8ca4k0g1inIM/i1gKvAO8H7iXE9GdT4RKcMbb8Bpp0HnznEnkWoU6WqS7v4QoO3ZReI2fTp8/XXcKaSa6ZOsIulu+3YwC0sDS0ZRgRdJZ+vWwaGHhh68ZBwVeJF0NnFiGJrp1CnuJBIDFXiRdOUO48dD9+5w9NFxp5EYqMCLpKucHPjgg7D2jGQkFXiRdDV+fNhU+/LL404iMdGm2yLpasgQyM4OywNLRlKBF0lXJ5wQbpKxNEQjko6eeAIWLow7hcRMBV4k3axfD3fcATNmxJ1EYqYCL5JunnsOdu6Eq6+OO4nETAVeJN1MngzHHqsPN4kKvEha+fxz+Mc/wqbakvFU4EXSyccfQ4sW2rVJAE2TFEkvvXrB2rVQs2bcSSQJqAcvki62boXCQhV32UUFXiRdPPIIHHlkKPQiqMCLpAf3MHumXTuoVy/uNJIkVOBF0sGSJeEC64ABcSeRJKICL5IOnnsujL336xd3EkkiKvAiqc4dpk6Fs86C5s3jTiNJJLJpkmbWAZhS7KkjgAfd/bGozimSkdzh0UehYcO4k0iSiazAu/tHwAkAZlYTWANo51+RqlajBvTtG3cKSULVNURzNrDC3T+vpvOJZI4nnoBPPok7hSSh6vok6wBgcmkNZjYUGArQtm3baoojkiY+/BBuuw3M4Kij4k4jSSbyHryZ1QH6As+V1u7uT7p7trtnZ2VlRR1HJL1Mmxa+avaMlKI6hmguAN5x9/XVcC6RzDJtGvToAa1axZ1EklB1FPgr2MfwjIgcgBUr4N134dJL404iSSrSAm9mDYBewPNRnkckI733HtStq+EZ2adIL7K6+7dAsyjPIZKx+vWDr77S2jOyT/okq0gqcg9fVdylDCrwIqlo3Djo2hU2bow7iSQxFXiRVDR9OmzYAE2bxp1EkpgKvEiq2bwZ/vpXuPji8AEnkX1QgRdJNXPmwPbtcMklcSeRJKcCL5JqZswIywL36BF3EklyZRZ4M7uq2P1T92i7JapQIlKGCy6ABx7Q5tpSrvJ68HcUuz9qj7bBVZxFRPbH1VeHBcZEylFegbd93C/tsYhE7c03Yd26uFNIiiivwPs+7pf2WESi5B421b7ppriTSIoob6mCH5nZUkJv/YeJ+yQeHxFpMhEp6Z13YM0a+OUv404iKaK8At+xWlKISPlefDFsz3fhhXEnkRRRZoHfc4s9M2sG9AT+5e45UQYTkT28+GKYGqmNcWQ/lTdNcqaZdU7cbwnkEmbP/MnMbo8+nogA4cLqkiVw0UVxJ5EUUt4QTXt3z03cvxaY5+7XmFkj4O/AY1GGE5GEli1h1aqw/rvIfiqvwBcUu382MBbA3TebWWFkqURkb61bx51AUkx50yRXmdmtZnYJ0AWYC2Bm9YDaUYcTESA/PywstmhR3EkkxZRX4K8DOgGDgP7uvinx/CnAH6OLJSK7zJ0LL7wABQXlHytSTHmzaL4Ebizl+VeBV6MKJSLFvPhimDlzyilxJ5EUU2aBN7MXy2p3975VG0dESigogNmzw9LAWlxMKqi8i6zdgVXAZOAttP6MSPV64w3YtAn6qi8lFVfeGPyhwM+BzsDjQC9gg7u/5u6vlffmZtbEzKaa2YdmttzMuh94ZJEMsn07dOsGvXrFnURSUJkF3t13uvtcd/83woXVT4H5FVgL/nFgrrv/CDgeWH5AaUUyzfnnw8KF0KBB3EkkBZU3RIOZHQRcCFwBtAOeAKbvx+saE5Y1GATg7tuB7ZWPKpJhvvkGateGevXiTiIpqrylCp4GFhDmwP+3u5/s7g+7+5r9eO/2QB7wRzN718z+YGZ7dUPMbKiZLTazxXl5eZX5M4ikp1GjoEWLUOhFKsHc972se+LTqt8mHhY/0AB394PLeG02sBA41d3fMrPHgW/c/T/39Zrs7GxfvHhxRfKLpK9u3cAsDNGI7IOZ5bh7dmlt5Y3B13D3RonbwcVujcoq7gmrgdXu/lbi8VTCbwIiUp5168InVzV7Rg5AebNoKs3dvyAsddAh8dTZwAdRnU8krbz0UviqAi8HoNyLrAfoVmCSmdUBPiOsSCki5XnhBWjfHjp1ijuJpLBIC7y7LwFKHRsSkTI8/DCsXx/G4EUqKeoevIhURhddrpIDF9kYvIhU0lNPwSuvxJ1C0oAKvEgyKSiAO++Ep5+OO4mkARV4kWTy+uthcbGLL447iaQBFXiRZDJjRlia4Nxz404iaUAFXiRZuIfpkb16Qf36caeRNKACL5Is1q6FHTs0PCNVRtMkRZJFq1awejXs3Bl3EkkTKvAiycIdatQIN5EqoH9JIsng44+hTRt4rdyN0kT2mwq8SDKYNg3WrIEjjog7iaQRFXiRZDBtGnTtGnrxIlVEBV4kbp9/Djk5cOmlcSeRNKMCLxK3558PX/v1izeHpB0VeJG4nXIKPPAAHHlk3EkkzWiapEjcuncPN5Eqph68SJwWLYIlS8IceJEqph68SJzuuy98evXDD+NOImlIPXiRuHzxBcyfD/37a2s+iYQKvEhcpk6FwsJQ4EUiEOkQjZmtBDYDO4Ed7q4NuEWKTJkCnTtDp05xJ5E0VR1j8Ge5+4ZqOI9I6ti0Cd59N4zBi0REF1lF4tCkCaxfH9Z/F4lI1GPwDrxsZjlmNjTic4mklgYNoHHjuFNIGou6wJ/m7l2AC4CbzaznngeY2VAzW2xmi/Py8iKOI5IEVqyAE08Mc+BFIhRpgXf3NYmvXwLTga6lHPOku2e7e3ZWVlaUcUSSw5Qp4cNNhx4adxJJc5EVeDNrYGaNiu4D5wK5UZ1PJCW4wzPPQI8e0LZt3GkkzUV5kfUHwHQLH+CoBTzj7nMjPJ9I8luyBJYtg9Gj404iGSCyAu/unwHHR/X+IilpwgSoUwcuvzzuJJIBNE1SpDr17AnNm0PTpnEnkQygAi9Snfr108YeUm20Fo1IdXnhBVi3Lu4UkkFU4EWqw1dfhXH3Rx6JO4lkEBV4keowZQps3w7/9m9xJ5EMogIvUh3Gj4djj4UTTog7iWQQFXiRqC1ZEpYlGDxYG3tItVKBF4nam29C/fpwzTVxJ5EMowIvErVbbgn7rmruu1QzFXiRKG3bFr4ecki8OSQjqcCLRMU9LCp2xx1xJ5EMpQIvEpVFi+Cdd+Doo+NOIhlKBV4kKmPGQMOGcOWVcSeRDKUCLxKFvDx49tlQ3Bs1ijuNZCgVeJEo/O538P33cNttcSeRDKbVJEWicNNNcOSR0LFj3Ekkg6kHLxKFrCyNvUvsVOBFqtLOnXDFFTB/ftxJRFTgRarUiy+Gi6sbNsSdREQFXqTKuMPIkdC+PVxySdxpRHSRVaTK/O1vsGAB/OY3ULNm3GlEou/Bm1lNM3vXzGZGfS6R2LjDAw9AmzYwZEjcaUSA6unB3wYsBw6uhnOJxKOwEAYNCouKHXRQ3GlEgIgLvJm1Bi4EhgNacUnSV82acOONcacQKSHqIZrHgHuAwn0dYGZDzWyxmS3Oy8uLOI5IBGbPhtGjYceOuJOIlBBZgTezPsCX7p5T1nHu/qS7Z7t7dlZWVlRxRKKxcyfcdReMGqXt+CTpRDlEcyrQ18x6A3WBg81sortfFeE5RarX738Py5fDtGmaOSNJJ7IevLvf5+6t3b0dMAB4RcVd0kpeHtx/P5x9tua9S1LSB51EKuvnP4ctW+CJJzQ8I0mpWj7o5O7zgfnVcS6RatO/P3TqBMccE3cSkVLpk6wilXXOOeEmkqQ0RCNSUY89BnffrWmRkvRU4EUqIjcXfvYz+PRTzZqRpKcCL7K/tm2Dq66CJk3gySd1YVWSnsbgRfbXgw/Ce+/BSy+FHZtEkpx68CL7Y8MG+O1vYehQ6NMn7jQi+0U9eJH90bw5vPlm2EhbJEWoBy9Slk2bYMqUcP+EE6BhwzjTiFSICrzIvuzYAQMGwNVXw2efxZ1GpMI0RCNSmp074dpr4S9/gbFj4Ygj4k4kUmHqwYvsqai4T5wIw4drCz5JWSrwInuaPx/+9Cd4+OGwoJhIitIQjUiRwkKoUSMs//v225CdHXcikQOiHrwIwMqV0K0bvPFGeKziLmlABV5k3jzo2hU++SQsRyCSJlTgJXN98w3ccAOcey40awZvvaXlfyWtqMBL5po0Cf7wh7Bp9jvvQIcOcScSqVK6yCqZ5bXX4Kuvwh6qQ4dC9+7hE6oiaUg9eEl/27fDn/8MZ5wBZ54Z5ra7h/XcVdwljanAS3r785/h8MPD/qmrVsH//m+YKaO13CUDaIhG0sfatfD662EYZtgw6NgRGjSAk06Cm26C887TLkySUSIr8GZWF3gdOChxnqnu/lBU55MMsnkzFBRA06ahqN98MyxZEuayAzRqFIZjOnaECy8MN5EMFGUPfhvwY3ffYma1gTfNbI67L4zwnJLM3MMKjdu3h8cNGoSvH30EW7bAd9+F4v3NN9CmDZx6avh06XXXwRdfhGK+Zg1s3Aj33AOPPAKNG4fXd+sGt94aCvvxx0Mt/XIqEtn/And3YEviYe3EzSM52fvvh2Vd9zRyJPTuHeY3Dx68d/vvfhcKwiuvhOKwpwkTwicaZ84MGy3vaerU0EucMgV+8Yu922fPDuO/Tz0Vxn739NprYSOJUaNgzJi9299+G+rXhxEj4OmnS7aZwbJl4f6DD4YsxTVsCIsWhft33glz5oQCW6RFi3B+CHPBi+4XHdO+PcydG+4PHBjey313+3HHwYwZ4f7554fvQWFhaN+5E047DaZPD+2dOsHHH4fiXuSii3a//vTTIS+vZP6BA0OBr1EDFiyAgw+Gdu2gR4/wtWfPcFyDBvDBB3v/3YlItGPwZlYTyAGOBH7r7m+VcsxQYChA27ZtK3eievXgmGP2fr5x4/C1fv3S24s2b2jUqPT2+vV3v09p7XXrhq+HHFJ6e5064WuzZqW3F40Ht2hRenuNxDXwli2hc+eSbcUvErZqtXd7vXq777dps7u96HWHHLK7vX17yM8v+d6HHbb78Y9+FJ4rukHJ5XO7dw/nKGqvWbPknPJBg8LGGbVr774Vbx83bnfmgw8O34/mzXe3f/ghIlJx5h5Np7rEScyaANOBW909d1/HZWdn++LFiyPPIyKSLswsx91LXTypWqZJuvsm4FXg/Oo4n4iIRFjgzSwr0XPHzOoBvQD9ri0iUk2iHINvCUxIjMPXAP7s7jMjPJ+IiBQT5SyapcCJUb2/iIiUTUsViIikKRV4EZE0pQIvIpKmVOBFRNJUtXzQaX+ZWR7weSVf3hzYUIVxqopyVYxyVYxyVUw65jrc3bNKa0iqAn8gzGzxvj7NFSflqhjlqhjlqphMy6UhGhGRNKUCLyKSptKpwD8Zd4B9UK6KUa6KUa6KyahcaTMGLyIiJaVTD15ERIpRgRcRSVNpWeDN7E4zczNrXv7R0TOzh81sqZktMbOXzeyw8l8VPTMbaWYfJrJNL1reOW5m9lMzW2ZmhWYW65Q2MzvfzD4ys0/N7N44sxRnZk+Z2Zdmts8NdOJgZm3M7FUz+yDxPbwt7kwAZlbXzBaZ2XuJXP8dd6YiZlbTzN41sypfbTftCryZtQHOBf4Vd5ZiRrr7ce5+AjATeDDmPEXmAZ3d/TjgY+C+mPMUyQX6Aa/HGSKx1PVvgQuAY4ArzKyUvRVjMZ7k3EBnB3Cnux8DnALcnCR/Z9uAH7v78cAJwPlmdkq8kXa5DVgexRunXYEH/g+4h6g2+K4Ed/+m2MMGJEk2d3/Z3Yt2wl4ItI4zTxF3X+7uH8WdA+gKfOrun7n7duBZ4KKYMwHg7q8DX8WdY0/uvs7d30nc30woXK3iTQUebEk8rJ24xf7/0MxaAxcCf4ji/dOqwJvZRcAad38v7ix7MrPhZrYKuJLk6cEXNxiYE3eIJNMKWFXs8WqSoFilCjNrR9gT4q2YowC7hkKWAF8C89w9GXI9RuiQFkbx5lHu6BQJM/srcGgpTfcDPycMz1S7snK5+wvufj9wv5ndB9wCPJQMuRLH3E/41XpSdWTa31ySusysITANuH2P32Bj4+47gRMS15qmm1lnd4/tGoaZ9QG+dPccMzszinOkXIF393NKe97MjgXaA++ZGYThhnfMrKu7fxFXrlJMAmZTTQW+vFxmNgjoA5zt1fihiAr8fcVpDdCm2OPWieekDGZWm1DcJ7n783Hn2ZO7bzKzVwnXMOK8SH0q0NfMegN1gYPNbKK7X1VVJ0ibIRp3f9/dW7h7O3dvR/h1ukt1FPfymNlRxR5eRJJsPm5m5xN+Pezr7t/FnScJvQ0cZWbtzawOMAB4MeZMSc1C72ocsNzd/1/ceYqYWVbRLDEzqwf0Iub/h+5+n7u3TtSrAcArVVncIY0KfJIbYWa5ZraUMISUFFPHgN8AjYB5iSmcY+IOBGBml5jZaqA7MMvM/hJHjsQF6FuAvxAuFv7Z3ZfFkWVPZjYZWAB0MLPVZnZd3JkSTgWuBn6c+De1JNFDjVtL4NXE/8G3CWPwVT4tMdloqQIRkTSlHryISJpSgRcRSVMq8CIiaUoFXkQkTanAi4ikKRV4SUpmtqX8o/b52lsSqz+WWFHUgicSbUvNrEuxtpZFq/mZ2ZlVtbKfmc3fnxUxzWxleaufmtlfzeyQqsglmUEFXtLR34FzgM/3eP4C4KjEbSgwuljbHcDYaklXeX8Cboo7hKQOFXhJaole98jEB8XeN7P+iedrmNnvEuvZzzOz2WZ2GYC7v+vuK0t5u4uApxMrCy4EmphZy0TbpcDcUs7f1cwWJNbr/oeZdUg8P8jMZiTOvTLxW8MdieMWmlnTYm9zdeIDP7lm1jXx+mYW9gZYZmZ/AKzYOWeYWU6ibWix93kRuKKyf5eSeVTgJdn1I6zffTyhVz4yUZT7Ae0I67RfTfjUa3lKXR3SzNoDX7v7tlJe8yFwurufSFgF9H+KtXVO5DgZGA58lzhuAXBNsePqJ/YCuAl4KvHcQ8Cb7t4JmA60LXb8YHc/CcgGhplZMwB3/xo4qOixSHlSbrExyTinAZMTKwGuN7PXCAX1NOA5dy8EvkgsHlVZLYG8fbQ1BiYk1hNywjriRV5NrHm+2czygZcSz78PHFfsuMkQ1nA3s4MTa6L0JPxwwN1nmdnXxY4fZmaXJO63IQwpbUw8/hI4rNhjkX1SD14yyb5Wh9xKWM2vNA8TCnln4Cd7HFe8x19Y7HEhJTtPe64Hss/1QRLLxp4DdE/sPvTuHuesm8grUi4VeEl2bwD9E5s1ZBF6vosIF1IvTYzF/wA4cz/e60XgmsS4/ilAvruvI2xX2G4fr2nM7iWCB1Xyz1B03eC0xDnzCdsRDkw8fwFQNDumMWG46Dsz+xFh2zsSxxlhDf2VlcwhGUYFXpLddGAp8B7wCnBPYgnoaYQx9A+AicA7QD6AmQ1LrEbZGliauIgJYR3+z4BPCTNmbgJw92+BFWZ2ZCnn/zXwKzN7l8oPaX6feP0YoGjVx/8GeprZMsJQTdEewnOBWma2HBhB2EqxyEnAwmLbLIqUSatJSsoys4buviVx0XERcGpl1/9PjHmf5O4PVGnIKmRmjwMvuvvf4s4iqUEXWSWVzUxcsKwDPHwgm7u4+/QUmJ2Sq+IuFaEevIhImtIYvIhImlKBFxFJUyrwIiJpSgVeRCRNqcCLiKSp/w8+IG8WofzrJgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "def R2(y_data, y_model):\n", + " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "\n", + "X = np.array( [ [ 2, 0], [0, 1], [0,0]])\n", + "y = np.array( [4, 2, 3])\n", + "\n", + "\n", + "# matrix inversion to find beta\n", + "OLSbeta = np.linalg.inv(X.T @ X) @ X.T @ y\n", + "print(OLSbeta)\n", + "# and then make the prediction\n", + "ytildeOLS = X @ OLSbeta\n", + "print(\"Training MSE for OLS\")\n", + "print(MSE(y,ytildeOLS))\n", + "ypredictOLS = X @ OLSbeta\n", + "\n", + "# Repeat now for Ridge regression and various values of the regularization parameter\n", + "I = np.eye(2,2)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 100\n", + "MSEPredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 4, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y\n", + "# print(Ridgebeta)\n", + " # and then make the prediction\n", + " ypredictRidge = X @ Ridgebeta\n", + " MSEPredict[i] = MSE(y,ypredictRidge)\n", + "# print(MSEPredict[i])\n", + " # Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Test')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see here that we reach a plateau. What is actually happening?\n", + "\n", + "\n", + "## With Lasso Regression" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2. 2.]\n", + "Training MSE for OLS\n", + "3.0\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEHCAYAAACp9y31AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqrElEQVR4nO3dd5xU1fnH8c/DUhYBQQGJCAgaRQwCwmogRLCBBSIQVLBFQMSoKLFGJUaNqESMJSY/CRpR1ODPny0EsKCBSKIoHWkWEBGDSO9ly/P748wWdgaWsrt3Z+b7fr3ua2fuc2fus4WHM+eee465OyIikj4qRZ2AiIiULxV+EZE0o8IvIpJmVPhFRNKMCr+ISJpR4RcRSTOVy+qNzexZoDvwvbu3jO07HPhfoCmwDLjY3deX9F716tXzpk2bllWqIiIpaebMmWvcvX7x/VZW4/jNrBOwBRhTpPA/DKxz9+FmdgdwmLv/uqT3ysrK8hkzZpRJniIiqcrMZrp7VvH9ZdbV4+4fAOuK7e4BPB97/DzQs6zOLyIiiZV3H38Dd18Ze/wd0GBPB5rZIDObYWYzVq9eXT7ZiYikgcgu7nroY9pjP5O7j3L3LHfPql8/rotKREQOUJld3N2DVWZ2pLuvNLMjge8P9I2ys7NZsWIFO3bsKMX0JAqZmZk0atSIKlWqRJ2KSFoo78I/DrgSGB77+vcDfaMVK1ZQq1YtmjZtipmVVn5SztydtWvXsmLFCpo1axZ1OiJpocy6esxsLPAR0NzMVpjZVYSC38XMvgDOjj0/IDt27KBu3boq+knOzKhbt64+uYmUozJr8bv7JXsInVVa51DRTw36PYqUr/Lu6hERSS55ebBzJ+zYAdu3h69mkN81OXcurFsH2dmFW+3acMYZIf73v8OaNZCbW7gddRT06hXif/kLrF8fzpO/nXACXHxx2X1P7l7ht3bt2nlxCxcujNtX3gC/7LLLCp5nZ2d7vXr1vFu3bu7u/t1333m3bt28VatW3qJFCz/vvPPc3f2rr77yzMxMb926dcH2/PPPx71/586d/fjjj/dWrVp5VlaWz549uyB23nnn+fr16+Nec8899/iIESMO6vtas2ZNQV4NGjTwhg0bFjzfuXNnia+fPn2633DDDft1zorw+5QUtX27+1dfuX/yifuECe7PP+/+l78Uxh9/3L1nT/czznA/5RT3Fi3cO3QojHft6g67byeeWBjv2DE+fuqphfFWreLjZ51VGG/WLD7es2epfOvADE9QU9XiPwg1atRg/vz5bN++nerVqzNp0iSOOuqogvhvf/tbunTpwpAhQwCYN29eQezYY49lzpw5JZ7jpZdeIisri9GjR3PbbbcxadIkACZOnFi630wRdevWLcjt3nvvpWbNmtx66627HZOTk0Plyon/fLKyssjKirtZUKRsbNwIixfDl1/CV1/B11/Dt9/C+PFQqRLccAM888zur6lWDQYNCo+XL4clS+DQQ6FuXTj6aGjYsPDY/v1D67169fC6zEyoV68w/vjjsGULVKlSuNWqVRifODG08jMyCrdq1Qrj8+eHTxAZGSFfs7CVIRX+g3T++eczYcIELrzwQsaOHcsll1zC1KlTAVi5ciVdu3YtOLZVq1YHfJ4OHTowYsSIgudNmzZlxowZ1KtXjwceeIDnn3+eI444gsaNG9OuXTsApk+fzlVXXUWlSpXo0qULb731FvPnzyc3N5c77riDKVOmsHPnTq6//nquueaaEnPo168fmZmZzJ49m44dO9K3b1+GDBnCjh07qF69OqNHj6Z58+ZMmTKFRx55hPHjx3PvvfeyfPlyli5dyvLly/nVr37FjTfeeMA/B0lju3aFIjlzZuheGTYM6tQJhffeewuPa9AAmjSBzZtDl8sVV0D79nDEEVC/fijahx8e2tZm8Ic/7P28ffvuPV5SI6dIYzChQw7Ze7wMpE7hP/30+H0XXwzXXQfbtsH558fH+/UL25o1cOGFu8emTNmn0/bt25ff/e53dO/enXnz5jFgwICCwn/99dfTp08f/vSnP3H22WfTv39/GsZaEkuWLKFNmzYF7/Pkk09y2mmn7fE8b7/9Nj179ozbP3PmTF5++WXmzJlDTk4Obdu2LSj8/fv35+mnn6ZDhw7ccccdBa/561//Su3atZk+fTo7d+6kY8eOdO3adZ+GU65YsYIPP/yQjIwMNm3axNSpU6lcuTLvvfced911F6+99lrcaxYvXszkyZPZvHkzzZs359prr9WYfdl3U6fCbbfB7Nmh+ENoUV99dSj8F18MbdvCD38ITZuGlnlRnTqFTQqkTuGPSKtWrVi2bBljx47l/GL/uZxzzjksXbqUt99+m7feeouTTz6Z+fPnA/ve1XPZZZexa9cutmzZkvD4qVOn0qtXLw6JtRouuOACADZs2MDmzZvp0KEDAJdeeinjx48H4N1332XevHm8+uqrAGzcuJEvvvhinwr/RRddREZGRsHrrrzySr744gvMjOzs7ISv6datG9WqVaNatWocccQRrFq1ikaNGpV4Lkkz7qFFP2FC6B655Rbo0QNq1oTKleHGG+GUU0ILu1mzwu6QFi3CJvssdQr/3lrohxyy93i9evvcwk/kggsu4NZbb2XKlCmsXbt2t9jhhx/OpZdeyqWXXkr37t354IMPClrk++Kll16iXbt23Hbbbdxwww28/vrrB5xnPnfnySef5Jxzztnv19aoUaPg8d13380ZZ5zBG2+8wbJlyzg90acuoFqR/syMjAxycnL2+7ySwnbsgIcfhpdfhkWLwr6TTw6jW/If//vf0eWXgrQQSykYMGAA99xzDyeddNJu+//5z3+ybds2ADZv3sySJUto0qTJfr+/mXH//fczbdo0Fi9evFusU6dOvPnmm2zfvp3Nmzfzj3/8A4A6depQq1YtPv74YwBefvnlgtecc845PPXUUwUt9M8//5ytW7fud14bN24suJj93HPP7ffrJY3t2gX5gx2qVYPnnw/973/+c7gwO2tW4XBHKXUq/KWgUaNGCS9Yzpw5k6ysLFq1akWHDh0YOHAgp5xyClDYx5+//fGPf9zrOapXr84tt9yy2wVegLZt29KnTx9at27NeeedV/D+EPryr776atq0acPWrVupXbs2AAMHDuTEE0+kbdu2tGzZkmuuueaAWuG33347d955JyeffLJa8bJvNm2C3/8+XHw988zCMfGffgr/+le4Jld0RI2UiTJbiKU0JVqIZdGiRbRQv95ebdmyhZo1awIwfPhwVq5cyRNPPBFxVonp95niNm2Cxx6DJ54INyt16QI33QTnnBOGMEqZ2NNCLKnTxy9xJkyYwEMPPUROTg5HH320umMkOrNnhyGXPXrA0KHhIq1ERoU/hfXp04c+ffpEnYakq08+gY8+giFDoHNn+OwzOP74qLMS1McvIqVtxw64807o0CF07ezcGfar6FcYKvwiUnrmzoV27WD48DDVwezZu09PIBWCunpEpHSsXx/ukK1ZE956C849N+qMZA9U+EXk4OTPeXPYYfDcc/CTn4T5cqTCUlfPQTAzLr/88oLnOTk51K9fn+7duwOwatUqunfvTuvWrTnxxBMLpnRYtmwZ1atX320c/5gxY+Le//TTT6f4MNby8MADDxTklZGRsc/3GuQbOHAgCxcuLOMspULYtAm6doX8OZp69VLRTwJq8R+E8piWOQpDhw5l6NChANSsWTMuz/w5vSvtYfz1M8WnwJXU9N13cN55YX6dK6+MOhvZD2rxH6T8aZmBgmmZ861cuXK3ycgOZlrmfMuWLeO0006jbdu2tG3blg8//LDgXJ06daJNmza0bNmSqVOnkpubS79+/WjZsiUnnXQSjz32GABz5syhffv2tGrVil69erF+/fp9Om/z5s35xS9+QcuWLfnmm2+49tprycrK4kc/+hH33HNPwbFFP6nUrFmToUOH0rp1a9q3b8+qVasO+mcgFcDSpdCxI3z+OfzjH1Dkk68kgUSrs1S0bZ9W4OrcOX77859DbOvWxPHRo0N89er42D6oUaOGz50713v37u3bt2/31q1b++TJkwtW4Hr77be9du3afvrpp/uwYcP822+/dffEK3B98MEHce/fuXNnnz59+m77tm7d6tu3b3d3988//9zzfzaPPPKIDxs2zN3dc3JyfNOmTT5jxgw/++yzC16bv2LXSSed5FOmTHF397vvvtuHDBmy1+8xP2cz848++qggtnbt2oLzde7c2efOnRuXN+Djxo1zd/fbbrvN77///oTn0QpcSWTNGvcmTdzr1nWfNi3qbGQv0ApcZaOsp2UuLjs7m8GDBzNnzhwyMjL4/PPPATjllFMYMGAA2dnZ9OzZkzZt2nDMMcewdOlSbrjhBrp160bXrl3ZuHEjGzZsoHPnzgBceeWVXHTRRft07qOPPpr27dsXPH/llVcYNWoUOTk5rFy5koULF8Z9qqlatWrBNY927doVrCAmSezww2HgQOjePcycKUkndQp/ik7LXNxjjz1GgwYNmDt3Lnl5eWRmZgJhls4PPviACRMm0K9fP26++WZ+8YtfMHfuXN555x1GjhzJK6+8UtDdcyCKTsn81Vdf8cgjjzB9+nQOO+ww+vXrx44dO+JeU6VKFSw2b7qmZE5yq1fD2rVhIfC77446GzkI6uMvBWU9LXNRGzdu5Mgjj6RSpUq88MIL5ObmAvD111/ToEEDrr76agYOHMisWbNYs2YNeXl59O7dm2HDhjFr1ixq167NYYcdVrBK2AsvvFDQ+t8fmzZtokaNGtSuXZtVq1bx1ltvHdT3JRXcjh1wwQVhUrX8O3ElaaVOiz9Ce5uWefDgwVSuXJm8vLyCaZmXLVsWt/TigAEDEr5Ht27dCpYp7NChAw8++CC9e/dmzJgxnHvuuQWt8ClTpjBixAiqVKlCzZo1GTNmDN9++y39+/cnL7agxUMPPQTA888/zy9/+Uu2bdvGMcccw+jRo/f7e27dujUnn3wyJ5xwAo0bN6Zjx477/R6SJNzhqqtg2rQwbFN34iY9TcssFYJ+nxXYAw/Ab34DDz4Y5uCRpLGnaZnV1SMiezZxYij6l18Od9wRdTZSSlT4RWTPOneG+++Hp58uXNxckl5S9/G7e8GIEUleydDdmHZ27YLsbKhRI7T4JaUkbYs/MzOTtWvXqmgkOXdn7dq1BcNSpYK4446wStaWLVFnImUgaVv8jRo1YsWKFaxevTrqVOQgZWZm7ja1hURs/PiwPu7gwWGKZUk5SVv4q1SpQrNmzaJOQyS1rF0bhm62bg0jRkSdjZSRpC38IlIGhgyBdetg0iRQ91vKiqSP38xuMrMFZjbfzMaamf7CRKK2dWuYbXPoUCiFmWSl4ir3Fr+ZHQXcCJzo7tvN7BWgL/BceeciIkXUqAEffhju1JWUFtWonspAdTOrDBwC/DeiPEQE4NlnYc0aqFwZYlOESOoq98Lv7t8CjwDLgZXARnd/t/hxZjbIzGaY2QyN3BEpQ9OmhQu6TzwRdSZSTsq98JvZYUAPoBnQEKhhZnHL97j7KHfPcves+vXrl3eaIukhNzcM22zYEG6/PepspJxE0dVzNvCVu69292zgdeAnEeQhIs8+CzNnwiOPQK1aUWcj5SSKwr8caG9mh1iYb+EsYFEEeYikt3XrwmybnTpB375RZyPlKIo+/o+BV4FZwKexHEaVdx4iaS87G84+G558UhOwpZmknY9fRET2TvPxi0ih4cPh00+jzkIioikbRNLNhx+Gvv2cHCi2TrSkB7X4RdKJexi2+YMfwE03RZ2NREQtfpF0Mm4c/Oc/MHJkmKJB0pJa/CLpIjc3dPEcf3y4U1fSllr8Iuli1y7o2RN+/OMwJ4+kLf32RdJF9erw4INRZyEVgLp6RNLBO++EJRWT4L4dKXtq8YukutxcuPHGsKLW+efrLl1R4RdJeWPHhpW1XnsNKulDvqirRyS15ebC/feHxdN79ow6G6kg1OIXSWWvvhpa+6++qta+FNBfgkgqy8iArl2hV6+oM5EKRIVfJJVdeGEY0aPWvhShvwaRVPXGG7BzZ9RZSAWkwi+Siv79b/j5z8PSiiLFqPCLpKKHHoJ69eDKK6PORCogFX6RVDNvHkycCEOGwCGHRJ2NVEAq/CKp5g9/CFMuX3dd1JlIBaXCL5JKcnPDuP2rroLDD486G6mgdAOXSCrJyAhLK2o0j+yFWvwiqWLrVli/PkzClpkZdTZSganwi6SKv/wFGjeG//436kykglPhF0kF2dnw+ONwyinQsGHU2UgFpz5+kVTwf/8H33wD//M/UWciSUAtfpFk5w6PPgrNm4eFVkRKoBa/SLL79FOYORNGjtRkbLJPVPhFkl2rVuFu3WOPjToTSRIq/CLJzD0M3zzppKgzkSSiz4UiyeyWW8JEbO5RZyJJRIVfJFmtXw+jRoXHZtHmIklFhV8kWT39dLhb96abos5Ekkwkhd/M6pjZq2a22MwWmVmHKPIQSVrZ2fDkk3DGGdCmTdTZSJKJ6uLuE8Db7n6hmVUFNGm4yP547TVYsQKeeirqTCQJlXvhN7PaQCegH4C77wJ2lXceIknttNPgwQd1w5YckCi6epoBq4HRZjbbzJ4xsxrFDzKzQWY2w8xmrF69uvyzFKnIjjoK7rxTN2zJAYnir6Yy0BZ4yt1PBrYCdxQ/yN1HuXuWu2fVr1+/vHMUqbgeeADefz/qLCSJRVH4VwAr3P3j2PNXCf8RiEhJli6F3/5WhV8OSrkXfnf/DvjGzJrHdp0FLCzvPESS0uOPh+6d66+POhNJYlGN6rkBeCk2omcp0D+iPESSx7p18OyzcOmloY9f5ABFUvjdfQ6QFcW5RZLWX/4Sbti65ZaoM5EkpyEBIsmifn3o1y/MxilyEFT4RZLFwIEwenTUWUgKUOEXqejcw526u3Sfo5QOFX6Rim7SJLjwQnjllagzkRShwi9S0Q0fDg0bwkUXRZ2JpAgVfpGK7OOPYfJkuPlmqFYt6mwkRajwi1Rkv/891KkDgwZFnYmkEBV+kYpqx44wRcPgwVCrVtTZSArRYusiFVVmJsyerdE8UurU4hepiNatg40bw1q66tuXUqbCL1IR3XcfHHMMbNkSdSaSgvZa+M3s8iKPOxaLDS6rpETS2sqVMGoU9OgBNWtGnY2koJJa/DcXefxksdiAUs5FRAAefjgspj50aNSZSIoqqfDbHh4nei4iB+u772DkSLj8cjj22KizkRRVUuH3PTxO9FxEDta4cWrtS5kraTjnCWY2j9C6Pzb2mNjzY8o0M5F0NGgQdOkCzZpFnYmksJIKf4tyyUJEYP16OOwwFX0pc3vt6nH3r4tuwBbCwuj1Ys9FpDQsXw6NG8MLL0SdiaSBkoZzjjezlrHHRwLzCaN5XjCzX5V9eiJp4p57ICcHOneOOhNJAyVd3G3m7vNjj/sDk9z9Z8CP0XBOkdKxYAGMGRPm5GnSJOpsJA2UVPizizw+C5gI4O6bgbyySkokrdx1V7hR6847o85E0kRJF3e/MbMbgBWEvv23AcysOlCljHMTSX1ffw1vvw333gt160adjaSJkgr/VcDvgLOBPu6+Iba/PaBVn0UO1tFHw2efQf36UWciaWSvhd/dvwd+mWD/ZGByWSUlkhZWroQjj4SmTaPORNLMXgu/mY3bW9zdLyjddETSxMaN0KYNDBwIDzwQdTaSZkrq6ukAfAOMBT5G8/OIlI5774XVq6F376gzkTRUUuH/AdAFuAS4FJgAjHX3BWWdmEjKWrAAnnwyTM/Qtm3U2UgaKunO3Vx3f9vdryRc0P0SmKK5+EUOUF4eXH89HHooDBsWdTaSpkpcc9fMqgHdCK3+psAfgTfKNi2RFPXVV7BwIYwYAfXqRZ2NpKmSLu6OAVoSbty6r8hdvCJyII49NgzfrFMn6kwkjZV05+7lwHHAEOBDM9sU2zab2aayT08kRbjD2LGQmxtm4DSNk5DolDSOX4uxi5SGZ54JF3MrVYI+faLORtJcZIXdzDLMbLaZjY8qB5FysXgx3HQTnHkmXHRR1NmIRFf4Cd1HiyI8v0jZ274dLr4YqlcPM3BW0odoiV4kf4Vm1ogwUuiZKM4vUm5uvhk+/RRefBGOOirqbESAfRjOWUYeB24Hau3pADMbBAwCaKI5yiVZ9esHxx8P55wTdSYiBcq9xW9m3YHv3X3m3o5z91HunuXuWfU1c6EkmzVrwtcf/zj074tUIFF09XQELjCzZcDLwJlm9mIEeYiUjWXLoGVLeOSRqDMRSajcC7+73+nujdy9KdAX+Ke7X17eeYiUiQ0boFs32LkzfBWpgKLq4xdJPdu2wc9/Dp9/Du++Cy1aRJ2RSEKRFn53nwJMiTIHkVLhDr16wZQpYdjmGWdEnZHIHqnFL1IazOCKK+Cyy+By9VxKxabCL3Iw1q8P4/Q7dVLBl6Sh2whFDtQXX0D79uEi7vffR52NyD5T4Rc5EP/8Zxijv24dTJwIRxwRdUYi+0yFX2R/3XsvdOkCDRvCJ5/AaadFnZHIflHhF9lfW7aEC7kffgjNmkWdjch+08VdkZJs2QK//z2cfjqcdRY8/LBm2ZSkpsIvsic5OWFWzbvugpUrw1j9s85S0Zekp79gkUTGjg2zavbvD40bw3/+A8OGRZ2VSKlQ4ReB0JqfNQs2xZaSXr8eGjSA11+Hjz6Cn/wk2vxESpEKv6SvnJzQkh86NMym2a4dvPJKiP3yl+Hiba9e6tqRlKM+fkkfGzbA5s2h62bLFmjSJLTsMzKgY0cYORJ69w7HqthLClPhl9Sxc2foqslfuGfUKJg7N9xhu2AB/Pe/cMEF8Pe/Q82aMGhQaOV36QJ16kSaukh5Sv3CP2MG5OXtvq9+/TD+2h2mT49/zQ9+EFqDOTmh37e4o44K265dMGdOfLxJk/Ae27eHeVyKa9o03Om5ZQssXBgfP/ZYqFs3FLHFi+Pjxx8fCtX69aGoFXfCCXDooWEVqKVL4+M/+hHUqBGmGVi2bPeYO7RuDZmZoVB+883uMYC2baFqVVi+HFasKNyff0yHDqEVvWRJeH1eXtifv3XpEo6dMwe+/hpyc8PPOjc3tLT79AnxCRPCz2fnzvCz3L495H3//SF+yy1hNsx168K2aRO0aQOzZ4f46NGwaBH88Idw9tnh+z711MJchw+P/9mIpAN3r/Bbu3bt/IAdckjRkhO2a64Jsdzc+Bi433priG/YkDh+330hvmJF4vijj4b4okWJ46NGhfgnnySOjx0b4u+/nzg+fnyIv/lm4vi//hXiL76YOD5rVog/9VTi+Oefh/jDDyeOr1wZ4nffnTi+ZUuI/+pX8TGzwt/NVVfFx2vVKoz36VO4v1Il95o13X/0o8L4r3/t3q2b++WXu994o/vvfuf+t78Vxrdt2/+/F5EUAszwBDXVvGhrrYLKysryGTNmHNiL33kntCSLatIkXMxzh7fein9Ns2ZhEY3sbJg0KT5+3HFh27EjzNlSXIsW4T22bIEPPoiPt2wZctiwIVxALK5NmzAdwJo1YUqA4rKywieGVatgZoKli9u3h8MPh2+/DV0dxXXsCLVrh9b2ggXx8c6dQ8t66VL47LPdY2Zhrvlq1eDLLws/UZgVHnPGGVC5cvg08u23IWYWWvNm4fwQzr92bTg2IyNsVavCMceE+JYt4Wu1auGYoucQkRKZ2Ux3z4rbn/KFX0QkTe2p8GvogohImlHhFxFJMyr8IiJpRoVfRCTNqPCLiKQZFX4RkTSjwi8ikmZU+EVE0owKv4hImlHhFxFJMyr8IiJpRoVfRCTNqPCLiKQZFX4RkTSjwi8ikmbKvfCbWWMzm2xmC81sgZkNKe8cRETSWRRr7uYAt7j7LDOrBcw0s0nunmDxWRERKW3l3uJ395XuPiv2eDOwCDiqvPMQEUlXkfbxm1lT4GTg4wSxQWY2w8xmrF69utxzExFJVZEVfjOrCbwG/MrdNxWPu/sod89y96z69euXf4IiIikqksJvZlUIRf8ld389ihxERNJVFKN6DPgrsMjdHy3v84uIpLsoWvwdgSuAM81sTmw7P4I8RETSUrkP53T3fwNW3ucVEZFAd+6KiKQZFX4RkTSjwi8ikmZU+EVE0owKv4hImlHhFxFJMyr8IiJpRoVfRCTNqPCLiKQZFX4RkTSjwi8ikmZU+EVE0owKv4hImlHhFxFJMyr8IiJpRoVfRCTNqPCLiKQZFX4RkTSjwi8ikmZU+EVE0ky5L7YuktRGjYK1a3ffd8wx0KdPePynP8HmzbvHTzgBevUKjx99FHbu3D3eqhV06xYeDx8O7rvHs7KgSxfIzoZHHonPqWNH6NQJtm6FP/4xPn766dChA6xfDyNHxse7doV27WDVKnj22fj4z34GLVvCN9/Aiy/Gx3v3huOPhyVL4JVX4uOXXAJNm8KiRfDmm/HxK6+Ehg1h7lyYODE+fvXVUK8eTJ8O770XH7/+ejj0UPj3v2Hq1Pj4TTdBZiZMngzTpsXHf/1rqFQJ3nkHZs3aPVa5Mtx2W3g8bhwsWLB7/JBDYMiQ8PjVV+GLL3aP16kD114bHv/tb/D117vHjzgCrroqPH7uOVi5cvf4OedA27bxOR8sd6/wW7t27Vyk3OXmur/0kvuddxbua9HCPZTmwu3ccwvjjRrFxy+6qDBeu3Z8fMCAwnilSvHxIUNCbNu2+Bi4/+Y3Ib5qVeL48OEh/uWXieN//nOIz56dOD5mTIh/8EHi+BtvhPjEiYnj770X4v/7v4nj06aF+DPPJI4vWBDijz2WOL58eYj/7neJ4+vXh/jttyeO5+SE+LXXxscyMwt/N5dfHh+vX78w3qNHfPyYYwrjZ54ZH2/dujB+6qnx8aee8oMBzPAENdVCrGLLysryGTNmRJ2GpJPsbBgwILRwjzsutEarV4ddu+Jb5GZQtWp4XLw1D6E1WaXKgcczMkLL0z2cv7TjlSuHY/Lywvdd2vEqVcL3mJsLOTkHHs/JCccUV7Vq+B0cbDw7O3wPxVWrFr4ebLykv51E8fyf7QEys5nunlV8v7p6RIrbtg0uvhgmTIBhw+Cuu8I/UCj8R7on+f/IyyJuVrbxSpXKNp6RsfciVlK8cuWwlVU8/z/fsoqX9LdTUrwUqfCLFNe7d+jvHTkSrrkm6mxESp0KvwiEj+h5eaFFeOut4YLbhRdGnZVImdBwTklv7mEkSOfO8Pvfh31nnaWiLylNhV/S09atMGZMGCrZqVMYatikSdRZiZQLFX5JD3l5sHhx4aiL664L48e3bw99+cuXwxVXRJujSDlRH7+kJvcwBHPcuHDjzsyZ4caqxYuhefNw003//qGLJ3/EjkiaUOGX1JKXF4YVjhsHPXuGot62bWjNn3IK1K8fjiuLuyFFkoQKvyS/JUvg3XfhpZegR49wi/2ZZ4bpB7p1C7fFi0iBSPr4zexcM/vMzL40szuiyEGSkHu4uSrfzTfDscfCD38Y+uzXrCks8rVqha4cFX2ROOXe4jezDODPQBdgBTDdzMa5+8LyzkXKmXu4ZT7/7skNG8K2bVsYZbN1a7htvWvXEB89Gj76KEwe9vXX8NVX0LgxzJ8f4itWhMnDbropvOa449RfL7IPoujqORX40t2XApjZy0APoGwKf1ZWGLlR1MUXwz33hP7gk06Kf03//uEmns2boX37+PjgwWHGvVWrQpdCcbffHkaMLF0aZjYs7t574aKL4NNPoW/f+PiIEXD++WEmwQED4uNPPRUuSr7/PtxwQ3w8f5jiP/4RZh4s7rXXoEULePlluO+++PlB3nkHjj4annkGHn64MJ7/ddq0MFvio4/CE0+E/Xl5hV+XLAmzFt51V5itMn+OlF27CudkgdBiHz1693Mfeihs3Bgev/9+mI2xQYNQ8E87LeSdL9FMkCJSoigK/1HAN0WerwB+XPwgMxsEDAJocjDjq084IX7iqyOPLHx84onxr2nQIHytVClxvF698LVy5cTxunXD12rVEscPOyx8rV49cbx27fC1Zs3Qoi2uVq3w9dBDE8cPOSR8rVMncTwzszDPVq3C46It5fz5Vho0CP+BFI2bFbbYmzaFM84I+ypVClv+Y4BTT4WBAwvnYKlaNcxn4h6O69cPfvpTqFGjcMv/3gFeeEEteJEyUO6zc5rZhcC57j4w9vwK4MfuPnhPr9HsnCIi+29Ps3NGcXH3W6BxkeeNYvtERKQcRFH4pwPHmVkzM6sK9AXGRZCHiEhaKvc+fnfPMbPBwDtABvCsuy8o4WUiIlJKIrmBy90nAgkW1xQRkbKmSdpERNKMCr+ISJpR4RcRSTMq/CIiaabcb+A6EGa2Gvj6AF9eD1hTiumUFuW1f5TX/lFe+ydV8zra3esX35kUhf9gmNmMRHeuRU157R/ltX+U1/5Jt7zU1SMikmZU+EVE0kw6FP5RUSewB8pr/yiv/aO89k9a5ZXyffwiIrK7dGjxi4hIESr8IiJpJq0Kv5ndYmZuZvWizgXAzO43s3lmNsfM3jWzhlHnBGBmI8xscSy3N8ysTtQ5AZjZRWa2wMzyzCzyoXdmdq6ZfWZmX5rZHVHnA2Bmz5rZ92Y2P+pcijKzxmY22cwWxn6HQ6LOCcDMMs3sEzObG8vrvqhzKsrMMsxstpmNL833TZvCb2aNga7A8qhzKWKEu7dy9zbAeOC3EeeTbxLQ0t1bAZ8Dd0acT775wM+BD6JOxMwygD8D5wEnApeYWYJ1NMvdc8C5USeRQA5wi7ufCLQHrq8gP6+dwJnu3hpoA5xrZgkW2o7MEGBRab9p2hR+4DHgdqDCXM12901FntagguTm7u+6e07s6TTCKmmRc/dF7v5Z1HnEnAp86e5L3X0X8DLQI+KccPcPgHVR51Gcu69091mxx5sJxeyoaLMCD7bEnlaJbRXi36GZNQK6Ac+U9nunReE3sx7At+4+N+pcijOzB8zsG+AyKk6Lv6gBwFtRJ1EBHQV8U+T5CipAIUsGZtYUOBn4OOJUgILulDnA98Akd68QeQGPExqreaX9xpEsxFIWzOw94AcJQkOBuwjdPOVub3m5+9/dfSgw1MzuBAYD91SEvGLHDCV8RH+pPHLa17wkeZlZTeA14FfFPvFGxt1zgTaxa1lvmFlLd4/0GomZdQe+d/eZZnZ6ab9/yhR+dz870X4zOwloBsw1MwjdFrPM7FR3/y6qvBJ4ibAqWbkU/pLyMrN+QHfgLC/Hmz324+cVtW+BxkWeN4rtkz0wsyqEov+Su78edT7FufsGM5tMuEYS9cXxjsAFZnY+kAkcamYvuvvlpfHmKd/V4+6fuvsR7t7U3ZsSPpK3LY+iXxIzO67I0x7A4qhyKcrMziV8xLzA3bdFnU8FNR04zsyamVlVoC8wLuKcKiwLra6/Aovc/dGo88lnZvXzR62ZWXWgCxXg36G73+nujWI1qy/wz9Iq+pAGhb+CG25m881sHqErqkIMcQP+BNQCJsWGmo6MOiEAM+tlZiuADsAEM3snqlxiF78HA+8QLlS+4u4Losonn5mNBT4CmpvZCjO7KuqcYjoCVwBnxv6m5sRas1E7Epgc+zc4ndDHX6pDJysiTdkgIpJm1OIXEUkzKvwiImlGhV9EJM2o8IuIpBkVfhGRNKPCL0nHzLaUfNQeXzs4NpvmbrO0WvDHWGyembUtEjsyf3ZEMzu9tGZKNLMp+zLLqJktK2lGWTN7z8wOK428JPWp8Eu6+Q9wNvB1sf3nAcfFtkHAU0ViNwNPl0t2B+4F4Lqok5DkoMIvSSvWSh8RuwnuUzPrE9tfycz+J7amwCQzm2hmFwK4+2x3X5bg7XoAY2KzNU4D6pjZkbFYb+DtBOc/1cw+is2X/qGZNY/t72dmb8bOvSz2KePm2HHTzOzwIm9zRexmpvlmdmrs9XUtrM+wwMyeAazIOd80s5mx2KAi7zMOuORAf5aSXlT4JZn9nDCHemtCK35ErFj/HGhKmCf/CsKdviVJONummTUD1rv7zgSvWQyc5u4nE2ZWfbBIrGUsj1OAB4BtseM+An5R5LhDYusxXAc8G9t3D/Bvd/8R8AbQpMjxA9y9HZAF3GhmdQHcfT1QLf+5yN6kzCRtkpZ+CoyNza64ysz+RSi0PwX+z93zgO9iE28dqCOB1XuI1Qaej8255IS53PNNjs07v9nMNgL/iO3/FGhV5LixEObRN7NDY/PGdCL8p4G7TzCz9UWOv9HMesUeNyZ0Ta2NPf8eaFjkuUhCavGLBHuabXM7YXbERO4nFPiWwM+KHVf0E0Jeked57N7gKj5nyh7nUIlNz3s20CG2YtTsYufMjOUrslcq/JLMpgJ9Ygtp1Ce0lD8hXMDtHevrbwCcvg/vNQ74Rey6QXtgo7uvJCw92XQPr6lN4VTM/Q7we8i/LvHT2Dk3EpaWvDS2/zwgf7RObUK30zYzO4GwhCGx44ywjsGyA8xD0ogKvySzN4B5wFzgn8Dtsem2XyP00S8EXgRmARsBzOzG2AyfjYB5sYunENZCWAp8SRjBcx2Au28FlpjZDxOc/2HgITObzYF3m+6IvX4kkD+T5n1AJzNbQOjyyV8n+m2gspktAoYTlsXM1w6YVmTJTJE90uyckpLMrKa7b4ld7PwE6HigazDE+tTbuftvSjXJUmRmTwDj3P39qHORik8XdyVVjY9dKK0K3H8wC++4+xtJMFpmvoq+7Cu1+EVE0oz6+EVE0owKv4hImlHhFxFJMyr8IiJpRoVfRCTN/D+ul/5ixPAjYgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn import linear_model\n", + "\n", + "def R2(y_data, y_model):\n", + " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "\n", + "X = np.array( [ [ 2, 0], [0, 1], [0,0]])\n", + "y = np.array( [4, 2, 3])\n", + "\n", + "\n", + "# matrix inversion to find beta\n", + "OLSbeta = np.linalg.inv(X.T @ X) @ X.T @ y\n", + "print(OLSbeta)\n", + "# and then make the prediction\n", + "ytildeOLS = X @ OLSbeta\n", + "print(\"Training MSE for OLS\")\n", + "print(MSE(y,ytildeOLS))\n", + "ypredictOLS = X @ OLSbeta\n", + "\n", + "# Repeat now for Ridge regression and various values of the regularization parameter\n", + "I = np.eye(2,2)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 100\n", + "MSERidgePredict = np.zeros(nlambdas)\n", + "MSELassoPredict = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 4, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " Ridgebeta = np.linalg.inv(X.T @ X+lmb*I) @ X.T @ y\n", + " print(Ridgebeta)\n", + " # and then make the prediction\n", + " ypredictRidge = X @ Ridgebeta\n", + " MSERidgePredict[i] = MSE(y,ypredictRidge)\n", + "# print(MSEPredict[i])\n", + " RegLasso = linear_model.Lasso(lmb)\n", + " RegLasso.fit(X,y)\n", + " ypredictLasso = RegLasso.predict(X)\n", + " print(RegLass)\n", + " MSELassoPredict[i] = MSE(y,ypredictLasso)\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSERidgePredict, 'r--', label = 'MSE Ridge Train')\n", + "plt.plot(np.log10(lambdas), MSELassoPredict, 'r--', label = 'MSE Lasso Train')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Another Example, now with a polynomial fit" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn import linear_model\n", + "\n", + "def R2(y_data, y_model):\n", + " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n", + "def MSE(y_data,y_model):\n", + " n = np.size(y_model)\n", + " return np.sum((y_data-y_model)**2)/n\n", + "\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", + "\n", + "x = np.random.rand(100)\n", + "y = 2.0+5*x*x+0.1*np.random.randn(100)\n", + "\n", + "# number of features p (here degree of polynomial\n", + "p = 3\n", + "# The design matrix now as function of a given polynomial\n", + "X = np.zeros((len(x),p))\n", + "X[:,0] = 1.0\n", + "X[:,1] = x\n", + "X[:,2] = x*x\n", + "# We split the data in test and training data\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n", + "\n", + "# matrix inversion to find beta\n", + "OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train\n", + "print(OLSbeta)\n", + "# and then make the prediction\n", + "ytildeOLS = X_train @ OLSbeta\n", + "print(\"Training R2 for OLS\")\n", + "print(R2(y_train,ytildeOLS))\n", + "print(\"Training MSE for OLS\")\n", + "print(MSE(y_train,ytildeOLS))\n", + "ypredictOLS = X_test @ OLSbeta\n", + "print(\"Test R2 for OLS\")\n", + "print(R2(y_test,ypredictOLS))\n", + "print(\"Test MSE OLS\")\n", + "print(MSE(y_test,ypredictOLS))\n", + "\n", + "# Repeat now for Lasso and Ridge regression and various values of the regularization parameter\n", + "I = np.eye(p,p)\n", + "# Decide which values of lambda to use\n", + "nlambdas = 100\n", + "MSEPredict = np.zeros(nlambdas)\n", + "MSETrain = np.zeros(nlambdas)\n", + "MSELassoPredict = np.zeros(nlambdas)\n", + "MSELassoTrain = np.zeros(nlambdas)\n", + "lambdas = np.logspace(-4, 4, nlambdas)\n", + "for i in range(nlambdas):\n", + " lmb = lambdas[i]\n", + " Ridgebeta = np.linalg.inv(X_train.T @ X_train+lmb*I) @ X_train.T @ y_train\n", + " # include lasso using Scikit-Learn\n", + " RegLasso = linear_model.Lasso(lmb)\n", + " RegLasso.fit(X_train,y_train)\n", + " # and then make the prediction\n", + " ytildeRidge = X_train @ Ridgebeta\n", + " ypredictRidge = X_test @ Ridgebeta\n", + " ytildeLasso = RegLasso.predict(X_train)\n", + " ypredictLasso = RegLasso.predict(X_test)\n", + " MSEPredict[i] = MSE(y_test,ypredictRidge)\n", + " MSETrain[i] = MSE(y_train,ytildeRidge)\n", + " MSELassoPredict[i] = MSE(y_test,ypredictLasso)\n", + " MSELassoTrain[i] = MSE(y_train,ytildeLasso)\n", + "\n", + "# Now plot the results\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), MSETrain, label = 'MSE Ridge train')\n", + "plt.plot(np.log10(lambdas), MSEPredict, 'r--', label = 'MSE Ridge Test')\n", + "plt.plot(np.log10(lambdas), MSELassoTrain, label = 'MSE Lasso train')\n", + "plt.plot(np.log10(lambdas), MSELassoPredict, 'r--', label = 'MSE Lasso Test')\n", + "\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Linking the regression analysis with a statistical interpretation\n", + "\n", + "\n", + "The\n", + "advantage of doing linear regression is that we actually end up with\n", + "analytical expressions for several statistical quantities. \n", + "Standard least squares and Ridge regression allow us to\n", + "derive quantities like the variance and other expectation values in a\n", + "rather straightforward way.\n", + "\n", + "\n", + "It is assumed that $\\varepsilon_i\n", + "\\sim \\mathcal{N}(0, \\sigma^2)$ and the $\\varepsilon_{i}$ are\n", + "independent, i.e.:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*} \n", + "\\mbox{Cov}(\\varepsilon_{i_1},\n", + "\\varepsilon_{i_2}) & = \\left\\{ \\begin{array}{lcc} \\sigma^2 & \\mbox{if}\n", + "& i_1 = i_2, \\\\ 0 & \\mbox{if} & i_1 \\not= i_2. \\end{array} \\right.\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The randomness of $\\varepsilon_i$ implies that\n", + "$\\mathbf{y}_i$ is also a random variable. In particular,\n", + "$\\mathbf{y}_i$ is normally distributed, because $\\varepsilon_i \\sim\n", + "\\mathcal{N}(0, \\sigma^2)$ and $\\mathbf{X}_{i,\\ast} \\, \\boldsymbol{\\beta}$ is a\n", + "non-random scalar. To specify the parameters of the distribution of\n", + "$\\mathbf{y}_i$ we need to calculate its first two moments. \n", + "\n", + "Recall that $\\boldsymbol{X}$ is a matrix of dimensionality $n\\times p$. The\n", + "notation above $\\mathbf{X}_{i,\\ast}$ means that we are looking at the\n", + "row number $i$ and perform a sum over all values $p$.\n", + "\n", + "\n", + "## Assumptions made\n", + "\n", + "The assumption we have made here can be summarized as (and this is going to be useful when we discuss the bias-variance trade off)\n", + "that there exists a function $f(\\boldsymbol{x})$ and a normal distributed error $\\boldsymbol{\\varepsilon}\\sim \\mathcal{N}(0, \\sigma^2)$\n", + "which describe our data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y} = f(\\boldsymbol{x})+\\boldsymbol{\\varepsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We approximate this function with our model from the solution of the linear regression equations, that is our\n", + "function $f$ is approximated by $\\boldsymbol{\\tilde{y}}$ where we want to minimize $(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2$, our MSE, with" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{\\tilde{y}} = \\boldsymbol{X}\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Expectation value and variance\n", + "\n", + "We can calculate the expectation value of $\\boldsymbol{y}$ for a given element $i$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*} \n", + "\\mathbb{E}(y_i) & =\n", + "\\mathbb{E}(\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}) + \\mathbb{E}(\\varepsilon_i)\n", + "\\, \\, \\, = \\, \\, \\, \\mathbf{X}_{i, \\ast} \\, \\beta, \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "while\n", + "its variance is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*} \\mbox{Var}(y_i) & = \\mathbb{E} \\{ [y_i\n", + "- \\mathbb{E}(y_i)]^2 \\} \\, \\, \\, = \\, \\, \\, \\mathbb{E} ( y_i^2 ) -\n", + "[\\mathbb{E}(y_i)]^2 \\\\ & = \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\,\n", + "\\beta + \\varepsilon_i )^2] - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \\\\ &\n", + "= \\mathbb{E} [ ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2 \\varepsilon_i\n", + "\\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} + \\varepsilon_i^2 ] - ( \\mathbf{X}_{i,\n", + "\\ast} \\, \\beta)^2 \\\\ & = ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 + 2\n", + "\\mathbb{E}(\\varepsilon_i) \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta} +\n", + "\\mathbb{E}(\\varepsilon_i^2 ) - ( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta})^2 \n", + "\\\\ & = \\mathbb{E}(\\varepsilon_i^2 ) \\, \\, \\, = \\, \\, \\,\n", + "\\mbox{Var}(\\varepsilon_i) \\, \\, \\, = \\, \\, \\, \\sigma^2. \n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hence, $y_i \\sim \\mathcal{N}( \\mathbf{X}_{i, \\ast} \\, \\boldsymbol{\\beta}, \\sigma^2)$, that is $\\boldsymbol{y}$ follows a normal distribution with \n", + "mean value $\\boldsymbol{X}\\boldsymbol{\\beta}$ and variance $\\sigma^2$ (not be confused with the singular values of the SVD). \n", + "\n", + "## Expectation value and variance for $\\boldsymbol{\\beta}$\n", + "\n", + "With the OLS expressions for the parameters $\\boldsymbol{\\beta}$ we can evaluate the expectation value" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}(\\boldsymbol{\\beta}) = \\mathbb{E}[ (\\mathbf{X}^{\\top} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1}\\mathbf{X}^{T} \\mathbb{E}[ \\mathbf{Y}]=(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\mathbf{X}^{T}\\mathbf{X}\\boldsymbol{\\beta}=\\boldsymbol{\\beta}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This means that the estimator of the regression parameters is unbiased.\n", + "\n", + "We can also calculate the variance\n", + "\n", + "The variance of $\\boldsymbol{\\beta}$ is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{eqnarray*}\n", + "\\mbox{Var}(\\boldsymbol{\\beta}) & = & \\mathbb{E} \\{ [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})] [\\boldsymbol{\\beta} - \\mathbb{E}(\\boldsymbol{\\beta})]^{T} \\}\n", + "\\\\\n", + "& = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} - \\boldsymbol{\\beta}]^{T} \\}\n", + "\\\\\n", + "% & = & \\mathbb{E} \\{ [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}] \\, [(\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y}]^{T} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & \\mathbb{E} \\{ (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\mathbf{Y} \\, \\mathbf{Y}^{T} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\mathbb{E} \\{ \\mathbf{Y} \\, \\mathbf{Y}^{T} \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\\\\n", + "& = & (\\mathbf{X}^{T} \\mathbf{X})^{-1} \\, \\mathbf{X}^{T} \\, \\{ \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} + \\sigma^2 \\} \\, \\mathbf{X} \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "% \\\\\n", + "% & = & (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^T \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T % \\mathbf{X})^{-1}\n", + "% \\\\\n", + "% & & + \\, \\, \\sigma^2 \\, (\\mathbf{X}^T \\mathbf{X})^{-1} \\, \\mathbf{X}^T \\, \\mathbf{X} \\, (\\mathbf{X}^T \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\boldsymbol{\\beta}^T\n", + "\\\\\n", + "& = & \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} + \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1} - \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T}\n", + "\\, \\, \\, = \\, \\, \\, \\sigma^2 \\, (\\mathbf{X}^{T} \\mathbf{X})^{-1},\n", + "\\end{eqnarray*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we have used that $\\mathbb{E} (\\mathbf{Y} \\mathbf{Y}^{T}) =\n", + "\\mathbf{X} \\, \\boldsymbol{\\beta} \\, \\boldsymbol{\\beta}^{T} \\, \\mathbf{X}^{T} +\n", + "\\sigma^2 \\, \\mathbf{I}_{nn}$. From $\\mbox{Var}(\\boldsymbol{\\beta}) = \\sigma^2\n", + "\\, (\\mathbf{X}^{T} \\mathbf{X})^{-1}$, one obtains an estimate of the\n", + "variance of the estimate of the $j$-th regression coefficient:\n", + "$\\boldsymbol{\\sigma}^2 (\\boldsymbol{\\beta}_j ) = \\boldsymbol{\\sigma}^2 \\sqrt{\n", + "[(\\mathbf{X}^{T} \\mathbf{X})^{-1}]_{jj} }$. This may be used to\n", + "construct a confidence interval for the estimates.\n", + "\n", + "\n", + "In a similar way, we can obtain analytical expressions for say the\n", + "expectation values of the parameters $\\boldsymbol{\\beta}$ and their variance\n", + "when we employ Ridge regression, allowing us again to define a confidence interval. \n", + "\n", + "It is rather straightforward to show that" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big]=(\\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I}_{pp})^{-1} (\\mathbf{X}^{\\top} \\mathbf{X})\\boldsymbol{\\beta}^{\\mathrm{OLS}}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We see clearly that \n", + "$\\mathbb{E} \\big[ \\boldsymbol{\\beta}^{\\mathrm{Ridge}} \\big] \\not= \\boldsymbol{\\beta}^{\\mathrm{OLS}}$ for any $\\lambda > 0$. We say then that the ridge estimator is biased.\n", + "\n", + "We can also compute the variance as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{Ridge}}]=\\sigma^2[ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1} \\mathbf{X}^{T} \\mathbf{X} \\{ [ \\mathbf{X}^{\\top} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and it is easy to see that if the parameter $\\lambda$ goes to infinity then the variance of Ridge parameters $\\boldsymbol{\\beta}$ goes to zero. \n", + "\n", + "With this, we can compute the difference" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mbox{Var}[\\boldsymbol{\\beta}^{\\mathrm{OLS}}]-\\mbox{Var}(\\boldsymbol{\\beta}^{\\mathrm{Ridge}})=\\sigma^2 [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}[ 2\\lambda\\mathbf{I} + \\lambda^2 (\\mathbf{X}^{T} \\mathbf{X})^{-1} ] \\{ [ \\mathbf{X}^{T} \\mathbf{X} + \\lambda \\mathbf{I} ]^{-1}\\}^{T}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The difference is non-negative definite since each component of the\n", + "matrix product is non-negative definite. \n", + "This means the variance we obtain with the standard OLS will always for $\\lambda > 0$ be larger than the variance of $\\boldsymbol{\\beta}$ obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below. \n", + "\n", + "\n", + "## Deriving OLS from a probability distribution\n", + "\n", + "Our basic assumption when we derived the OLS equations was to assume\n", + "that our output is determined by a given continuous function\n", + "$f(\\boldsymbol{x})$ and a random noise $\\boldsymbol{\\epsilon}$ given by the normal\n", + "distribution with zero mean value and an undetermined variance\n", + "$\\sigma^2$.\n", + "\n", + "We found above that the outputs $\\boldsymbol{y}$ have a mean value given by\n", + "$\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$ and variance $\\sigma^2$. Since the entries to\n", + "the design matrix are not stochastic variables, we can assume that the\n", + "probability distribution of our targets is also a normal distribution\n", + "but now with mean value $\\boldsymbol{X}\\hat{\\boldsymbol{\\beta}}$. This means that a\n", + "single output $y_i$ is given by the Gaussian distribution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "y_i\\sim \\mathcal{N}(\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta}, \\sigma^2)=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Independent and Identically Distrubuted (iid)\n", + "\n", + "We assume now that the various $y_i$ values are stochastically distributed according to the above Gaussian distribution. \n", + "We define this distribution as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(y_i\\vert \\boldsymbol{X};\\boldsymbol{\\beta})=\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which reads as finding the likelihood of an event $y_i$ given the input variables $\\boldsymbol{X}$ and the parameters (to be determined) $\\boldsymbol{\\beta}$.\n", + "\n", + "Since these events are assumed to be independent and identicall distributed we can build the probability distribution function (PDF) for all possible event $\\boldsymbol{y}$ as the product of the single events, that is we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(\\boldsymbol{y}\\vert \\boldsymbol{X};\\boldsymbol{\\beta})=\\prod_{i=0}^{n-1}\\frac{1}{\\sqrt{2\\pi\\sigma^2}}\\exp{\\left[-\\frac{(y_i-\\boldsymbol{X}_{i,*}\\boldsymbol{\\beta})^2}{2\\sigma^2}\\right]}=\\prod_{i=0}^{n-1}p(y_i\\vert \\boldsymbol{X};\\boldsymbol{\\beta}).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Maximum Likelihood Estimation (MLE)\n", + "\n", + "In statistics, maximum likelihood estimation (MLE) is a method of\n", + "estimating the parameters of an assumed probability distribution,\n", + "given some observed data. This is achieved by maximizing a likelihood\n", + "function so that, under the assumed statistical model, the observed\n", + "data is the most probable. \n", + "\n", + "\n", + "We will assume here that our events are given by the above Gaussian\n", + "distribution and we will determine the optimal parameters $\\beta$ by\n", + "maximizing the above PDF. However, computing the derivatives of a\n", + "product function is cumbersome and can easily lead to overflow and/or\n", + "underflowproblems, with potentials for loss of numerical precision.\n", + "\n", + "\n", + "In practice, it is more convenient to maximize the logarithm of the\n", + "PDF because it is a monotonically increasing function of the argument.\n", + "Alternatively, and this will be our option, we will minimize the\n", + "negative of the logarithm since this is a monotonically decreasing\n", + "function.\n", + "\n", + "Note also that maximization/minimization of the logarithm of the PDF\n", + "is equivalent to the maximization/minimization of the function itself.\n", + "\n", + "\n", + "\n", + "## A new Cost Function\n", + "\n", + "We could now define a new cost function to minimize, namely the negative logarithm of the above PDF" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=-\\log{\\prod_{i=0}^{n-1}p(y_i\\vert \\boldsymbol{X};\\boldsymbol{\\beta})}=-\\sum_{i=0}^{n-1}\\log{p(y_i\\vert \\boldsymbol{X};\\boldsymbol{\\beta})},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which becomes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{\\beta}=\\frac{n}{2}\\log{2\\pi\\sigma^2}+\\frac{\\vert\\vert (\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\vert\\vert_2^2}{2\\sigma^2}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Taking the derivative of the *new* cost function with respect to the parameters $\\beta$ we recognize our familiar OLS equation, namely" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{X}^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right) =0,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which leads to the well-known OLS equation for the optimal paramters $\\beta$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\hat{\\boldsymbol{\\beta}}^{\\mathrm{OLS}}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}!\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Before we make a similar analysis for Ridge and Lasso regression, we need a new reminder on statistics. \n", + "\n", + "## More basic Statistics and Bayes' theorem\n", + "\n", + "A central theorem in statistics is Bayes' theorem. This theorem plays a similar role as the good old Pythagoras' theorem in geometry.\n", + "Bayes' theorem is extremely simple to derive. But to do so we need some basic axioms from statistics.\n", + "\n", + "Assume we have two domains of events $X=[x_0,x_1,\\dots,x_{n-1}]$ and $Y=[y_0,y_1,\\dots,y_{n-1}]$.\n", + "\n", + "We define also the likelihood for $X$ and $Y$ as $p(X)$ and $p(Y)$ respectively.\n", + "The likelihood of a specific event $x_i$ (or $y_i$) is then written as $p(X=x_i)$ or just $p(x_i)=p_i$. \n", + "\n", + "**Union of events is given by.**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X)+p(Y)-p(X \\cap Y).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**The product rule (aka joint probability) is given by.**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(X \\cup Y)= p(X,Y)= p(X\\vert Y)p(Y)=p(Y\\vert X)p(x),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where we read $p(X\\vert Y)$ as the likelihood of obtaining $X$ given $Y$.\n", + "\n", + "\n", + "\n", + "If we have independent events then $p(X,Y)=p(X)p(Y)$.\n", + "\n", + "\n", + "## Marginal Probability\n", + "\n", + "The marginal probability is defined in terms of only of the set of variables $X,Y$. For a discrete probability we have\n", + "**Discrete Probability.**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(X)=\\sum_{i=0}^{n-1}p(X,Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert Y=y_i)p(Y=y_i)=\\sum_{i=0}^{n-1}p(X\\vert y_i)p(y_i).\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conditional Probability\n", + "\n", + "The conditional probability, if $p(Y) > 0$, is" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)}=\\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bayes' Theorem\n", + "\n", + "If we combine the conditional probability with the marginal probability and the standard product rule, we have" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{p(Y)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which we can rewrite as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "p(X\\vert Y)= \\frac{p(X,Y)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)}=\\frac{p(Y\\vert X)p(X)}{\\sum_{i=0}^{n-1}p(Y\\vert X=x_i)p(x_i)},\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which is Bayes' theorem. More text to be added here by Friday September 10.\n", + "\n", + "\n", + "## Frliday September 10\n", + "\n", + "Depending on where we end on Thursday September 9, we will revise the material for Friday. On Friday we will also discuss project 1.\n", + "\n", + "\n", + "## Why resampling methods\n", + "\n", + "Before we proceed, we need to rethink what we have been doing. In our\n", + "eager to fit the data, we have omitted several important elements in\n", + "our regression analysis. In what follows we will\n", + "1. look at statistical properties, including a discussion of mean values, variance and the so-called bias-variance tradeoff\n", + "\n", + "2. introduce resampling techniques like cross-validation, bootstrapping and jackknife and more\n", + "\n", + "This will allow us to link the standard linear algebra methods we have discussed above to a statistical interpretation of the methods. \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "## Resampling methods\n", + "Resampling methods are an indispensable tool in modern\n", + "statistics. They involve repeatedly drawing samples from a training\n", + "set and refitting a model of interest on each sample in order to\n", + "obtain additional information about the fitted model. For example, in\n", + "order to estimate the variability of a linear regression fit, we can\n", + "repeatedly draw different samples from the training data, fit a linear\n", + "regression to each new sample, and then examine the extent to which\n", + "the resulting fits differ. Such an approach may allow us to obtain\n", + "information that would not be available from fitting the model only\n", + "once using the original training sample.\n", + "\n", + "Two resampling methods are often used in Machine Learning analyses,\n", + "1. The **bootstrap method**\n", + "\n", + "2. and **Cross-Validation**\n", + "\n", + "In addition there are several other methods such as the Jackknife and the Blocking methods. We will discuss in particular\n", + "cross-validation and the bootstrap method.\n", + "\n", + "\n", + "\n", + "\n", + "## Resampling approaches can be computationally expensive\n", + "\n", + "Resampling approaches can be computationally expensive, because they\n", + "involve fitting the same statistical method multiple times using\n", + "different subsets of the training data. However, due to recent\n", + "advances in computing power, the computational requirements of\n", + "resampling methods generally are not prohibitive. In this chapter, we\n", + "discuss two of the most commonly used resampling methods,\n", + "cross-validation and the bootstrap. Both methods are important tools\n", + "in the practical application of many statistical learning\n", + "procedures. For example, cross-validation can be used to estimate the\n", + "test error associated with a given statistical learning method in\n", + "order to evaluate its performance, or to select the appropriate level\n", + "of flexibility. The process of evaluating a model’s performance is\n", + "known as model assessment, whereas the process of selecting the proper\n", + "level of flexibility for a model is known as model selection. The\n", + "bootstrap is widely used.\n", + "\n", + "\n", + "\n", + "## Why resampling methods ?\n", + "**Statistical analysis.**\n", + "\n", + "\n", + "* Our simulations can be treated as *computer experiments*. This is particularly the case for Monte Carlo methods\n", + "\n", + "* The results can be analysed with the same statistical tools as we would use analysing experimental data.\n", + "\n", + "* As in all experiments, we are looking for expectation values and an estimate of how accurate they are, i.e., possible sources for errors.\n", + "\n", + " \n", + "\n", + "## Statistical analysis\n", + "\n", + "* As in other experiments, many numerical experiments have two classes of errors:\n", + "\n", + " * Statistical errors\n", + "\n", + " * Systematical errors\n", + "\n", + "\n", + "* Statistical errors can be estimated using standard tools from statistics\n", + "\n", + "* Systematical errors are method specific and must be treated differently from case to case.\n", + "\n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "## Resampling methods\n", + "\n", + "With all these analytical equations for both the OLS and Ridge\n", + "regression, we will now outline how to assess a given model. This will\n", + "lead us to a discussion of the so-called bias-variance tradeoff (see\n", + "below) and so-called resampling methods.\n", + "\n", + "One of the quantities we have discussed as a way to measure errors is\n", + "the mean-squared error (MSE), mainly used for fitting of continuous\n", + "functions. Another choice is the absolute error.\n", + "\n", + "In the discussions below we will focus on the MSE and in particular since we will split the data into test and training data,\n", + "we discuss the\n", + "1. prediction error or simply the **test error** $\\mathrm{Err_{Test}}$, where we have a fixed training set and the test error is the MSE arising from the data reserved for testing. We discuss also the \n", + "\n", + "2. training error $\\mathrm{Err_{Train}}$, which is the average loss over the training data.\n", + "\n", + "As our model becomes more and more complex, more of the training data tends to used. The training may thence adapt to more complicated structures in the data. This may lead to a decrease in the bias (see below for code example) and a slight increase of the variance for the test error.\n", + "For a certain level of complexity the test error will reach minimum, before starting to increase again. The\n", + "training error reaches a saturation.\n", + "\n", + "\n", + "\n", + "\n", + "## Resampling methods: Jackknife and Bootstrap\n", + "\n", + "Two famous\n", + "resampling methods are the **independent bootstrap** and **the jackknife**. \n", + "\n", + "The jackknife is a special case of the independent bootstrap. Still, the jackknife was made\n", + "popular prior to the independent bootstrap. And as the popularity of\n", + "the independent bootstrap soared, new variants, such as **the dependent bootstrap**.\n", + "\n", + "The Jackknife and independent bootstrap work for\n", + "independent, identically distributed random variables.\n", + "If these conditions are not\n", + "satisfied, the methods will fail. Yet, it should be said that if the data are\n", + "independent, identically distributed, and we only want to estimate the\n", + "variance of $\\overline{X}$ (which often is the case), then there is no\n", + "need for bootstrapping. \n", + "\n", + "## Resampling methods: Jackknife\n", + "\n", + "The Jackknife works by making many replicas of the estimator $\\widehat{\\theta}$. \n", + "The jackknife is a resampling method where we systematically leave out one observation from the vector of observed values $\\boldsymbol{x} = (x_1,x_2,\\cdots,X_n)$. \n", + "Let $\\boldsymbol{x}_i$ denote the vector" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{x}_i = (x_1,x_2,\\cdots,x_{i-1},x_{i+1},\\cdots,x_n),\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which equals the vector $\\boldsymbol{x}$ with the exception that observation\n", + "number $i$ is left out. Using this notation, define\n", + "$\\widehat{\\theta}_i$ to be the estimator\n", + "$\\widehat{\\theta}$ computed using $\\vec{X}_i$. \n", + "\n", + "\n", + "## Jackknife code example" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from numpy import *\n", + "from numpy.random import randint, randn\n", + "from time import time\n", + "\n", + "def jackknife(data, stat):\n", + " n = len(data);t = zeros(n); inds = arange(n); t0 = time()\n", + " ## 'jackknifing' by leaving out an observation for each i \n", + " for i in range(n):\n", + " t[i] = stat(delete(data,i) )\n", + "\n", + " # analysis \n", + " print(\"Runtime: %g sec\" % (time()-t0)); print(\"Jackknife Statistics :\")\n", + " print(\"original bias std. error\")\n", + " print(\"%8g %14g %15g\" % (stat(data),(n-1)*mean(t)/n, (n*var(t))**.5))\n", + "\n", + " return t\n", + "\n", + "\n", + "# Returns mean of data samples \n", + "def stat(data):\n", + " return mean(data)\n", + "\n", + "\n", + "mu, sigma = 100, 15\n", + "datapoints = 10000\n", + "x = mu + sigma*random.randn(datapoints)\n", + "# jackknife returns the data sample \n", + "t = jackknife(x, stat)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Resampling methods: Bootstrap\n", + "Bootstrapping is a nonparametric approach to statistical inference\n", + "that substitutes computation for more traditional distributional\n", + "assumptions and asymptotic results. Bootstrapping offers a number of\n", + "advantages: \n", + "1. The bootstrap is quite general, although there are some cases in which it fails. \n", + "\n", + "2. Because it does not require distributional assumptions (such as normally distributed errors), the bootstrap can provide more accurate inferences when the data are not well behaved or when the sample size is small. \n", + "\n", + "3. It is possible to apply the bootstrap to statistics with sampling distributions that are difficult to derive, even asymptotically. \n", + "\n", + "4. It is relatively simple to apply the bootstrap to complex data-collection plans (such as stratified and clustered samples).\n", + "\n", + "\n", + "\n", + "\n", + "## Resampling methods: Bootstrap background\n", + "\n", + "Since $\\widehat{\\theta} = \\widehat{\\theta}(\\boldsymbol{X})$ is a function of random variables,\n", + "$\\widehat{\\theta}$ itself must be a random variable. Thus it has\n", + "a pdf, call this function $p(\\boldsymbol{t})$. The aim of the bootstrap is to\n", + "estimate $p(\\boldsymbol{t})$ by the relative frequency of\n", + "$\\widehat{\\theta}$. You can think of this as using a histogram\n", + "in the place of $p(\\boldsymbol{t})$. If the relative frequency closely\n", + "resembles $p(\\vec{t})$, then using numerics, it is straight forward to\n", + "estimate all the interesting parameters of $p(\\boldsymbol{t})$ using point\n", + "estimators. \n", + "\n", + "\n", + "## Resampling methods: More Bootstrap background\n", + "\n", + "In the case that $\\widehat{\\theta}$ has\n", + "more than one component, and the components are independent, we use the\n", + "same estimator on each component separately. If the probability\n", + "density function of $X_i$, $p(x)$, had been known, then it would have\n", + "been straight forward to do this by: \n", + "1. Drawing lots of numbers from $p(x)$, suppose we call one such set of numbers $(X_1^*, X_2^*, \\cdots, X_n^*)$. \n", + "\n", + "2. Then using these numbers, we could compute a replica of $\\widehat{\\theta}$ called $\\widehat{\\theta}^*$. \n", + "\n", + "By repeated use of (1) and (2), many\n", + "estimates of $\\widehat{\\theta}$ could have been obtained. The\n", + "idea is to use the relative frequency of $\\widehat{\\theta}^*$\n", + "(think of a histogram) as an estimate of $p(\\boldsymbol{t})$.\n", + "\n", + "## Resampling methods: Bootstrap approach\n", + "\n", + "But\n", + "unless there is enough information available about the process that\n", + "generated $X_1,X_2,\\cdots,X_n$, $p(x)$ is in general\n", + "unknown. Therefore, [Efron in 1979](https://projecteuclid.org/euclid.aos/1176344552) asked the\n", + "question: What if we replace $p(x)$ by the relative frequency\n", + "of the observation $X_i$; if we draw observations in accordance with\n", + "the relative frequency of the observations, will we obtain the same\n", + "result in some asymptotic sense? The answer is yes.\n", + "\n", + "\n", + "Instead of generating the histogram for the relative\n", + "frequency of the observation $X_i$, just draw the values\n", + "$(X_1^*,X_2^*,\\cdots,X_n^*)$ with replacement from the vector\n", + "$\\boldsymbol{X}$. \n", + "\n", + "## Resampling methods: Bootstrap steps\n", + "\n", + "The independent bootstrap works like this: \n", + "\n", + "1. Draw with replacement $n$ numbers for the observed variables $\\boldsymbol{x} = (x_1,x_2,\\cdots,x_n)$. \n", + "\n", + "2. Define a vector $\\boldsymbol{x}^*$ containing the values which were drawn from $\\boldsymbol{x}$. \n", + "\n", + "3. Using the vector $\\boldsymbol{x}^*$ compute $\\widehat{\\theta}^*$ by evaluating $\\widehat \\theta$ under the observations $\\boldsymbol{x}^*$. \n", + "\n", + "4. Repeat this process $k$ times. \n", + "\n", + "When you are done, you can draw a histogram of the relative frequency\n", + "of $\\widehat \\theta^*$. This is your estimate of the probability\n", + "distribution $p(t)$. Using this probability distribution you can\n", + "estimate any statistics thereof. In principle you never draw the\n", + "histogram of the relative frequency of $\\widehat{\\theta}^*$. Instead\n", + "you use the estimators corresponding to the statistic of interest. For\n", + "example, if you are interested in estimating the variance of $\\widehat\n", + "\\theta$, apply the etsimator $\\widehat \\sigma^2$ to the values\n", + "$\\widehat \\theta ^*$.\n", + "\n", + "\n", + "## Code example for the Bootstrap method\n", + "\n", + "The following code starts with a Gaussian distribution with mean value\n", + "$\\mu =100$ and variance $\\sigma=15$. We use this to generate the data\n", + "used in the bootstrap analysis. The bootstrap analysis returns a data\n", + "set after a given number of bootstrap operations (as many as we have\n", + "data points). This data set consists of estimated mean values for each\n", + "bootstrap operation. The histogram generated by the bootstrap method\n", + "shows that the distribution for these mean values is also a Gaussian,\n", + "centered around the mean value $\\mu=100$ but with standard deviation\n", + "$\\sigma/\\sqrt{n}$, where $n$ is the number of bootstrap samples (in\n", + "this case the same as the number of original data points). The value\n", + "of the standard deviation is what we expect from the central limit\n", + "theorem." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from numpy import *\n", + "from numpy.random import randint, randn\n", + "from time import time\n", + "import matplotlib.mlab as mlab\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Returns mean of bootstrap samples \n", + "def stat(data):\n", + " return mean(data)\n", + "\n", + "# Bootstrap algorithm\n", + "def bootstrap(data, statistic, R):\n", + " t = zeros(R); n = len(data); inds = arange(n); t0 = time()\n", + " # non-parametric bootstrap \n", + " for i in range(R):\n", + " t[i] = statistic(data[randint(0,n,n)])\n", + "\n", + " # analysis \n", + " print(\"Runtime: %g sec\" % (time()-t0)); print(\"Bootstrap Statistics :\")\n", + " print(\"original bias std. error\")\n", + " print(\"%8g %8g %14g %15g\" % (statistic(data), std(data),mean(t),std(t)))\n", + " return t\n", + "\n", + "\n", + "mu, sigma = 100, 15\n", + "datapoints = 10000\n", + "x = mu + sigma*random.randn(datapoints)\n", + "# bootstrap returns the data sample \n", + "t = bootstrap(x, stat, datapoints)\n", + "# the histogram of the bootstrapped data \n", + "n, binsboot, patches = plt.hist(t, 50, normed=1, facecolor='red', alpha=0.75)\n", + "\n", + "# add a 'best fit' line \n", + "y = mlab.normpdf( binsboot, mean(t), std(t))\n", + "lt = plt.plot(binsboot, y, 'r--', linewidth=1)\n", + "plt.xlabel('Smarts')\n", + "plt.ylabel('Probability')\n", + "plt.axis([99.5, 100.6, 0, 3.0])\n", + "plt.grid(True)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Various steps in cross-validation\n", + "\n", + "When the repetitive splitting of the data set is done randomly,\n", + "samples may accidently end up in a fast majority of the splits in\n", + "either training or test set. Such samples may have an unbalanced\n", + "influence on either model building or prediction evaluation. To avoid\n", + "this $k$-fold cross-validation structures the data splitting. The\n", + "samples are divided into $k$ more or less equally sized exhaustive and\n", + "mutually exclusive subsets. In turn (at each split) one of these\n", + "subsets plays the role of the test set while the union of the\n", + "remaining subsets constitutes the training set. Such a splitting\n", + "warrants a balanced representation of each sample in both training and\n", + "test set over the splits. Still the division into the $k$ subsets\n", + "involves a degree of randomness. This may be fully excluded when\n", + "choosing $k=n$. This particular case is referred to as leave-one-out\n", + "cross-validation (LOOCV). \n", + "\n", + "\n", + "## How to set up the cross-validation for Ridge and/or Lasso\n", + "\n", + "* Define a range of interest for the penalty parameter.\n", + "\n", + "* Divide the data set into training and test set comprising samples $\\{1, \\ldots, n\\} \\setminus i$ and $\\{ i \\}$, respectively.\n", + "\n", + "* Fit the linear regression model by means of ridge estimation for each $\\lambda$ in the grid using the training set, and the corresponding estimate of the error variance $\\boldsymbol{\\sigma}_{-i}^2(\\lambda)$, as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "\\boldsymbol{\\beta}_{-i}(\\lambda) & = ( \\boldsymbol{X}_{-i, \\ast}^{T}\n", + "\\boldsymbol{X}_{-i, \\ast} + \\lambda \\boldsymbol{I}_{pp})^{-1}\n", + "\\boldsymbol{X}_{-i, \\ast}^{T} \\boldsymbol{y}_{-i}\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* Evaluate the prediction performance of these models on the test set by $\\log\\{L[y_i, \\boldsymbol{X}_{i, \\ast}; \\boldsymbol{\\beta}_{-i}(\\lambda), \\boldsymbol{\\sigma}_{-i}^2(\\lambda)]\\}$. Or, by the prediction error $|y_i - \\boldsymbol{X}_{i, \\ast} \\boldsymbol{\\beta}_{-i}(\\lambda)|$, the relative error, the error squared or the R2 score function.\n", + "\n", + "* Repeat the first three steps such that each sample plays the role of the test set once.\n", + "\n", + "* Average the prediction performances of the test sets at each grid point of the penalty bias/parameter. It is an estimate of the prediction performance of the model corresponding to this value of the penalty parameter on novel data. It is defined as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\begin{align*}\n", + "\\frac{1}{n} \\sum_{i = 1}^n \\log\\{L[y_i, \\mathbf{X}_{i, \\ast}; \\boldsymbol{\\beta}_{-i}(\\lambda), \\boldsymbol{\\sigma}_{-i}^2(\\lambda)]\\}.\n", + "\\end{align*}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cross-validation in brief\n", + "\n", + "For the various values of $k$\n", + "\n", + "1. shuffle the dataset randomly.\n", + "\n", + "2. Split the dataset into $k$ groups.\n", + "\n", + "3. For each unique group:\n", + "\n", + "a. Decide which group to use as set for test data\n", + "\n", + "b. Take the remaining groups as a training data set\n", + "\n", + "c. Fit a model on the training set and evaluate it on the test set\n", + "\n", + "d. Retain the evaluation score and discard the model\n", + "\n", + "\n", + "5. Summarize the model using the sample of model evaluation scores\n", + "\n", + "## Code Example for Cross-validation and $k$-fold Cross-validation\n", + "\n", + "The code here uses Ridge regression with cross-validation (CV) resampling and $k$-fold CV in order to fit a specific polynomial." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import KFold\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.model_selection import cross_val_score\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "# Useful for eventual debugging.\n", + "np.random.seed(3155)\n", + "\n", + "# Generate the data.\n", + "nsamples = 100\n", + "x = np.random.randn(nsamples)\n", + "y = 3*x**2 + np.random.randn(nsamples)\n", + "\n", + "## Cross-validation on Ridge regression using KFold only\n", + "\n", + "# Decide degree on polynomial to fit\n", + "poly = PolynomialFeatures(degree = 6)\n", + "\n", + "# Decide which values of lambda to use\n", + "nlambdas = 500\n", + "lambdas = np.logspace(-3, 5, nlambdas)\n", + "\n", + "# Initialize a KFold instance\n", + "k = 5\n", + "kfold = KFold(n_splits = k)\n", + "\n", + "# Perform the cross-validation to estimate MSE\n", + "scores_KFold = np.zeros((nlambdas, k))\n", + "\n", + "i = 0\n", + "for lmb in lambdas:\n", + " ridge = Ridge(alpha = lmb)\n", + " j = 0\n", + " for train_inds, test_inds in kfold.split(x):\n", + " xtrain = x[train_inds]\n", + " ytrain = y[train_inds]\n", + "\n", + " xtest = x[test_inds]\n", + " ytest = y[test_inds]\n", + "\n", + " Xtrain = poly.fit_transform(xtrain[:, np.newaxis])\n", + " ridge.fit(Xtrain, ytrain[:, np.newaxis])\n", + "\n", + " Xtest = poly.fit_transform(xtest[:, np.newaxis])\n", + " ypred = ridge.predict(Xtest)\n", + "\n", + " scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred)\n", + "\n", + " j += 1\n", + " i += 1\n", + "\n", + "\n", + "estimated_mse_KFold = np.mean(scores_KFold, axis = 1)\n", + "\n", + "## Cross-validation using cross_val_score from sklearn along with KFold\n", + "\n", + "# kfold is an instance initialized above as:\n", + "# kfold = KFold(n_splits = k)\n", + "\n", + "estimated_mse_sklearn = np.zeros(nlambdas)\n", + "i = 0\n", + "for lmb in lambdas:\n", + " ridge = Ridge(alpha = lmb)\n", + "\n", + " X = poly.fit_transform(x[:, np.newaxis])\n", + " estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold)\n", + "\n", + " # cross_val_score return an array containing the estimated negative mse for every fold.\n", + " # we have to the the mean of every array in order to get an estimate of the mse of the model\n", + " estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n", + "\n", + " i += 1\n", + "\n", + "## Plot and compare the slightly different ways to perform cross-validation\n", + "\n", + "plt.figure()\n", + "\n", + "plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n", + "plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold')\n", + "\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('mse')\n", + "\n", + "plt.legend()\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The bias-variance tradeoff\n", + "\n", + "\n", + "We will discuss the bias-variance tradeoff in the context of\n", + "continuous predictions such as regression. However, many of the\n", + "intuitions and ideas discussed here also carry over to classification\n", + "tasks. Consider a dataset $\\mathcal{L}$ consisting of the data\n", + "$\\mathbf{X}_\\mathcal{L}=\\{(y_j, \\boldsymbol{x}_j), j=0\\ldots n-1\\}$. \n", + "\n", + "Let us assume that the true data is generated from a noisy model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\boldsymbol{y}=f(\\boldsymbol{x}) + \\boldsymbol{\\epsilon}\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "where $\\epsilon$ is normally distributed with mean zero and standard deviation $\\sigma^2$.\n", + "\n", + "In our derivation of the ordinary least squares method we defined then\n", + "an approximation to the function $f$ in terms of the parameters\n", + "$\\boldsymbol{\\beta}$ and the design matrix $\\boldsymbol{X}$ which embody our model,\n", + "that is $\\boldsymbol{\\tilde{y}}=\\boldsymbol{X}\\boldsymbol{\\beta}$. \n", + "\n", + "Thereafter we found the parameters $\\boldsymbol{\\beta}$ by optimizing the means squared error via the so-called cost function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "C(\\boldsymbol{X},\\boldsymbol{\\beta}) =\\frac{1}{n}\\sum_{i=0}^{n-1}(y_i-\\tilde{y}_i)^2=\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right].\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can rewrite this as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\frac{1}{n}\\sum_i(f_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2+\\frac{1}{n}\\sum_i(\\tilde{y}_i-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2+\\sigma^2.\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The three terms represent the square of the bias of the learning\n", + "method, which can be thought of as the error caused by the simplifying\n", + "assumptions built into the method. The second term represents the\n", + "variance of the chosen model and finally the last terms is variance of\n", + "the error $\\boldsymbol{\\epsilon}$.\n", + "\n", + "To derive this equation, we need to recall that the variance of $\\boldsymbol{y}$ and $\\boldsymbol{\\epsilon}$ are both equal to $\\sigma^2$. The mean value of $\\boldsymbol{\\epsilon}$ is by definition equal to zero. Furthermore, the function $f$ is not a stochastics variable, idem for $\\boldsymbol{\\tilde{y}}$.\n", + "We use a more compact notation in terms of the expectation value" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{f}+\\boldsymbol{\\epsilon}-\\boldsymbol{\\tilde{y}})^2\\right],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and adding and subtracting $\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]$ we get" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{f}+\\boldsymbol{\\epsilon}-\\boldsymbol{\\tilde{y}}+\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right]-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2\\right],\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which, using the abovementioned expectation values can be rewritten as" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$\n", + "\\mathbb{E}\\left[(\\boldsymbol{y}-\\boldsymbol{\\tilde{y}})^2\\right]=\\mathbb{E}\\left[(\\boldsymbol{y}-\\mathbb{E}\\left[\\boldsymbol{\\tilde{y}}\\right])^2\\right]+\\mathrm{Var}\\left[\\boldsymbol{\\tilde{y}}\\right]+\\sigma^2,\n", + "$$" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "that is the rewriting in terms of the so-called bias, the variance of the model $\\boldsymbol{\\tilde{y}}$ and the variance of $\\boldsymbol{\\epsilon}$.\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "## Example code for Bias-Variance tradeoff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.utils import resample\n", + "\n", + "np.random.seed(2018)\n", + "\n", + "n = 500\n", + "n_boostraps = 100\n", + "degree = 18 # A quite high value, just to show.\n", + "noise = 0.1\n", + "\n", + "# Make data set.\n", + "x = np.linspace(-1, 3, n).reshape(-1, 1)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1, x.shape)\n", + "\n", + "# Hold out some test data that is never used in training.\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "# Combine x transformation and model into one operation.\n", + "# Not neccesary, but convenient.\n", + "model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n", + "\n", + "# The following (m x n_bootstraps) matrix holds the column vectors y_pred\n", + "# for each bootstrap iteration.\n", + "y_pred = np.empty((y_test.shape[0], n_boostraps))\n", + "for i in range(n_boostraps):\n", + " x_, y_ = resample(x_train, y_train)\n", + "\n", + " # Evaluate the new model on the same test data each time.\n", + " y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()\n", + "\n", + "# Note: Expectations and variances taken w.r.t. different training\n", + "# data sets, hence the axis=1. Subsequent means are taken across the test data\n", + "# set in order to obtain a total value, but before this we have error/bias/variance\n", + "# calculated per data point in the test set.\n", + "# Note 2: The use of keepdims=True is important in the calculation of bias as this \n", + "# maintains the column vector form. Dropping this yields very unexpected results.\n", + "error = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n", + "bias = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n", + "variance = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n", + "print('Error:', error)\n", + "print('Bias^2:', bias)\n", + "print('Var:', variance)\n", + "print('{} >= {} + {} = {}'.format(error, bias, variance, bias+variance))\n", + "\n", + "plt.plot(x[::5, :], y[::5, :], label='f(x)')\n", + "plt.scatter(x_test, y_test, label='Data points')\n", + "plt.scatter(x_test, np.mean(y_pred, axis=1), label='Pred')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Understanding what happens" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.utils import resample\n", + "\n", + "np.random.seed(2018)\n", + "\n", + "n = 40\n", + "n_boostraps = 100\n", + "maxdegree = 14\n", + "\n", + "\n", + "# Make data set.\n", + "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", + "error = np.zeros(maxdegree)\n", + "bias = np.zeros(maxdegree)\n", + "variance = np.zeros(maxdegree)\n", + "polydegree = np.zeros(maxdegree)\n", + "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "for degree in range(maxdegree):\n", + " model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n", + " y_pred = np.empty((y_test.shape[0], n_boostraps))\n", + " for i in range(n_boostraps):\n", + " x_, y_ = resample(x_train, y_train)\n", + " y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()\n", + "\n", + " polydegree[degree] = degree\n", + " error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n", + " bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n", + " variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n", + " print('Polynomial degree:', degree)\n", + " print('Error:', error[degree])\n", + " print('Bias^2:', bias[degree])\n", + " print('Var:', variance[degree])\n", + " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n", + "\n", + "plt.plot(polydegree, error, label='Error')\n", + "plt.plot(polydegree, bias, label='bias')\n", + "plt.plot(polydegree, variance, label='Variance')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Summing up\n", + "\n", + "\n", + "\n", + "\n", + "The bias-variance tradeoff summarizes the fundamental tension in\n", + "machine learning, particularly supervised learning, between the\n", + "complexity of a model and the amount of training data needed to train\n", + "it. Since data is often limited, in practice it is often useful to\n", + "use a less-complex model with higher bias, that is a model whose asymptotic\n", + "performance is worse than another model because it is easier to\n", + "train and less sensitive to sampling noise arising from having a\n", + "finite-sized training dataset (smaller variance). \n", + "\n", + "\n", + "\n", + "The above equations tell us that in\n", + "order to minimize the expected test error, we need to select a\n", + "statistical learning method that simultaneously achieves low variance\n", + "and low bias. Note that variance is inherently a nonnegative quantity,\n", + "and squared bias is also nonnegative. Hence, we see that the expected\n", + "test MSE can never lie below $Var(\\epsilon)$, the irreducible error.\n", + "\n", + "\n", + "What do we mean by the variance and bias of a statistical learning\n", + "method? The variance refers to the amount by which our model would change if we\n", + "estimated it using a different training data set. Since the training\n", + "data are used to fit the statistical learning method, different\n", + "training data sets will result in a different estimate. But ideally the\n", + "estimate for our model should not vary too much between training\n", + "sets. However, if a method has high variance then small changes in\n", + "the training data can result in large changes in the model. In general, more\n", + "flexible statistical methods have higher variance.\n", + "\n", + "\n", + "You may also find this recent [article](https://www.pnas.org/content/116/32/15849) of interest.\n", + "\n", + "## Another Example from Scikit-Learn's Repository" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\"\"\"\n", + "============================\n", + "Underfitting vs. Overfitting\n", + "============================\n", + "\n", + "This example demonstrates the problems of underfitting and overfitting and\n", + "how we can use linear regression with polynomial features to approximate\n", + "nonlinear functions. The plot shows the function that we want to approximate,\n", + "which is a part of the cosine function. In addition, the samples from the\n", + "real function and the approximations of different models are displayed. The\n", + "models have polynomial features of different degrees. We can see that a\n", + "linear function (polynomial with degree 1) is not sufficient to fit the\n", + "training samples. This is called **underfitting**. A polynomial of degree 4\n", + "approximates the true function almost perfectly. However, for higher degrees\n", + "the model will **overfit** the training data, i.e. it learns the noise of the\n", + "training data.\n", + "We evaluate quantitatively **overfitting** / **underfitting** by using\n", + "cross-validation. We calculate the mean squared error (MSE) on the validation\n", + "set, the higher, the less likely the model generalizes correctly from the\n", + "training data.\n", + "\"\"\"\n", + "\n", + "print(__doc__)\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "from sklearn.linear_model import LinearRegression\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "def true_fun(X):\n", + " return np.cos(1.5 * np.pi * X)\n", + "\n", + "np.random.seed(0)\n", + "\n", + "n_samples = 30\n", + "degrees = [1, 4, 15]\n", + "\n", + "X = np.sort(np.random.rand(n_samples))\n", + "y = true_fun(X) + np.random.randn(n_samples) * 0.1\n", + "\n", + "plt.figure(figsize=(14, 5))\n", + "for i in range(len(degrees)):\n", + " ax = plt.subplot(1, len(degrees), i + 1)\n", + " plt.setp(ax, xticks=(), yticks=())\n", + "\n", + " polynomial_features = PolynomialFeatures(degree=degrees[i],\n", + " include_bias=False)\n", + " linear_regression = LinearRegression()\n", + " pipeline = Pipeline([(\"polynomial_features\", polynomial_features),\n", + " (\"linear_regression\", linear_regression)])\n", + " pipeline.fit(X[:, np.newaxis], y)\n", + "\n", + " # Evaluate the models using crossvalidation\n", + " scores = cross_val_score(pipeline, X[:, np.newaxis], y,\n", + " scoring=\"neg_mean_squared_error\", cv=10)\n", + "\n", + " X_test = np.linspace(0, 1, 100)\n", + " plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label=\"Model\")\n", + " plt.plot(X_test, true_fun(X_test), label=\"True function\")\n", + " plt.scatter(X, y, edgecolor='b', s=20, label=\"Samples\")\n", + " plt.xlabel(\"x\")\n", + " plt.ylabel(\"y\")\n", + " plt.xlim((0, 1))\n", + " plt.ylim((-2, 2))\n", + " plt.legend(loc=\"best\")\n", + " plt.title(\"Degree {}\\nMSE = {:.2e}(+/- {:.2e})\".format(\n", + " degrees[i], -scores.mean(), scores.std()))\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## More examples on bootstrap and cross-validation and errors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.utils import resample\n", + "from sklearn.metrics import mean_squared_error\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organize the data into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "\n", + "Maxpolydegree = 30\n", + "X = np.zeros((len(Density),Maxpolydegree))\n", + "X[:,0] = 1.0\n", + "testerror = np.zeros(Maxpolydegree)\n", + "trainingerror = np.zeros(Maxpolydegree)\n", + "polynomial = np.zeros(Maxpolydegree)\n", + "\n", + "trials = 100\n", + "for polydegree in range(1, Maxpolydegree):\n", + " polynomial[polydegree] = polydegree\n", + " for degree in range(polydegree):\n", + " X[:,degree] = Density**(degree/3.0)\n", + "\n", + "# loop over trials in order to estimate the expectation value of the MSE\n", + " testerror[polydegree] = 0.0\n", + " trainingerror[polydegree] = 0.0\n", + " for samples in range(trials):\n", + " x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)\n", + " model = LinearRegression(fit_intercept=True).fit(x_train, y_train)\n", + " ypred = model.predict(x_train)\n", + " ytilde = model.predict(x_test)\n", + " testerror[polydegree] += mean_squared_error(y_test, ytilde)\n", + " trainingerror[polydegree] += mean_squared_error(y_train, ypred) \n", + "\n", + " testerror[polydegree] /= trials\n", + " trainingerror[polydegree] /= trials\n", + " print(\"Degree of polynomial: %3d\"% polynomial[polydegree])\n", + " print(\"Mean squared error on training data: %.8f\" % trainingerror[polydegree])\n", + " print(\"Mean squared error on test data: %.8f\" % testerror[polydegree])\n", + "\n", + "plt.plot(polynomial, np.log10(trainingerror), label='Training Error')\n", + "plt.plot(polynomial, np.log10(testerror), label='Test Error')\n", + "plt.xlabel('Polynomial degree')\n", + "plt.ylabel('log10[MSE]')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## The same example but now with cross-validation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Common imports\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import KFold\n", + "from sklearn.model_selection import cross_val_score\n", + "\n", + "\n", + "# Where to save the figures and data files\n", + "PROJECT_ROOT_DIR = \"Results\"\n", + "FIGURE_ID = \"Results/FigureFiles\"\n", + "DATA_ID = \"DataFiles/\"\n", + "\n", + "if not os.path.exists(PROJECT_ROOT_DIR):\n", + " os.mkdir(PROJECT_ROOT_DIR)\n", + "\n", + "if not os.path.exists(FIGURE_ID):\n", + " os.makedirs(FIGURE_ID)\n", + "\n", + "if not os.path.exists(DATA_ID):\n", + " os.makedirs(DATA_ID)\n", + "\n", + "def image_path(fig_id):\n", + " return os.path.join(FIGURE_ID, fig_id)\n", + "\n", + "def data_path(dat_id):\n", + " return os.path.join(DATA_ID, dat_id)\n", + "\n", + "def save_fig(fig_id):\n", + " plt.savefig(image_path(fig_id) + \".png\", format='png')\n", + "\n", + "infile = open(data_path(\"EoS.csv\"),'r')\n", + "\n", + "# Read the EoS data as csv file and organize the data into two arrays with density and energies\n", + "EoS = pd.read_csv(infile, names=('Density', 'Energy'))\n", + "EoS['Energy'] = pd.to_numeric(EoS['Energy'], errors='coerce')\n", + "EoS = EoS.dropna()\n", + "Energies = EoS['Energy']\n", + "Density = EoS['Density']\n", + "# The design matrix now as function of various polytrops\n", + "\n", + "Maxpolydegree = 30\n", + "X = np.zeros((len(Density),Maxpolydegree))\n", + "X[:,0] = 1.0\n", + "estimated_mse_sklearn = np.zeros(Maxpolydegree)\n", + "polynomial = np.zeros(Maxpolydegree)\n", + "k =5\n", + "kfold = KFold(n_splits = k)\n", + "\n", + "for polydegree in range(1, Maxpolydegree):\n", + " polynomial[polydegree] = polydegree\n", + " for degree in range(polydegree):\n", + " X[:,degree] = Density**(degree/3.0)\n", + " OLS = LinearRegression()\n", + "# loop over trials in order to estimate the expectation value of the MSE\n", + " estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold)\n", + "#[:, np.newaxis]\n", + " estimated_mse_sklearn[polydegree] = np.mean(-estimated_mse_folds)\n", + "\n", + "plt.plot(polynomial, np.log10(estimated_mse_sklearn), label='Test Error')\n", + "plt.xlabel('Polynomial degree')\n", + "plt.ylabel('log10[MSE]')\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cross-validation with Ridge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from sklearn.model_selection import KFold\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.model_selection import cross_val_score\n", + "from sklearn.preprocessing import PolynomialFeatures\n", + "\n", + "# A seed just to ensure that the random numbers are the same for every run.\n", + "np.random.seed(3155)\n", + "# Generate the data.\n", + "n = 100\n", + "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", + "# Decide degree on polynomial to fit\n", + "poly = PolynomialFeatures(degree = 10)\n", + "\n", + "# Decide which values of lambda to use\n", + "nlambdas = 500\n", + "lambdas = np.logspace(-3, 5, nlambdas)\n", + "# Initialize a KFold instance\n", + "k = 5\n", + "kfold = KFold(n_splits = k)\n", + "estimated_mse_sklearn = np.zeros(nlambdas)\n", + "i = 0\n", + "for lmb in lambdas:\n", + " ridge = Ridge(alpha = lmb)\n", + " estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)\n", + " estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n", + " i += 1\n", + "plt.figure()\n", + "plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n", + "plt.xlabel('log10(lambda)')\n", + "plt.ylabel('MSE')\n", + "plt.legend()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}