Morten Hjorth-Jensen
702 lines
33 KiB
HTML
702 lines
33 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Exercises week 36 — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
<!--
|
||
this give us a css class that will be invisible only if js is disabled
|
||
-->
|
||
<noscript>
|
||
<style>
|
||
.pst-js-only { display: none !important; }
|
||
|
||
</style>
|
||
</noscript>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=26a4bc78f4c0ddb94549" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=26a4bc78f4c0ddb94549" rel="stylesheet" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=a3416100" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.4510f1fc1dee50b3e5859aac5469c37c29e427902b24a333a5f9fcb2f0b3ac41.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- So that users can add custom icons -->
|
||
<script src="_static/scripts/fontawesome.js?digest=26a4bc78f4c0ddb94549"></script>
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek36';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Week 36: Linear Regression and Statistical interpretations" href="week36.html" />
|
||
<link rel="prev" title="Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression" href="week35.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
<meta name="docsearch:version" content="" />
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<dialog id="pst-search-dialog">
|
||
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form>
|
||
</dialog>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<dialog id="pst-primary-sidebar-modal"></dialog>
|
||
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<img src="_static/logo.png" class="logo__image only-dark pst-js-only" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Teaching schedule with links to material</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 36</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Statistical interpretations</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week38.html">Week 38: Logistic Regression and Optimization</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek39.html">Exercises week 39</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week39.html">Week 39: Optimization and Gradient Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week40.html">Week 40: Gradient descent methods (continued) and start Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek41.html">Exercises week 41</a></li>
|
||
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="week41.html">Week 41 Neural networks and constructing a neural network code</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek42.html">Exercises week 42</a></li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="week42.html">Week 42 Constructing a Neural Network code with examples</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="additionweek42.html">Exercises Week 42: Logistic Regression and Optimization, reminders from week 38 and week 40</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week43.html">Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek43.html">Exercises week 43</a></li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="toctree-l1"><a class="reference internal" href="week44.html">Week 44, Convolutional Neural Networks (CNN)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week45.html">Week 45, Convolutional Neural Networks (CCNs) and Recurrent Neural Networks (RNNs)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week46.html">Week 46: Decision Trees, Ensemble methods and Random Forests</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 7 (midnight), 2024</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="project2.html">Project 2 on Machine Learning, deadline November 4 (Midnight)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="project3.html">Project 3 on Machine Learning, deadline December 9 (midnight), 2024</a></li>
|
||
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-launch-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Launch interactive content">
|
||
<i class="fas fa-rocket"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="https://mybinder.org/v2/git/https%3A//compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/index.html/master?urlpath=tree/exercisesweek36.ipynb" target="_blank"
|
||
class="btn btn-sm dropdown-item"
|
||
title="Launch on Binder"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
|
||
<img alt="Binder logo" src="_static/images/logo_binder.svg">
|
||
</span>
|
||
<span class="btn__text-container">Binder</span>
|
||
</a>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/exercisesweek36.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
|
||
</button>
|
||
|
||
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Exercises week 36</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">Overarching aims of the exercises this week</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-analytical-exercises">Exercise 1: Analytical exercises</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#a-expression-for-ridge-regression">a) Expression for Ridge regression</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#b-the-singular-value-decomposition">b) The singular value decomposition</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-adding-ridge-regression">Exercise 2: Adding Ridge Regression</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html exercisesweek36.do.txt -->
|
||
<!-- dom:TITLE: Exercises week 36 --><section class="tex2jax_ignore mathjax_ignore" id="exercises-week-36">
|
||
<h1>Exercises week 36<a class="headerlink" href="#exercises-week-36" title="Link to this heading">#</a></h1>
|
||
<p><strong>September 2-6, 2024</strong></p>
|
||
<p>Date: <strong>Deadline is Friday September 6 at midnight</strong></p>
|
||
<section id="overarching-aims-of-the-exercises-this-week">
|
||
<h2>Overarching aims of the exercises this week<a class="headerlink" href="#overarching-aims-of-the-exercises-this-week" title="Link to this heading">#</a></h2>
|
||
<p>This set of exercises form an important part of the first project. The
|
||
analytical exercises deal with the material covered last week on the
|
||
mathematical interpretations of ordinary least squares and of Ridge
|
||
regression. The numerical exercises can be seen as a continuation of
|
||
exercise 3 from week 35, with the inclusion of Ridge regression. This
|
||
material enters also the discussions of the first project.</p>
|
||
</section>
|
||
<section id="exercise-1-analytical-exercises">
|
||
<h2>Exercise 1: Analytical exercises<a class="headerlink" href="#exercise-1-analytical-exercises" title="Link to this heading">#</a></h2>
|
||
<p>The aim here is to derive the expression for the optimal parameters
|
||
using Ridge regression. Furthermore, using the singular value
|
||
decomposition, we will analyze the difference between the ordinary
|
||
least squares approach and Ridge regression.</p>
|
||
<p>The expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, was given by the
|
||
optimization problem</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
||
\]</div>
|
||
<p>which we can also write as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2,
|
||
\]</div>
|
||
<p>where we have used the definition of a norm-2 vector, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
||
\]</div>
|
||
<p>By minimizing the above equation with respect to the parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> we could then obtain an analytical expression for the
|
||
parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>.</p>
|
||
<p>We can add a regularization parameter <span class="math notranslate nohighlight">\(\lambda\)</span> by
|
||
defining a new cost function to be optimized, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2
|
||
\]</div>
|
||
<p>which leads to the Ridge regression minimization problem. One can require as part of the optimization problem
|
||
that <span class="math notranslate nohighlight">\(\vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t\)</span>, where <span class="math notranslate nohighlight">\(t\)</span> is
|
||
a finite number larger than zero. We will not implement that here.</p>
|
||
<section id="a-expression-for-ridge-regression">
|
||
<h3>a) Expression for Ridge regression<a class="headerlink" href="#a-expression-for-ridge-regression" title="Link to this heading">#</a></h3>
|
||
<p>Show that the optimal parameters</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{I}\)</span> being a <span class="math notranslate nohighlight">\(p\times p\)</span> identity matrix.</p>
|
||
<p>The ordinary least squares result is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
</section>
|
||
<section id="b-the-singular-value-decomposition">
|
||
<h3>b) The singular value decomposition<a class="headerlink" href="#b-the-singular-value-decomposition" title="Link to this heading">#</a></h3>
|
||
<p>Here we will use the singular value decomposition of an <span class="math notranslate nohighlight">\(n\times p\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> (our design matrix)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>to study properties of Ridge regression and ordinary least squares regression.
|
||
Here <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are orthogonal matrices of dimensions
|
||
<span class="math notranslate nohighlight">\(n\times n\)</span> and <span class="math notranslate nohighlight">\(p\times p\)</span>, respectively, and <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> is an
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span> matrix which contains the singular values only. This material was discussed during the lectures of week 35.</p>
|
||
<p>Show that you can write the
|
||
OLS solutions in terms of the eigenvectors (the columns) of the orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}_{\mathrm{OLS}}=\boldsymbol{X}\boldsymbol{\beta} = \sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>For Ridge regression, show that the corresponding equation is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}_{\mathrm{Ridge}}=\boldsymbol{X}\boldsymbol{\beta}_{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{\Sigma}^2\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y},
|
||
\]</div>
|
||
<p>with the vectors <span class="math notranslate nohighlight">\(\boldsymbol{u}_j\)</span> being the columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> from the SVD of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
<p>Give an interpretation of the results. <a class="reference external" href="https://link.springer.com/book/10.1007/978-0-387-84858-7">Section 3.4 of Hastie et al’s textbook gives a good discussion of the above results</a>.</p>
|
||
</section>
|
||
</section>
|
||
<section id="exercise-2-adding-ridge-regression">
|
||
<h2>Exercise 2: Adding Ridge Regression<a class="headerlink" href="#exercise-2-adding-ridge-regression" title="Link to this heading">#</a></h2>
|
||
<p>This exercise is a continuation of exercise 3 from week 35, see <a class="reference external" href="https://compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/exercisesweek35.html">https://compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/exercisesweek35.html</a>. We will use the same function to
|
||
generate our data set, still staying with a simple function <span class="math notranslate nohighlight">\(y(x)\)</span>
|
||
which we want to fit using linear regression, but now extending the
|
||
analysis to include the Ridge regression method.</p>
|
||
<p>In this exercise you need to include the same elements from last week, that is</p>
|
||
<ol class="arabic simple">
|
||
<li><p>scale your data by subtracting the mean value from each column in the design matrix.</p></li>
|
||
<li><p>perform a split of the data in a training set and a test set.</p></li>
|
||
</ol>
|
||
<p>The addition to the analysis this time is the introduction of the hyperparameter <span class="math notranslate nohighlight">\(\lambda\)</span> when introducing Ridge regression.</p>
|
||
<p>Extend the code from exercise 3 from <a class="reference external" href="https://compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/exercisesweek35.html">week 35</a> to include Ridge regression with the hyperparameter <span class="math notranslate nohighlight">\(\lambda\)</span>. The optimal parameters <span class="math notranslate nohighlight">\(\hat{\beta}\)</span> for Ridge regression can be obtained by matrix inversion in a similar way as done for ordinary least squares. You need to add to your code the following equations</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The ordinary least squares result you encoded last week is given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>Use these results to compute the mean squared error for ordinary least
|
||
squares and Ridge regression first for a polynomial of degree five
|
||
with <span class="math notranslate nohighlight">\(n=100\)</span> data points and five selected values of
|
||
<span class="math notranslate nohighlight">\(\lambda=[0.0001,0.001, 0.01,0.1,1.0]\)</span>. Compute thereafter the mean
|
||
squared error for the same values of <span class="math notranslate nohighlight">\(\lambda\)</span> for polynomials of degree ten
|
||
and <span class="math notranslate nohighlight">\(15\)</span>. Discuss your results for the training MSE and test MSE with
|
||
Ridge regression and ordinary least squares.</p>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="week35.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="week36.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Week 36: Linear Regression and Statistical interpretations</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<dialog id="pst-secondary-sidebar-modal"></dialog>
|
||
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#overarching-aims-of-the-exercises-this-week">Overarching aims of the exercises this week</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-analytical-exercises">Exercise 1: Analytical exercises</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#a-expression-for-ridge-regression">a) Expression for Ridge regression</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#b-the-singular-value-decomposition">b) The singular value decomposition</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-adding-ridge-regression">Exercise 2: Adding Ridge Regression</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script defer src="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549"></script>
|
||
<script defer src="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |