Morten Hjorth-Jensen
664 lines
33 KiB
HTML
664 lines
33 KiB
HTML
|
|
|
|
<!DOCTYPE html>
|
|
|
|
|
|
<html lang="en" >
|
|
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />
|
|
|
|
<title>Exercises week 35 — Applied Data Analysis and Machine Learning</title>
|
|
|
|
|
|
|
|
<script data-cfasync="false">
|
|
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
|
document.documentElement.dataset.theme = localStorage.getItem("theme") || "light";
|
|
</script>
|
|
|
|
<!-- Loaded before other Sphinx assets -->
|
|
<link href="_static/styles/theme.css?digest=ac02cc09edc035673794" rel="stylesheet" />
|
|
<link href="_static/styles/bootstrap.css?digest=ac02cc09edc035673794" rel="stylesheet" />
|
|
<link href="_static/styles/pydata-sphinx-theme.css?digest=ac02cc09edc035673794" rel="stylesheet" />
|
|
|
|
|
|
<link href="_static/vendor/fontawesome/6.1.2/css/all.min.css?digest=ac02cc09edc035673794" rel="stylesheet" />
|
|
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.1.2/webfonts/fa-solid-900.woff2" />
|
|
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.1.2/webfonts/fa-brands-400.woff2" />
|
|
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.1.2/webfonts/fa-regular-400.woff2" />
|
|
|
|
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
|
<link rel="stylesheet" href="_static/styles/sphinx-book-theme.css?digest=14f4ca6b54d191a8c7657f6c759bf11a5fb86285" type="text/css" />
|
|
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
|
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
|
<link rel="stylesheet" type="text/css" href="_static/mystnb.4510f1fc1dee50b3e5859aac5469c37c29e427902b24a333a5f9fcb2f0b3ac41.css" />
|
|
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
|
<link rel="stylesheet" type="text/css" href="_static/design-style.4045f2051d55cab465a707391d5b2007.min.css" />
|
|
|
|
<!-- Pre-loaded scripts that we'll load fully later -->
|
|
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=ac02cc09edc035673794" />
|
|
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=ac02cc09edc035673794" />
|
|
<script src="_static/vendor/fontawesome/6.1.2/js/all.min.js?digest=ac02cc09edc035673794"></script>
|
|
|
|
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
|
<script src="_static/jquery.js"></script>
|
|
<script src="_static/underscore.js"></script>
|
|
<script src="_static/_sphinx_javascript_frameworks_compat.js"></script>
|
|
<script src="_static/doctools.js"></script>
|
|
<script src="_static/clipboard.min.js"></script>
|
|
<script src="_static/copybutton.js"></script>
|
|
<script src="_static/scripts/sphinx-book-theme.js?digest=5a5c038af52cf7bc1a1ec88eea08e6366ee68824"></script>
|
|
<script>let toggleHintShow = 'Click to show';</script>
|
|
<script>let toggleHintHide = 'Click to hide';</script>
|
|
<script>let toggleOpenOnPrint = 'true';</script>
|
|
<script src="_static/togglebutton.js"></script>
|
|
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
|
<script src="_static/design-tabs.js"></script>
|
|
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
|
const thebe_selector = ".thebe,.cell"
|
|
const thebe_selector_input = "pre"
|
|
const thebe_selector_output = ".output, .cell_output"
|
|
</script>
|
|
<script async="async" src="_static/sphinx-thebe.js"></script>
|
|
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
|
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
|
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek35';</script>
|
|
<link rel="index" title="Index" href="genindex.html" />
|
|
<link rel="search" title="Search" href="search.html" />
|
|
<link rel="next" title="Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression" href="week35.html" />
|
|
<link rel="prev" title="Week 34: Introduction to the course, Logistics and Practicalities" href="week34.html" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|
<meta name="docsearch:language" content="en"/>
|
|
</head>
|
|
|
|
|
|
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
|
|
|
|
|
|
|
<a class="skip-link" href="#main-content">Skip to main content</a>
|
|
|
|
<div id="pst-scroll-pixel-helper"></div>
|
|
|
|
|
|
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
|
<i class="fa-solid fa-arrow-up"></i>
|
|
Back to top
|
|
</button>
|
|
|
|
|
|
<input type="checkbox"
|
|
class="sidebar-toggle"
|
|
name="__primary"
|
|
id="__primary"/>
|
|
<label class="overlay overlay-primary" for="__primary"></label>
|
|
|
|
<input type="checkbox"
|
|
class="sidebar-toggle"
|
|
name="__secondary"
|
|
id="__secondary"/>
|
|
<label class="overlay overlay-secondary" for="__secondary"></label>
|
|
|
|
<div class="search-button__wrapper">
|
|
<div class="search-button__overlay"></div>
|
|
<div class="search-button__search-container">
|
|
<form class="bd-search d-flex align-items-center"
|
|
action="search.html"
|
|
method="get">
|
|
<i class="fa-solid fa-magnifying-glass"></i>
|
|
<input type="search"
|
|
class="form-control"
|
|
name="q"
|
|
id="search-input"
|
|
placeholder="Search this book..."
|
|
aria-label="Search this book..."
|
|
autocomplete="off"
|
|
autocorrect="off"
|
|
autocapitalize="off"
|
|
spellcheck="false"/>
|
|
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
|
</form></div>
|
|
</div>
|
|
|
|
<nav class="bd-header navbar navbar-expand-lg bd-navbar">
|
|
</nav>
|
|
|
|
<div class="bd-container">
|
|
<div class="bd-container__inner bd-page-width">
|
|
|
|
<div class="bd-sidebar-primary bd-sidebar">
|
|
|
|
|
|
|
|
<div class="sidebar-header-items sidebar-primary__section">
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
<div class="sidebar-primary-items__start sidebar-primary__section">
|
|
<div class="sidebar-primary-item">
|
|
|
|
|
|
<a class="navbar-brand logo" href="intro.html">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<img src="_static/logo.png" class="logo__image only-light" alt="Logo image"/>
|
|
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Logo image"/>`);</script>
|
|
|
|
|
|
</a></div>
|
|
<div class="sidebar-primary-item"><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
|
<div class="bd-toc-item navbar-nav active">
|
|
|
|
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
|
<li class="toctree-l1">
|
|
<a class="reference internal" href="intro.html">
|
|
Applied Data Analysis and Machine Learning
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Teaching schedule with links to material</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
|
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
|
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
|
<ul class="current nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
|
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 35</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Statistical interpretation of Linear Regression and Resampling techniques</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week38.html">Week 38: Logistic Regression and Optimization</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 9 (midnight), 2023</a></li>
|
|
</ul>
|
|
|
|
</div>
|
|
</nav></div>
|
|
</div>
|
|
|
|
|
|
<div class="sidebar-primary-items__end sidebar-primary__section">
|
|
</div>
|
|
|
|
<div id="rtd-footer-container"></div>
|
|
|
|
|
|
</div>
|
|
|
|
<main id="main-content" class="bd-main">
|
|
|
|
|
|
|
|
<div class="sbt-scroll-pixel-helper"></div>
|
|
|
|
<div class="bd-content">
|
|
<div class="bd-article-container">
|
|
|
|
<div class="bd-header-article">
|
|
<div class="header-article-items header-article__inner">
|
|
|
|
<div class="header-article-items__start">
|
|
|
|
<div class="header-article-item"><label class="sidebar-toggle primary-toggle btn btn-sm" for="__primary" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<span class="fa-solid fa-bars"></span>
|
|
</label></div>
|
|
|
|
</div>
|
|
|
|
|
|
<div class="header-article-items__end">
|
|
|
|
<div class="header-article-item">
|
|
|
|
<div class="article-header-buttons">
|
|
|
|
|
|
|
|
|
|
|
|
<div class="dropdown dropdown-launch-buttons">
|
|
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Launch interactive content">
|
|
<i class="fas fa-rocket"></i>
|
|
</button>
|
|
<ul class="dropdown-menu">
|
|
|
|
|
|
|
|
<li><a href="https://mybinder.org/v2/git/https%3A//compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/index.html/master?urlpath=tree/exercisesweek35.ipynb" target="_blank"
|
|
class="btn btn-sm dropdown-item"
|
|
title="Launch onBinder"
|
|
data-bs-placement="left" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
|
|
<img src="_static/images/logo_binder.svg">
|
|
</span>
|
|
<span class="btn__text-container">Binder</span>
|
|
</a>
|
|
</li>
|
|
|
|
</ul>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="dropdown dropdown-download-buttons">
|
|
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
|
<i class="fas fa-download"></i>
|
|
</button>
|
|
<ul class="dropdown-menu">
|
|
|
|
|
|
|
|
<li><a href="_sources/exercisesweek35.ipynb" target="_blank"
|
|
class="btn btn-sm btn-download-source-button dropdown-item"
|
|
title="Download source file"
|
|
data-bs-placement="left" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
<i class="fas fa-file"></i>
|
|
</span>
|
|
<span class="btn__text-container">.ipynb</span>
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
<li>
|
|
<button onclick="window.print()"
|
|
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
|
title="Print to PDF"
|
|
data-bs-placement="left" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
<i class="fas fa-file-pdf"></i>
|
|
</span>
|
|
<span class="btn__text-container">.pdf</span>
|
|
</button>
|
|
</li>
|
|
|
|
</ul>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<button onclick="toggleFullScreen()"
|
|
class="btn btn-sm btn-fullscreen-button"
|
|
title="Fullscreen mode"
|
|
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
<i class="fas fa-expand"></i>
|
|
</span>
|
|
|
|
</button>
|
|
|
|
|
|
<script>
|
|
document.write(`
|
|
<button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
|
|
<span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
|
|
<span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
|
|
</button>
|
|
`);
|
|
</script>
|
|
|
|
<script>
|
|
document.write(`
|
|
<button class="btn btn-sm navbar-btn search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
|
</button>
|
|
`);
|
|
</script>
|
|
<label class="sidebar-toggle secondary-toggle btn btn-sm" for="__secondary"title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<span class="fa-solid fa-list"></span>
|
|
</label>
|
|
</div></div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div id="jb-print-docs-body" class="onlyprint">
|
|
<h1>Exercises week 35</h1>
|
|
<!-- Table of contents -->
|
|
<div id="print-main-content">
|
|
<div id="jb-print-toc">
|
|
|
|
<div>
|
|
<h2> Contents </h2>
|
|
</div>
|
|
<nav aria-label="Page">
|
|
<ul class="visible nav section-nav flex-column">
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-analytical-exercises">Exercise 1: Analytical exercises</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-making-your-own-data-and-exploring-scikit-learn">Exercise 2: making your own data and exploring scikit-learn</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-split-data-in-test-and-training-data">Exercise 3: Split data in test and training data</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div id="searchbox"></div>
|
|
<article class="bd-article" role="main">
|
|
|
|
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
|
doconce format html exercisesweek35.do.txt -->
|
|
<!-- dom:TITLE: Exercises week 35 --><section class="tex2jax_ignore mathjax_ignore" id="exercises-week-35">
|
|
<h1>Exercises week 35<a class="headerlink" href="#exercises-week-35" title="Permalink to this heading">#</a></h1>
|
|
<p><strong>August 28-September 1, 2023</strong></p>
|
|
<p>Date: <strong>Deadline is Friday September 1 at midnight</strong></p>
|
|
<section id="exercise-1-analytical-exercises">
|
|
<h2>Exercise 1: Analytical exercises<a class="headerlink" href="#exercise-1-analytical-exercises" title="Permalink to this heading">#</a></h2>
|
|
<p>In this exercise we derive the expressions for various derivatives of
|
|
products of vectors and matrices. Such derivatives are central to the
|
|
optimization of various cost functions. Although we will often use
|
|
automatic differentiation in actual calculations, to be able to have
|
|
analytical expressions is extremely helpful in case we have simpler
|
|
derivatives as well as when we analyze various properties (like second
|
|
derivatives) of the chosen cost functions. Vectors are always written
|
|
as boldfaced lower case letters and matrices as upper case boldfaced
|
|
letters. You will find useful the notes from week 35 on derivatives of vectors and matrices.</p>
|
|
<p>Show that</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial (\boldsymbol{b}^T\boldsymbol{a})}{\partial \boldsymbol{a}} = \boldsymbol{b},
|
|
\]</div>
|
|
<p>and</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial (\boldsymbol{a}^T\boldsymbol{A}\boldsymbol{a})}{\partial \boldsymbol{a}} = \boldsymbol{a}^T(\boldsymbol{A}+\boldsymbol{A}^T),
|
|
\]</div>
|
|
<p>and</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial \left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)^T\left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)}{\partial \boldsymbol{s}} = -2\left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)^T\boldsymbol{A},
|
|
\]</div>
|
|
<p>and finally find the second derivative of this function with respect to the vector <span class="math notranslate nohighlight">\(\boldsymbol{s}\)</span>. If we replace the vector <span class="math notranslate nohighlight">\(\boldsymbol{s}\)</span> with the unknown parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> used to define the ordinary least squares method, we end up with the equations that determine these parameters. The matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is then the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> here has to be replaced with the outputs <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span>.</p>
|
|
<p>The second derivative of the mean squared error is then proportional to the so-called Hessian matrix <span class="math notranslate nohighlight">\(\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}\)</span>.</p>
|
|
<p><strong>Hint</strong>: In these exercises it is always useful to write out with summation indices the various quantities.
|
|
As an example, consider the function</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
f(\boldsymbol{x}) =\boldsymbol{A}\boldsymbol{x},
|
|
\]</div>
|
|
<p>which reads for a specific component <span class="math notranslate nohighlight">\(f_i\)</span> (we define the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> to have dimension <span class="math notranslate nohighlight">\(n\times n\)</span> and the vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> to have length <span class="math notranslate nohighlight">\(n\)</span>)</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
f_i =\sum_{j=0}^{n-1}a_{ij}x_j,
|
|
\]</div>
|
|
<p>which leads to</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial f_i}{\partial x_j}= a_{ij},
|
|
\]</div>
|
|
<p>and written out in terms of the vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> we have</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial f(\boldsymbol{x})}{\partial \boldsymbol{x}}= \boldsymbol{A}.
|
|
\]</div>
|
|
</section>
|
|
<section id="exercise-2-making-your-own-data-and-exploring-scikit-learn">
|
|
<h2>Exercise 2: making your own data and exploring scikit-learn<a class="headerlink" href="#exercise-2-making-your-own-data-and-exploring-scikit-learn" title="Permalink to this heading">#</a></h2>
|
|
<p>We will generate our own dataset for a function <span class="math notranslate nohighlight">\(y(x)\)</span> where <span class="math notranslate nohighlight">\(x \in
|
|
[0,1]\)</span> and defined by random numbers computed with the uniform
|
|
distribution. The function <span class="math notranslate nohighlight">\(y\)</span> is a quadratic polynomial in <span class="math notranslate nohighlight">\(x\)</span> with
|
|
added stochastic noise according to the normal distribution <span class="math notranslate nohighlight">\(\cal
|
|
{N}(0,1)\)</span>. The following simple Python instructions define our <span class="math notranslate nohighlight">\(x\)</span>
|
|
and <span class="math notranslate nohighlight">\(y\)</span> values (with 100 data points).</p>
|
|
<div class="cell docutils container">
|
|
<div class="cell_input docutils container">
|
|
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
|
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
|
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">5</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mf">0.1</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
|
</pre></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<ol class="arabic simple">
|
|
<li><p>Write your own code (following the examples under the <a class="reference external" href="https://compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/chapter1.html">regression notes</a>) for computing the parametrization of the data set fitting a second-order polynomial.</p></li>
|
|
<li><p>Use thereafter <strong>scikit-learn</strong> (see again the examples in the slides for week 35) and compare with your own code. Note here that <strong>scikit-learn</strong> does not include, by default, the intercept. See the discussions on scaling your data in the slides for this week. This type of problems appear in particular if we fit a polynomial with an intercept.</p></li>
|
|
<li><p>Using scikit-learn, compute also the mean squared error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as</p></li>
|
|
</ol>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
MSE(\boldsymbol{y},\boldsymbol{\tilde{y}}) = \frac{1}{n}
|
|
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
|
\]</div>
|
|
<p>and the <span class="math notranslate nohighlight">\(R^2\)</span> score function.
|
|
If <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{y}}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i-th\)</span> sample and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the score <span class="math notranslate nohighlight">\(R^2\)</span> is defined as</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
R^2(\boldsymbol{y}, \tilde{\boldsymbol{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
|
\]</div>
|
|
<p>where we have defined the mean value of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> as</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
|
\]</div>
|
|
<p>You can use the functionality included in scikit-learn. If you feel for it, you can use your own program and define functions which compute the above two functions.
|
|
Discuss the meaning of these results. Try also to vary the coefficient in front of the added stochastic noise term and discuss the quality of the fits.</p>
|
|
</section>
|
|
<section id="exercise-3-split-data-in-test-and-training-data">
|
|
<h2>Exercise 3: Split data in test and training data<a class="headerlink" href="#exercise-3-split-data-in-test-and-training-data" title="Permalink to this heading">#</a></h2>
|
|
<p>In this exercise we want you to to compute the MSE for the training
|
|
data and the test data as function of the complexity of a polynomial,
|
|
that is the degree of a given polynomial.</p>
|
|
<p>The aim is to reproduce Figure 2.11 of <a class="reference external" href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf">Hastie et al</a>.
|
|
Feel free to read the discussions leading to figure 2.11 of Hastie et al.</p>
|
|
<p>Our data is defined by <span class="math notranslate nohighlight">\(x\in [-3,3]\)</span> with a total of for example <span class="math notranslate nohighlight">\(n=100\)</span> data points. You should try to vary the number of data points <span class="math notranslate nohighlight">\(n\)</span> in your analysis.</p>
|
|
<div class="cell docutils container">
|
|
<div class="cell_input docutils container">
|
|
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">()</span>
|
|
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
|
<span class="c1"># Make data set.</span>
|
|
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
|
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
|
</pre></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<p>where <span class="math notranslate nohighlight">\(y\)</span> is the function we want to fit with a given polynomial.</p>
|
|
<p><strong>a)</strong>
|
|
Write a first code which sets up a design matrix <span class="math notranslate nohighlight">\(X\)</span> defined by a fifth-order polynomial and split your data set in training and test data.</p>
|
|
<p><strong>b)</strong>
|
|
Write thereafter (using either <strong>scikit-learn</strong> or your matrix inversion code using for example <strong>numpy</strong>)
|
|
and perform an ordinary least squares fitting and compute the mean squared error for the training data and the test data. These calculations should apply to a model given by a fifth-order polynomial. If you compare your own code with <em>scikit_learn</em>, not that the latter does not include by default the intercept. See the discussions on scaling your data in the slides for this week.</p>
|
|
<p><strong>c)</strong>
|
|
Add now a model which allows you to make polynomials up to degree <span class="math notranslate nohighlight">\(15\)</span>. Perform a standard OLS fitting of the training data and compute the MSE for the training and test data and plot both test and training data MSE as functions of the polynomial degree. Compare what you see with Figure 2.11 of Hastie et al. Comment your results. For which polynomial degree do you find an optimal MSE (smallest value)?</p>
|
|
</section>
|
|
</section>
|
|
|
|
<script type="text/x-thebe-config">
|
|
{
|
|
requestKernel: true,
|
|
binderOptions: {
|
|
repo: "binder-examples/jupyter-stacks-datascience",
|
|
ref: "master",
|
|
},
|
|
codeMirrorConfig: {
|
|
theme: "abcdef",
|
|
mode: "python"
|
|
},
|
|
kernelOptions: {
|
|
name: "python3",
|
|
path: "./."
|
|
},
|
|
predefinedOutput: true
|
|
}
|
|
</script>
|
|
<script>kernelName = 'python3'</script>
|
|
|
|
</article>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<footer class="prev-next-footer">
|
|
<!-- Previous / next buttons -->
|
|
<div class="prev-next-area">
|
|
<a class="left-prev"
|
|
href="week34.html"
|
|
title="previous page">
|
|
<i class="fa-solid fa-angle-left"></i>
|
|
<div class="prev-next-info">
|
|
<p class="prev-next-subtitle">previous</p>
|
|
<p class="prev-next-title">Week 34: Introduction to the course, Logistics and Practicalities</p>
|
|
</div>
|
|
</a>
|
|
<a class="right-next"
|
|
href="week35.html"
|
|
title="next page">
|
|
<div class="prev-next-info">
|
|
<p class="prev-next-subtitle">next</p>
|
|
<p class="prev-next-title">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</p>
|
|
</div>
|
|
<i class="fa-solid fa-angle-right"></i>
|
|
</a>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
|
|
|
<div class="sidebar-secondary-item">
|
|
<div class="page-toc tocsection onthispage">
|
|
<i class="fa-solid fa-list"></i> Contents
|
|
</div>
|
|
<nav class="bd-toc-nav page-toc">
|
|
<ul class="visible nav section-nav flex-column">
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-analytical-exercises">Exercise 1: Analytical exercises</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-making-your-own-data-and-exploring-scikit-learn">Exercise 2: making your own data and exploring scikit-learn</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-split-data-in-test-and-training-data">Exercise 3: Split data in test and training data</a></li>
|
|
</ul>
|
|
</nav></div>
|
|
|
|
</div></div>
|
|
|
|
|
|
</div>
|
|
<footer class="bd-footer-content">
|
|
|
|
<div class="bd-footer-content__inner container">
|
|
|
|
<div class="footer-item">
|
|
|
|
<p class="component-author">
|
|
By Morten Hjorth-Jensen
|
|
</p>
|
|
|
|
</div>
|
|
|
|
<div class="footer-item">
|
|
|
|
<p class="copyright">
|
|
|
|
© Copyright 2022.
|
|
<br/>
|
|
|
|
</p>
|
|
|
|
</div>
|
|
|
|
<div class="footer-item">
|
|
|
|
</div>
|
|
|
|
<div class="footer-item">
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</footer>
|
|
|
|
|
|
</main>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
|
<script src="_static/scripts/bootstrap.js?digest=ac02cc09edc035673794"></script>
|
|
<script src="_static/scripts/pydata-sphinx-theme.js?digest=ac02cc09edc035673794"></script>
|
|
|
|
<footer class="bd-footer">
|
|
</footer>
|
|
</body>
|
|
</html> |