Morten Hjorth-Jensen
702 lines
36 KiB
HTML
702 lines
36 KiB
HTML
|
|
<!DOCTYPE html>
|
|
|
|
|
|
<html lang="en" data-content_root="./" >
|
|
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
|
|
<title>Exercises week 35 — Applied Data Analysis and Machine Learning</title>
|
|
|
|
|
|
|
|
<script data-cfasync="false">
|
|
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
|
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
|
</script>
|
|
<!--
|
|
this give us a css class that will be invisible only if js is disabled
|
|
-->
|
|
<noscript>
|
|
<style>
|
|
.pst-js-only { display: none !important; }
|
|
|
|
</style>
|
|
</noscript>
|
|
|
|
<!-- Loaded before other Sphinx assets -->
|
|
<link href="_static/styles/theme.css?digest=26a4bc78f4c0ddb94549" rel="stylesheet" />
|
|
<link href="_static/styles/pydata-sphinx-theme.css?digest=26a4bc78f4c0ddb94549" rel="stylesheet" />
|
|
|
|
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
|
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=a3416100" />
|
|
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
|
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
|
<link rel="stylesheet" type="text/css" href="_static/mystnb.4510f1fc1dee50b3e5859aac5469c37c29e427902b24a333a5f9fcb2f0b3ac41.css?v=be8a1c11" />
|
|
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
|
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
|
|
|
<!-- So that users can add custom icons -->
|
|
<script src="_static/scripts/fontawesome.js?digest=26a4bc78f4c0ddb94549"></script>
|
|
<!-- Pre-loaded scripts that we'll load fully later -->
|
|
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549" />
|
|
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549" />
|
|
|
|
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
|
<script src="_static/doctools.js?v=9a2dae69"></script>
|
|
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
|
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
|
<script src="_static/copybutton.js?v=f281be69"></script>
|
|
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
|
<script>let toggleHintShow = 'Click to show';</script>
|
|
<script>let toggleHintHide = 'Click to hide';</script>
|
|
<script>let toggleOpenOnPrint = 'true';</script>
|
|
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
|
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
|
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
|
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
|
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
|
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
|
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
|
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
|
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
|
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek35';</script>
|
|
<link rel="index" title="Index" href="genindex.html" />
|
|
<link rel="search" title="Search" href="search.html" />
|
|
<link rel="next" title="Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression" href="week35.html" />
|
|
<link rel="prev" title="Week 34: Introduction to the course, Logistics and Practicalities" href="week34.html" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|
<meta name="docsearch:language" content="en"/>
|
|
<meta name="docsearch:version" content="" />
|
|
</head>
|
|
|
|
|
|
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
|
|
|
|
|
|
|
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
|
|
|
<div id="pst-scroll-pixel-helper"></div>
|
|
|
|
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
|
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
|
|
|
|
|
<dialog id="pst-search-dialog">
|
|
|
|
<form class="bd-search d-flex align-items-center"
|
|
action="search.html"
|
|
method="get">
|
|
<i class="fa-solid fa-magnifying-glass"></i>
|
|
<input type="search"
|
|
class="form-control"
|
|
name="q"
|
|
placeholder="Search this book..."
|
|
aria-label="Search this book..."
|
|
autocomplete="off"
|
|
autocorrect="off"
|
|
autocapitalize="off"
|
|
spellcheck="false"/>
|
|
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
|
</form>
|
|
</dialog>
|
|
|
|
<div class="pst-async-banner-revealer d-none">
|
|
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
|
</div>
|
|
|
|
|
|
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
|
</header>
|
|
|
|
|
|
<div class="bd-container">
|
|
<div class="bd-container__inner bd-page-width">
|
|
|
|
|
|
|
|
<dialog id="pst-primary-sidebar-modal"></dialog>
|
|
<div id="pst-primary-sidebar" class="bd-sidebar-primary bd-sidebar">
|
|
|
|
|
|
|
|
<div class="sidebar-header-items sidebar-primary__section">
|
|
|
|
|
|
|
|
|
|
</div>
|
|
|
|
<div class="sidebar-primary-items__start sidebar-primary__section">
|
|
<div class="sidebar-primary-item">
|
|
|
|
|
|
|
|
|
|
|
|
<a class="navbar-brand logo" href="intro.html">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
|
<img src="_static/logo.png" class="logo__image only-dark pst-js-only" alt="Applied Data Analysis and Machine Learning - Home"/>
|
|
|
|
|
|
</a></div>
|
|
<div class="sidebar-primary-item">
|
|
|
|
<button class="btn search-button-field search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<i class="fa-solid fa-magnifying-glass"></i>
|
|
<span class="search-button__default-text">Search</span>
|
|
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
|
</button></div>
|
|
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
|
<div class="bd-toc-item navbar-nav active">
|
|
|
|
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
|
<li class="toctree-l1">
|
|
<a class="reference internal" href="intro.html">
|
|
Applied Data Analysis and Machine Learning
|
|
</a>
|
|
</li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Teaching schedule with links to material</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
|
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
|
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
|
<ul class="current nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
|
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 35</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Statistical interpretations</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Statistical interpretations and Resampling Methods</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week38.html">Week 38: Logistic Regression and Optimization</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek39.html">Exercises week 39</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week39.html">Week 39: Optimization and Gradient Methods</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week40.html">Week 40: Gradient descent methods (continued) and start Neural networks</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek41.html">Exercises week 41</a></li>
|
|
|
|
|
|
<li class="toctree-l1"><a class="reference internal" href="week41.html">Week 41 Neural networks and constructing a neural network code</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek42.html">Exercises week 42</a></li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="toctree-l1"><a class="reference internal" href="week42.html">Week 42 Constructing a Neural Network code with examples</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="additionweek42.html">Exercises Week 42: Logistic Regression and Optimization, reminders from week 38 and week 40</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week43.html">Week 43: Deep Learning: Constructing a Neural Network code and solving differential equations</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek43.html">Exercises week 43</a></li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="toctree-l1"><a class="reference internal" href="week44.html">Week 44, Convolutional Neural Networks (CNN)</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week45.html">Week 45, Convolutional Neural Networks (CCNs) and Recurrent Neural Networks (RNNs)</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week46.html">Week 46: Decision Trees, Ensemble methods and Random Forests</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="week47.html">Week 47: From Decision Trees to Ensemble Methods, Random Forests and Boosting Methods</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="exercisesweek47.html">Exercise week 47</a></li>
|
|
|
|
</ul>
|
|
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
|
<ul class="nav bd-sidenav">
|
|
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 7 (midnight), 2024</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="project2.html">Project 2 on Machine Learning, deadline November 4 (Midnight)</a></li>
|
|
<li class="toctree-l1"><a class="reference internal" href="project3.html">Project 3 on Machine Learning, deadline December 9 (midnight), 2024</a></li>
|
|
|
|
</ul>
|
|
|
|
</div>
|
|
</nav></div>
|
|
</div>
|
|
|
|
|
|
<div class="sidebar-primary-items__end sidebar-primary__section">
|
|
</div>
|
|
|
|
<div id="rtd-footer-container"></div>
|
|
|
|
|
|
</div>
|
|
|
|
<main id="main-content" class="bd-main" role="main">
|
|
|
|
|
|
|
|
<div class="sbt-scroll-pixel-helper"></div>
|
|
|
|
<div class="bd-content">
|
|
<div class="bd-article-container">
|
|
|
|
<div class="bd-header-article d-print-none">
|
|
<div class="header-article-items header-article__inner">
|
|
|
|
<div class="header-article-items__start">
|
|
|
|
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<span class="fa-solid fa-bars"></span>
|
|
</button></div>
|
|
|
|
</div>
|
|
|
|
|
|
<div class="header-article-items__end">
|
|
|
|
<div class="header-article-item">
|
|
|
|
<div class="article-header-buttons">
|
|
|
|
|
|
|
|
|
|
|
|
<div class="dropdown dropdown-launch-buttons">
|
|
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Launch interactive content">
|
|
<i class="fas fa-rocket"></i>
|
|
</button>
|
|
<ul class="dropdown-menu">
|
|
|
|
|
|
|
|
<li><a href="https://mybinder.org/v2/git/https%3A//compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/index.html/master?urlpath=tree/exercisesweek35.ipynb" target="_blank"
|
|
class="btn btn-sm dropdown-item"
|
|
title="Launch on Binder"
|
|
data-bs-placement="left" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
|
|
<img alt="Binder logo" src="_static/images/logo_binder.svg">
|
|
</span>
|
|
<span class="btn__text-container">Binder</span>
|
|
</a>
|
|
</li>
|
|
|
|
</ul>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<div class="dropdown dropdown-download-buttons">
|
|
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
|
<i class="fas fa-download"></i>
|
|
</button>
|
|
<ul class="dropdown-menu">
|
|
|
|
|
|
|
|
<li><a href="_sources/exercisesweek35.ipynb" target="_blank"
|
|
class="btn btn-sm btn-download-source-button dropdown-item"
|
|
title="Download source file"
|
|
data-bs-placement="left" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
<i class="fas fa-file"></i>
|
|
</span>
|
|
<span class="btn__text-container">.ipynb</span>
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
<li>
|
|
<button onclick="window.print()"
|
|
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
|
title="Print to PDF"
|
|
data-bs-placement="left" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
<i class="fas fa-file-pdf"></i>
|
|
</span>
|
|
<span class="btn__text-container">.pdf</span>
|
|
</button>
|
|
</li>
|
|
|
|
</ul>
|
|
</div>
|
|
|
|
|
|
|
|
|
|
<button onclick="toggleFullScreen()"
|
|
class="btn btn-sm btn-fullscreen-button"
|
|
title="Fullscreen mode"
|
|
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
|
>
|
|
|
|
|
|
<span class="btn__icon-container">
|
|
<i class="fas fa-expand"></i>
|
|
</span>
|
|
|
|
</button>
|
|
|
|
|
|
|
|
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button pst-js-only" aria-label="Color mode" data-bs-title="Color mode" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light" title="Light"></i>
|
|
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark" title="Dark"></i>
|
|
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto" title="System Settings"></i>
|
|
</button>
|
|
|
|
|
|
<button class="btn btn-sm pst-navbar-icon search-button search-button__button pst-js-only" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
|
</button>
|
|
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
|
<span class="fa-solid fa-list"></span>
|
|
</button>
|
|
</div></div>
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div id="jb-print-docs-body" class="onlyprint">
|
|
<h1>Exercises week 35</h1>
|
|
<!-- Table of contents -->
|
|
<div id="print-main-content">
|
|
<div id="jb-print-toc">
|
|
|
|
<div>
|
|
<h2> Contents </h2>
|
|
</div>
|
|
<nav aria-label="Page">
|
|
<ul class="visible nav section-nav flex-column">
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-analytical-exercises">Exercise 1: Analytical exercises</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-making-your-own-data-and-exploring-scikit-learn">Exercise 2: making your own data and exploring scikit-learn</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-split-data-in-test-and-training-data">Exercise 3: Split data in test and training data</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
|
|
<div id="searchbox"></div>
|
|
<article class="bd-article">
|
|
|
|
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
|
doconce format html exercisesweek35.do.txt -->
|
|
<!-- dom:TITLE: Exercises week 35 --><section class="tex2jax_ignore mathjax_ignore" id="exercises-week-35">
|
|
<h1>Exercises week 35<a class="headerlink" href="#exercises-week-35" title="Link to this heading">#</a></h1>
|
|
<p><strong>August 26-30, 2024</strong></p>
|
|
<p>Date: <strong>Deadline is Friday August 30 at midnight</strong></p>
|
|
<section id="exercise-1-analytical-exercises">
|
|
<h2>Exercise 1: Analytical exercises<a class="headerlink" href="#exercise-1-analytical-exercises" title="Link to this heading">#</a></h2>
|
|
<p>In this exercise we derive the expressions for various derivatives of
|
|
products of vectors and matrices. Such derivatives are central to the
|
|
optimization of various cost functions. Although we will often use
|
|
automatic differentiation in actual calculations, to be able to have
|
|
analytical expressions is extremely helpful in case we have simpler
|
|
derivatives as well as when we analyze various properties (like second
|
|
derivatives) of the chosen cost functions. Vectors are always written
|
|
as boldfaced lower case letters and matrices as upper case boldfaced
|
|
letters. You will find useful the notes from week 35 on derivatives of vectors and matrices.
|
|
See also the textbook of Faisal at al, chapter 5 and in particular sections 5.3-5.5 at <a class="github reference external" href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/MathMLbook.pdf">CompPhysics/MachineLearning</a></p>
|
|
<p>Show that</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial (\boldsymbol{a}^T\boldsymbol{x})}{\partial \boldsymbol{x}} = \boldsymbol{a}^T,
|
|
\]</div>
|
|
<p>and</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial (\boldsymbol{a}^T\boldsymbol{A}\boldsymbol{a})}{\partial \boldsymbol{a}} = \boldsymbol{a}^T(\boldsymbol{A}+\boldsymbol{A}^T),
|
|
\]</div>
|
|
<p>and</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial \left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)^T\left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)}{\partial \boldsymbol{s}} = -2\left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)^T\boldsymbol{A},
|
|
\]</div>
|
|
<p>and finally find the second derivative of this function with respect to the vector <span class="math notranslate nohighlight">\(\boldsymbol{s}\)</span>. If we replace the vector <span class="math notranslate nohighlight">\(\boldsymbol{s}\)</span> with the unknown parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> used to define the ordinary least squares method, we end up with the equations that determine these parameters. The matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is then the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> here has to be replaced with the outputs <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span>.</p>
|
|
<p>The second derivative of the mean squared error is then proportional to the so-called Hessian matrix <span class="math notranslate nohighlight">\(\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}\)</span>.</p>
|
|
<p><strong>Hint</strong>: In these exercises it is always useful to write out with summation indices the various quantities. Take also a look at the weekly slides from week 35 and the various examples included there.</p>
|
|
<p>As an example, consider the function</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
f(\boldsymbol{x}) =\boldsymbol{A}\boldsymbol{x},
|
|
\]</div>
|
|
<p>which reads for a specific component <span class="math notranslate nohighlight">\(f_i\)</span> (we define the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> to have dimension <span class="math notranslate nohighlight">\(n\times n\)</span> and the vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> to have length <span class="math notranslate nohighlight">\(n\)</span>)</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
f_i =\sum_{j=0}^{n-1}a_{ij}x_j,
|
|
\]</div>
|
|
<p>which leads to</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial f_i}{\partial x_j}= a_{ij},
|
|
\]</div>
|
|
<p>and written out in terms of the vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> we have</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\frac{\partial f(\boldsymbol{x})}{\partial \boldsymbol{x}}= \boldsymbol{A}.
|
|
\]</div>
|
|
</section>
|
|
<section id="exercise-2-making-your-own-data-and-exploring-scikit-learn">
|
|
<h2>Exercise 2: making your own data and exploring scikit-learn<a class="headerlink" href="#exercise-2-making-your-own-data-and-exploring-scikit-learn" title="Link to this heading">#</a></h2>
|
|
<p>We will generate our own dataset for a function <span class="math notranslate nohighlight">\(y(x)\)</span> where <span class="math notranslate nohighlight">\(x \in
|
|
[0,1]\)</span> and defined by random numbers computed with the uniform
|
|
distribution. The function <span class="math notranslate nohighlight">\(y\)</span> is a quadratic polynomial in <span class="math notranslate nohighlight">\(x\)</span> with
|
|
added stochastic noise according to the normal distribution <span class="math notranslate nohighlight">\(\cal
|
|
{N}(0,1)\)</span>. The following simple Python instructions define our <span class="math notranslate nohighlight">\(x\)</span>
|
|
and <span class="math notranslate nohighlight">\(y\)</span> values (with 100 data points).</p>
|
|
<div class="cell docutils container">
|
|
<div class="cell_input docutils container">
|
|
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
|
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
|
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">5</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mf">0.1</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
|
|
</pre></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<ol class="arabic simple">
|
|
<li><p>Write your own code (following the examples under the <a class="reference external" href="https://compphysics.github.io/MachineLearning/doc/LectureNotes/_build/html/chapter1.html">regression notes</a>) for computing the parametrization of the data set fitting a second-order polynomial.</p></li>
|
|
<li><p>Use thereafter <strong>scikit-learn</strong> (see again the examples in the slides for week 35) and compare with your own code. Note here that <strong>scikit-learn</strong> does not include, by default, the intercept. See the discussions on scaling your data in the slides for this week. This type of problems appear in particular if we fit a polynomial with an intercept.</p></li>
|
|
<li><p>Using scikit-learn, compute also the mean squared error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as</p></li>
|
|
</ol>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
MSE(\boldsymbol{y},\boldsymbol{\tilde{y}}) = \frac{1}{n}
|
|
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
|
|
\]</div>
|
|
<p>and the <span class="math notranslate nohighlight">\(R^2\)</span> score function.
|
|
If <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{y}}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i-th\)</span> sample and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the score <span class="math notranslate nohighlight">\(R^2\)</span> is defined as</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
R^2(\boldsymbol{y}, \tilde{\boldsymbol{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
|
|
\]</div>
|
|
<p>where we have defined the mean value of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> as</p>
|
|
<div class="math notranslate nohighlight">
|
|
\[
|
|
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
|
|
\]</div>
|
|
<p>You can use the functionality included in scikit-learn. If you feel for it, you can use your own program and define functions which compute the above two functions.
|
|
Discuss the meaning of these results. Try also to vary the coefficient in front of the added stochastic noise term and discuss the quality of the fits.</p>
|
|
</section>
|
|
<section id="exercise-3-split-data-in-test-and-training-data">
|
|
<h2>Exercise 3: Split data in test and training data<a class="headerlink" href="#exercise-3-split-data-in-test-and-training-data" title="Link to this heading">#</a></h2>
|
|
<p>In this exercise we want you to to compute the MSE for the training
|
|
data and the test data as function of the complexity of a polynomial,
|
|
that is the degree of a given polynomial.</p>
|
|
<p>The aim is to reproduce Figure 2.11 of <a class="reference external" href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf">Hastie et al</a>.
|
|
Feel free to read the discussions leading to figure 2.11 of Hastie et al.</p>
|
|
<p>Our data is defined by <span class="math notranslate nohighlight">\(x\in [-3,3]\)</span> with a total of for example <span class="math notranslate nohighlight">\(n=100\)</span> data points. You should try to vary the number of data points <span class="math notranslate nohighlight">\(n\)</span> in your analysis.</p>
|
|
<div class="cell docutils container">
|
|
<div class="cell_input docutils container">
|
|
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">()</span>
|
|
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
|
<span class="c1"># Make data set.</span>
|
|
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
|
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
|
</pre></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<p>where <span class="math notranslate nohighlight">\(y\)</span> is the function we want to fit with a given polynomial.</p>
|
|
<p><strong>a)</strong>
|
|
Write a first code which sets up a design matrix <span class="math notranslate nohighlight">\(X\)</span> defined by a fifth-order polynomial and split your data set in training and test data.</p>
|
|
<p><strong>b)</strong>
|
|
Write thereafter (using either <strong>scikit-learn</strong> or your matrix inversion code using for example <strong>numpy</strong>)
|
|
and perform an ordinary least squares fitting and compute the mean squared error for the training data and the test data. These calculations should apply to a model given by a fifth-order polynomial. If you compare your own code with <em>scikit_learn</em>, not that the latter does not include by default the intercept. See the discussions on scaling your data in the slides for this week.</p>
|
|
<p><strong>c)</strong>
|
|
Add now a model which allows you to make polynomials up to degree <span class="math notranslate nohighlight">\(15\)</span>. Perform a standard OLS fitting of the training data and compute the MSE for the training and test data and plot both test and training data MSE as functions of the polynomial degree. Compare what you see with Figure 2.11 of Hastie et al. Comment your results. For which polynomial degree do you find an optimal MSE (smallest value)?</p>
|
|
</section>
|
|
</section>
|
|
|
|
<script type="text/x-thebe-config">
|
|
{
|
|
requestKernel: true,
|
|
binderOptions: {
|
|
repo: "binder-examples/jupyter-stacks-datascience",
|
|
ref: "master",
|
|
},
|
|
codeMirrorConfig: {
|
|
theme: "abcdef",
|
|
mode: "python"
|
|
},
|
|
kernelOptions: {
|
|
name: "python3",
|
|
path: "./."
|
|
},
|
|
predefinedOutput: true
|
|
}
|
|
</script>
|
|
<script>kernelName = 'python3'</script>
|
|
|
|
</article>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<footer class="prev-next-footer d-print-none">
|
|
|
|
<div class="prev-next-area">
|
|
<a class="left-prev"
|
|
href="week34.html"
|
|
title="previous page">
|
|
<i class="fa-solid fa-angle-left"></i>
|
|
<div class="prev-next-info">
|
|
<p class="prev-next-subtitle">previous</p>
|
|
<p class="prev-next-title">Week 34: Introduction to the course, Logistics and Practicalities</p>
|
|
</div>
|
|
</a>
|
|
<a class="right-next"
|
|
href="week35.html"
|
|
title="next page">
|
|
<div class="prev-next-info">
|
|
<p class="prev-next-subtitle">next</p>
|
|
<p class="prev-next-title">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</p>
|
|
</div>
|
|
<i class="fa-solid fa-angle-right"></i>
|
|
</a>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
|
|
|
|
<dialog id="pst-secondary-sidebar-modal"></dialog>
|
|
<div id="pst-secondary-sidebar" class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
|
|
|
|
|
<div class="sidebar-secondary-item">
|
|
<div class="page-toc tocsection onthispage">
|
|
<i class="fa-solid fa-list"></i> Contents
|
|
</div>
|
|
<nav class="bd-toc-nav page-toc">
|
|
<ul class="visible nav section-nav flex-column">
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-analytical-exercises">Exercise 1: Analytical exercises</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-making-your-own-data-and-exploring-scikit-learn">Exercise 2: making your own data and exploring scikit-learn</a></li>
|
|
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-split-data-in-test-and-training-data">Exercise 3: Split data in test and training data</a></li>
|
|
</ul>
|
|
</nav></div>
|
|
|
|
</div></div>
|
|
|
|
|
|
</div>
|
|
<footer class="bd-footer-content">
|
|
|
|
<div class="bd-footer-content__inner container">
|
|
|
|
<div class="footer-item">
|
|
|
|
<p class="component-author">
|
|
By Morten Hjorth-Jensen
|
|
</p>
|
|
|
|
</div>
|
|
|
|
<div class="footer-item">
|
|
|
|
|
|
<p class="copyright">
|
|
|
|
© Copyright 2023.
|
|
<br/>
|
|
|
|
</p>
|
|
|
|
</div>
|
|
|
|
<div class="footer-item">
|
|
|
|
</div>
|
|
|
|
<div class="footer-item">
|
|
|
|
</div>
|
|
|
|
</div>
|
|
</footer>
|
|
|
|
|
|
</main>
|
|
</div>
|
|
</div>
|
|
|
|
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
|
<script defer src="_static/scripts/bootstrap.js?digest=26a4bc78f4c0ddb94549"></script>
|
|
<script defer src="_static/scripts/pydata-sphinx-theme.js?digest=26a4bc78f4c0ddb94549"></script>
|
|
|
|
<footer class="bd-footer">
|
|
</footer>
|
|
</body>
|
|
</html> |