Morten Hjorth-Jensen
768 lines
46 KiB
HTML
768 lines
46 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Exercises week 35 — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'exercisesweek35';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression" href="week35.html" />
|
||
<link rel="prev" title="Week 34: Introduction to the course, Logistics and Practicalities" href="week34.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Applied Data Analysis and Machine Learning - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Course setting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Exercises week 35</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Gradient descent methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 37</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/exercisesweek35.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Exercises week 35</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#deriving-and-implementing-ordinary-least-squares">Deriving and Implementing Ordinary Least Squares</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-goals">Learning goals</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#deliverables">Deliverables</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#how-to-take-derivatives-of-matrix-vector-expressions">How to take derivatives of Matrix-Vector expressions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-finding-the-derivative-of-matrix-vector-expressions">Exercise 1 - Finding the derivative of Matrix-Vector expressions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-deriving-the-expression-for-ols">Exercise 2 - Deriving the expression for OLS</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-creating-feature-matrix-and-implementing-ols-using-the-analytical-expression">Exercise 3 - Creating feature matrix and implementing OLS using the analytical expression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-fitting-a-polynomial">Exercise 4 - Fitting a polynomial</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-5-comparing-your-code-with-sklearn">Exercise 5 - Comparing your code with sklearn</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section class="tex2jax_ignore mathjax_ignore" id="exercises-week-35">
|
||
<h1>Exercises week 35<a class="headerlink" href="#exercises-week-35" title="Link to this heading">#</a></h1>
|
||
<section id="deriving-and-implementing-ordinary-least-squares">
|
||
<h2>Deriving and Implementing Ordinary Least Squares<a class="headerlink" href="#deriving-and-implementing-ordinary-least-squares" title="Link to this heading">#</a></h2>
|
||
<p>This week you will be deriving the analytical expressions for linear regression, building up the model from scratch. This will include taking several derivatives of products of vectors and matrices. Such derivatives are central to the optimization of many machine learning models. Although we will often use automatic differentiation in actual calculations, to be able to have analytical expressions is extremely helpful in case we have simpler derivatives as well as when we analyze various properties (like second derivatives) of the chosen cost functions.</p>
|
||
<p>Vectors are always written as boldfaced lower case letters and matrices as upper case boldfaced letters. You will find useful the notes from week 35 on derivatives of vectors and matrices. See also the textbook of Faisal at al, chapter 5 and in particular sections 5.3-5.5 at <a class="github reference external" href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/MathMLbook.pdf">CompPhysics/MachineLearning</a></p>
|
||
<section id="learning-goals">
|
||
<h3>Learning goals<a class="headerlink" href="#learning-goals" title="Link to this heading">#</a></h3>
|
||
<p>After completing these exercises, you will know how to</p>
|
||
<ul class="simple">
|
||
<li><p>Take the derivatives of simple products between vectors and matrices</p></li>
|
||
<li><p>Implement OLS using the analytical expressions</p></li>
|
||
<li><p>Create a feature matrix from a set of data</p></li>
|
||
<li><p>Create a feature matrix for a polynomial model</p></li>
|
||
<li><p>Evaluate the MSE score of various model on training and test data, and comparing their performance</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="deliverables">
|
||
<h3>Deliverables<a class="headerlink" href="#deliverables" title="Link to this heading">#</a></h3>
|
||
<p>Complete the following exercises while working in a jupyter notebook. Then, in canvas, include</p>
|
||
<ul class="simple">
|
||
<li><p>The jupyter notebook with the exercises completed</p></li>
|
||
<li><p>An exported PDF of the notebook (<a class="reference external" href="https://code.visualstudio.com/docs/datascience/jupyter-notebooks#_export-your-jupyter-notebook">https://code.visualstudio.com/docs/datascience/jupyter-notebooks#_export-your-jupyter-notebook</a>)</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
<section id="how-to-take-derivatives-of-matrix-vector-expressions">
|
||
<h2>How to take derivatives of Matrix-Vector expressions<a class="headerlink" href="#how-to-take-derivatives-of-matrix-vector-expressions" title="Link to this heading">#</a></h2>
|
||
<p>In these exercises it is always useful to write out with summation indices the various quantities. Take also a look at the weekly slides from week 35 and the various examples included there.</p>
|
||
<p>As an example, consider the function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f(\boldsymbol{x}) =\boldsymbol{A}\boldsymbol{x},
|
||
\]</div>
|
||
<p>which reads for a specific component <span class="math notranslate nohighlight">\(f_i\)</span> (we define the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> to have dimension <span class="math notranslate nohighlight">\(n\times n\)</span> and the vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> to have length <span class="math notranslate nohighlight">\(n\)</span>)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f_i =\sum_{j=0}^{n-1}a_{ij}x_j,
|
||
\]</div>
|
||
<p>which leads to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial f_i}{\partial x_j}= a_{ij},
|
||
\]</div>
|
||
<p>and written out in terms of the vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial f(\boldsymbol{x})}{\partial \boldsymbol{x}}= \boldsymbol{A}.
|
||
\]</div>
|
||
</section>
|
||
<section id="exercise-1-finding-the-derivative-of-matrix-vector-expressions">
|
||
<h2>Exercise 1 - Finding the derivative of Matrix-Vector expressions<a class="headerlink" href="#exercise-1-finding-the-derivative-of-matrix-vector-expressions" title="Link to this heading">#</a></h2>
|
||
<p><strong>a)</strong> Consider the expression</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial (\boldsymbol{a}^T\boldsymbol{x})}{\partial \boldsymbol{x}},
|
||
\]</div>
|
||
<p>Where <span class="math notranslate nohighlight">\(\boldsymbol{a}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> are column-vectors with length <span class="math notranslate nohighlight">\(n\)</span>.</p>
|
||
<p>What is the <em>shape</em> of the expression we are taking the derivative of?</p>
|
||
<p>What is the <em>shape</em> of the thing we are taking the derivative with respect to?</p>
|
||
<p>What is the <em>shape</em> of the result of the expression?</p>
|
||
<p><strong>b)</strong> Show that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial (\boldsymbol{a}^T\boldsymbol{x})}{\partial \boldsymbol{x}} = \boldsymbol{a}^T,
|
||
\]</div>
|
||
<p><strong>c)</strong> Show that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial (\boldsymbol{a}^T\boldsymbol{A}\boldsymbol{a})}{\partial \boldsymbol{a}} = \boldsymbol{a}^T(\boldsymbol{A}+\boldsymbol{A}^T),
|
||
\]</div>
|
||
</section>
|
||
<section id="exercise-2-deriving-the-expression-for-ols">
|
||
<h2>Exercise 2 - Deriving the expression for OLS<a class="headerlink" href="#exercise-2-deriving-the-expression-for-ols" title="Link to this heading">#</a></h2>
|
||
<p>The ordinary least squares method finds the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> which minimizes the squared error between our model <span class="math notranslate nohighlight">\(\boldsymbol{X\theta}\)</span> and the true values <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span>.</p>
|
||
<p>To find the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> which minimizes this error, we take the derivative of the squared error expression with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>, and set it equal to 0.</p>
|
||
<p><strong>a)</strong> Very briefly explain why the approach above finds the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> which minimizes this error.</p>
|
||
<p>We typically write the squared error as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert\boldsymbol{y} - \boldsymbol{X\theta}\vert\vert^2
|
||
\]</div>
|
||
<p>which we can rewrite in matrix-vector form as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)
|
||
\]</div>
|
||
<p><strong>b)</strong> If <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is invertible, what is the expression for the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>? (<strong>Hint:</strong> Don’t compute any derivatives, but solve <span class="math notranslate nohighlight">\(\boldsymbol{X\theta}=\boldsymbol{y}\)</span> for <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>)</p>
|
||
<p><strong>c)</strong> Show that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)^T\left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)}{\partial \boldsymbol{s}} = -2\left(\boldsymbol{x}-\boldsymbol{A}\boldsymbol{s}\right)^T\boldsymbol{A},
|
||
\]</div>
|
||
<p><strong>d)</strong> Using the expression from <strong>c)</strong>, but substituting back in <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>, <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>, find the expression for the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> in the case that <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is not invertible, but <span class="math notranslate nohighlight">\(\boldsymbol{X^T X}\)</span> is, which is most often the case.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\hat{\theta}_{OLS}} = ...
|
||
\]</div>
|
||
</section>
|
||
<section id="exercise-3-creating-feature-matrix-and-implementing-ols-using-the-analytical-expression">
|
||
<h2>Exercise 3 - Creating feature matrix and implementing OLS using the analytical expression<a class="headerlink" href="#exercise-3-creating-feature-matrix-and-implementing-ols-using-the-analytical-expression" title="Link to this heading">#</a></h2>
|
||
<p>With the expression for <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\theta}_{OLS}}\)</span>, you now have what you need to implement OLS regression with your input data and target data <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span>. But before you can do that, you need to set up you input data as a feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
<p>In a feature matrix, each row is a datapoint and each column is a feature of that data. If you want to predict someones spending based on their income and number of children, for instance, you would create a row for each person in your dataset, with the montly income and the number of children as columns.</p>
|
||
<p>We typically also include an intercept in our models. The intercept is a value that is added to our prediction regardless of the value of the other features. The intercept tries to account for constant effects in our data that are not dependant on anything else. In our current example, the intercept could account for living expenses which are typical regardless of income or childcare expenses.</p>
|
||
<p>We calculate the optimal intercept by including a feature with the constant value of 1 in our model, which is then multplied by some parameter <span class="math notranslate nohighlight">\(\theta_0\)</span> from the OLS method into the optimal intercept value (which will be <span class="math notranslate nohighlight">\(\theta_0\)</span>). In practice, we include the intercept in our model by adding a column of ones to the start of our feature matrix.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">n</span> <span class="o">=</span> <span class="mi">20</span>
|
||
<span class="n">income</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">116.</span><span class="p">,</span> <span class="mf">161.</span><span class="p">,</span> <span class="mf">167.</span><span class="p">,</span> <span class="mf">118.</span><span class="p">,</span> <span class="mf">172.</span><span class="p">,</span> <span class="mf">163.</span><span class="p">,</span> <span class="mf">179.</span><span class="p">,</span> <span class="mf">173.</span><span class="p">,</span> <span class="mf">162.</span><span class="p">,</span> <span class="mf">116.</span><span class="p">,</span> <span class="mf">101.</span><span class="p">,</span> <span class="mf">176.</span><span class="p">,</span> <span class="mf">178.</span><span class="p">,</span> <span class="mf">172.</span><span class="p">,</span> <span class="mf">143.</span><span class="p">,</span> <span class="mf">135.</span><span class="p">,</span> <span class="mf">160.</span><span class="p">,</span> <span class="mf">101.</span><span class="p">,</span> <span class="mf">149.</span><span class="p">,</span> <span class="mf">125.</span><span class="p">])</span>
|
||
<span class="n">children</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">])</span>
|
||
<span class="n">spending</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">152.</span><span class="p">,</span> <span class="mf">141.</span><span class="p">,</span> <span class="mf">102.</span><span class="p">,</span> <span class="mf">136.</span><span class="p">,</span> <span class="mf">161.</span><span class="p">,</span> <span class="mf">129.</span><span class="p">,</span> <span class="mf">99.</span><span class="p">,</span> <span class="mf">159.</span><span class="p">,</span> <span class="mf">160.</span><span class="p">,</span> <span class="mf">107.</span><span class="p">,</span> <span class="mf">98.</span><span class="p">,</span> <span class="mf">164.</span><span class="p">,</span> <span class="mf">121.</span><span class="p">,</span> <span class="mf">93.</span><span class="p">,</span> <span class="mf">112.</span><span class="p">,</span> <span class="mf">127.</span><span class="p">,</span> <span class="mf">117.</span><span class="p">,</span> <span class="mf">69.</span><span class="p">,</span> <span class="mf">156.</span><span class="p">,</span> <span class="mf">131.</span><span class="p">])</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>a)</strong> Create a feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> for the features income and children, including an intercept column of ones at the start.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">n</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
|
||
<span class="c1">#X[:, 0] = ...</span>
|
||
<span class="c1">#X[:, 1] = ...</span>
|
||
<span class="c1">#X[:, 2] = ...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>b)</strong> Use the expression from <strong>3d)</strong> to find the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\beta}_{OLS}}\)</span> for predicting spending based on these features. Create a function for this operation, as you are going to need to use it a lot.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">OLS_parameters</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="o">...</span>
|
||
|
||
<span class="c1">#beta = OLS_parameters(X, y)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="exercise-4-fitting-a-polynomial">
|
||
<h2>Exercise 4 - Fitting a polynomial<a class="headerlink" href="#exercise-4-fitting-a-polynomial" title="Link to this heading">#</a></h2>
|
||
<p>In this course, we typically do linear regression using polynomials, though in real world applications it is also very common to make linear models based on measured features like you did in the previous exercise.</p>
|
||
<p>When fitting a polynomial with linear regression, we make each polynomial degree(<span class="math notranslate nohighlight">\(x, x^2, x^3, ..., x^p\)</span>) its own feature.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>a)</strong> Create a feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> for the features <span class="math notranslate nohighlight">\(x, x^2, x^3, x^4, x^5\)</span>, including an intercept column of ones at the start. Make this into a function, as you will do this a lot over the next weeks.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">polynomial_features</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">p</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">n</span><span class="p">,</span> <span class="n">p</span> <span class="o">+</span> <span class="mi">1</span><span class="p">))</span>
|
||
<span class="c1">#X[:, 0] = ...</span>
|
||
<span class="c1">#X[:, 1] = ...</span>
|
||
<span class="c1">#X[:, 2] = ...</span>
|
||
<span class="c1"># could this be a loop?</span>
|
||
|
||
<span class="c1">#X = polynomial_features(x, 5)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>b)</strong> Use the expression from <strong>3d)</strong> to find the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\hat{\beta}_{OLS}}\)</span> for predicting <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> based on these features. If you have done everything right so far, this code will not need changing.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1">#beta = OLS_parameters(X, y)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>c)</strong> Like in exercise 4 last week, split your feature matrix and target data into a training split and test split.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
|
||
<span class="c1">#X_train, X_test, y_train, y_test = ...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>d)</strong> Train your model on the training data(find the parameters which best fit) and compute the MSE on both the training and test data.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Ellipsis
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>e)</strong> Do the same for each polynomial degree from 2 to 10, and plot the MSE on both the training and test data as a function of polynomial degree. The aim is to reproduce Figure 2.11 of <a class="reference external" href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf">Hastie et al</a>. Feel free to read the discussions leading to figure 2.11 of Hastie et al.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">...</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Ellipsis
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p><strong>f)</strong> Interpret the graph. Why do the lines move as they do? What does it tell us about model performance and generalizability?</p>
|
||
</section>
|
||
<section id="exercise-5-comparing-your-code-with-sklearn">
|
||
<h2>Exercise 5 - Comparing your code with sklearn<a class="headerlink" href="#exercise-5-comparing-your-code-with-sklearn" title="Link to this heading">#</a></h2>
|
||
<p>When implementing different algorithms for the first time, it can be helpful to double check your results with established implementations before you go on to add more complexity.</p>
|
||
<p><strong>a)</strong> Make sure your <code class="docutils literal notranslate"><span class="pre">polynomial_features</span></code> function creates the same feature matrix as sklearns PolynomialFeatures.</p>
|
||
<p>(<a class="reference external" href="https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html">https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.PolynomialFeatures.html</a>)</p>
|
||
<p><strong>b)</strong> Make sure your <code class="docutils literal notranslate"><span class="pre">OLS_parameters</span></code> function computes the same parameters as sklearns LinearRegression with fit_intercept set to False, since the intercept is included in the feature matrix. Use <code class="docutils literal notranslate"><span class="pre">your_model_object.coef_</span></code> to extract the computed parameters.</p>
|
||
<p>(<a class="reference external" href="https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html">https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LinearRegression.html</a>)</p>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="week34.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Week 34: Introduction to the course, Logistics and Practicalities</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="week35.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#deriving-and-implementing-ordinary-least-squares">Deriving and Implementing Ordinary Least Squares</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#learning-goals">Learning goals</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#deliverables">Deliverables</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#how-to-take-derivatives-of-matrix-vector-expressions">How to take derivatives of Matrix-Vector expressions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-1-finding-the-derivative-of-matrix-vector-expressions">Exercise 1 - Finding the derivative of Matrix-Vector expressions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-2-deriving-the-expression-for-ols">Exercise 2 - Deriving the expression for OLS</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-3-creating-feature-matrix-and-implementing-ols-using-the-analytical-expression">Exercise 3 - Creating feature matrix and implementing OLS using the analytical expression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-4-fitting-a-polynomial">Exercise 4 - Fitting a polynomial</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#exercise-5-comparing-your-code-with-sklearn">Exercise 5 - Comparing your code with sklearn</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |