Morten Hjorth-Jensen
3018 lines
267 KiB
HTML
3018 lines
267 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'week35';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Exercises week 36" href="exercisesweek36.html" />
|
||
<link rel="prev" title="Exercises week 35" href="exercisesweek35.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Applied Data Analysis and Machine Learning - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Course setting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Gradient descent methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek39.html">Weekly Exercises 6</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/week35.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#plans-for-week-35">Plans for week 35</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#reading-recommendations">Reading recommendations:</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#reminder-from-last-week">Reminder from last week</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-equations-for-ordinary-least-squares">The equations for ordinary least squares</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-cost-loss-function">The cost/loss function</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#interpretations-and-optimizing-our-parameters">Interpretations and optimizing our parameters</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id1">Interpretations and optimizing our parameters</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#some-useful-matrix-and-vector-expressions">Some useful matrix and vector expressions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-jacobian">The Jacobian</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#derivatives-example-1">Derivatives, example 1</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-2">Example 2</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-3">Example 3</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-4">Example 4</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-mean-squared-error-and-its-derivative">The mean squared error and its derivative</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#meet-the-hessian-matrix">Meet the Hessian Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id2">Interpretations and optimizing our parameters</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-relevant-for-the-exercises">Example relevant for the exercises</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#own-code-for-ordinary-least-squares">Own code for Ordinary Least Squares</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adding-error-analysis-and-training-set-up">Adding error analysis and training set up</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#splitting-our-data-in-training-and-test-data">Splitting our Data in Training and Test data</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-complete-code-with-a-simple-data-set">The complete code with a simple data set</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#making-your-own-test-train-splitting">Making your own test-train splitting</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#reducing-the-number-of-degrees-of-freedom-overarching-view">Reducing the number of degrees of freedom, overarching view</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#preprocessing-our-data">Preprocessing our data</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#functionality-in-scikit-learn">Functionality in Scikit-Learn</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-preprocessing">More preprocessing</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#frequently-used-scaling-functions">Frequently used scaling functions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-of-own-standard-scaling">Example of own Standard scaling</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#min-max-scaling">Min-Max Scaling</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing-the-means-squared-error-as-function-of-complexity">Testing the Means Squared Error as function of Complexity</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematical-interpretation-of-ordinary-least-squares">Mathematical Interpretation of Ordinary Least Squares</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#residual-error">Residual Error</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#simple-case">Simple case</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-singular-value-decomposition">The singular value decomposition</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#linear-regression-problems">Linear Regression Problems</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#fixing-the-singularity">Fixing the singularity</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ridge-and-lasso-regression">Ridge and LASSO Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#deriving-the-ridge-regression-equations">Deriving the Ridge Regression Equations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#basic-math-of-the-svd">Basic math of the SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-svd-a-fantastic-algorithm">The SVD, a Fantastic Algorithm</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#economy-size-svd">Economy-size SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#codes-for-the-svd">Codes for the SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#note-about-svd-calculations">Note about SVD Calculations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematics-of-the-svd-and-implications">Mathematics of the SVD and implications</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-matrix">Example Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#setting-up-the-matrix-to-be-inverted">Setting up the Matrix to be inverted</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#further-properties-important-for-our-analyses-later">Further properties (important for our analyses later)</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#meet-the-covariance-matrix">Meet the Covariance Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#introducing-the-covariance-and-correlation-functions">Introducing the Covariance and Correlation functions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#covariance-and-correlation-matrix">Covariance and Correlation Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#correlation-function-and-design-feature-matrix">Correlation Function and Design/Feature Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#covariance-matrix-examples">Covariance Matrix Examples</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#correlation-matrix">Correlation Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#correlation-matrix-with-pandas">Correlation Matrix with Pandas</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rewriting-the-covariance-and-or-correlation-matrix">Rewriting the Covariance and/or Correlation Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#linking-with-the-svd">Linking with the SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-does-it-mean">What does it mean?</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#and-finally-boldsymbol-x-boldsymbol-x-t">And finally <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span></a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#back-to-ridge-and-lasso-regression">Back to Ridge and LASSO Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#interpreting-the-ridge-results">Interpreting the Ridge results</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-interpretations">More interpretations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#deriving-the-lasso-regression-equations">Deriving the Lasso Regression Equations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#material-for-exercises-week-35">Material for exercises week 35</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#important-technicalities-more-on-rescaling-data">Important technicalities: More on Rescaling data</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html week35.do.txt --no_mako -->
|
||
<!-- dom:TITLE: Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression --><section class="tex2jax_ignore mathjax_ignore" id="week-35-from-ordinary-linear-regression-to-ridge-and-lasso-regression">
|
||
<h1>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression<a class="headerlink" href="#week-35-from-ordinary-linear-regression-to-ridge-and-lasso-regression" title="Link to this heading">#</a></h1>
|
||
<p><strong>Morten Hjorth-Jensen</strong>, Department of Physics, University of Oslo</p>
|
||
<p>Date: <strong>August 25-29, 2025</strong></p>
|
||
<section id="plans-for-week-35">
|
||
<h2>Plans for week 35<a class="headerlink" href="#plans-for-week-35" title="Link to this heading">#</a></h2>
|
||
<p>The main topics are:</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Brief repetition from last week</p></li>
|
||
<li><p>Discussions of the equations for ordinary least squares (<strong>OLS</strong>)</p></li>
|
||
<li><p>Discussion on how to prepare data and examples of applications of linear regression</p></li>
|
||
<li><p>Mathematical interpretations of OLS</p></li>
|
||
<li><p>Introduction of Ridge and Lasso regression</p></li>
|
||
</ol>
|
||
<section id="reading-recommendations">
|
||
<h3>Reading recommendations:<a class="headerlink" href="#reading-recommendations" title="Link to this heading">#</a></h3>
|
||
<ol class="arabic simple">
|
||
<li><p>These lecture notes</p></li>
|
||
<li><p>Video of lecture at <a class="reference external" href="https://youtu.be/2mvizAQFST8">https://youtu.be/2mvizAQFST8</a></p></li>
|
||
<li><p>Whiteboard notes at <a class="github reference external" href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2025/FYSSTKweek35.pdf">CompPhysics/MachineLearning</a></p></li>
|
||
<li><p>Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra</p></li>
|
||
<li><p>Raschka et al on preprocessing of data, relevant for exercise 3 this week, see chapter 4.</p></li>
|
||
<li><p>For exercise 1 of week 35, the book by A. Aldo Faisal, Cheng Soon Ong, and Marc Peter Deisenroth on the Mathematics of Machine Learning, may be very relevant. In particular chapter 5 at URL”<a class="reference external" href="https://mml-book.github.io/">https://mml-book.github.io/</a>” (section 5.5 on derivatives) is very useful for exercise 1 this coming week.</p></li>
|
||
</ol>
|
||
</section>
|
||
</section>
|
||
<section id="reminder-from-last-week">
|
||
<h2>Reminder from last week<a class="headerlink" href="#reminder-from-last-week" title="Link to this heading">#</a></h2>
|
||
<p>We need first a reminder from last week about linear regression. We are going to fit a continuous function with linear parameterization in terms of the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> and our first encounter is ordinary least squares.</p>
|
||
<ul class="simple">
|
||
<li><p>It is the method of choice for fitting a continuous function</p></li>
|
||
<li><p>Gives an excellent introduction to central Machine Learning features with <strong>understandable pedagogical</strong> links to other methods like <strong>Neural Networks</strong>, <strong>Support Vector Machines</strong> etc</p></li>
|
||
<li><p>Analytical expression for the fitting parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span></p></li>
|
||
<li><p>Analytical expressions for statistical propertiers like mean values, variances, confidence intervals and more</p></li>
|
||
<li><p>Analytical relation with probabilistic interpretations</p></li>
|
||
<li><p>Easy to introduce basic concepts like bias-variance tradeoff, cross-validation, resampling and regularization techniques and many other ML topics</p></li>
|
||
<li><p>Easy to code! And links well with classification problems and logistic regression and neural networks</p></li>
|
||
<li><p>Allows for <strong>easy</strong> hands-on understanding of gradient descent methods</p></li>
|
||
<li><p>and many more features</p></li>
|
||
</ul>
|
||
<p>For more discussions of Ridge and Lasso regression, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.
|
||
Similarly, <a class="reference external" href="https://arxiv.org/abs/1803.08823">Mehta et al’s article</a> is also recommended.</p>
|
||
</section>
|
||
<section id="the-equations-for-ordinary-least-squares">
|
||
<h2>The equations for ordinary least squares<a class="headerlink" href="#the-equations-for-ordinary-least-squares" title="Link to this heading">#</a></h2>
|
||
<p>Our data which we want to apply a machine learning method on, consist
|
||
of a set of inputs <span class="math notranslate nohighlight">\(\boldsymbol{x}^T=[x_0,x_1,x_2,\dots,x_{n-1}]\)</span> and the
|
||
outputs we want to model <span class="math notranslate nohighlight">\(\boldsymbol{y}^T=[y_0,y_1,y_2,\dots,y_{n-1}]\)</span>.
|
||
We assume that the output data can be represented (for a regression case) by a continuous function <span class="math notranslate nohighlight">\(f\)</span>
|
||
through</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y_i=f(x_i)+\epsilon_i,
|
||
\]</div>
|
||
<p>or in general</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{y}=f(\boldsymbol{x})+\boldsymbol{\epsilon},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{\epsilon}\)</span> represents some noise which is normally assumed to
|
||
be distributed via a normal probability distribution with zero mean
|
||
value and a variance <span class="math notranslate nohighlight">\(\sigma^2\)</span>.</p>
|
||
<p>In linear regression we approximate the unknown function with another
|
||
continuous function <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{y}}(\boldsymbol{x})\)</span> which depends linearly on
|
||
some unknown parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\theta}^T=[\theta_0,\theta_1,\theta_2,\dots,\theta_{p-1}]\)</span>.</p>
|
||
<p>Last week we introduced the so-called design matrix in order to define
|
||
the approximation <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}}\)</span> via the unknown quantity
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\tilde{y}}= \boldsymbol{X}\boldsymbol{\theta},
|
||
\]</div>
|
||
<p>and in order to find the optimal parameters <span class="math notranslate nohighlight">\(\theta_i\)</span> we defined a function which
|
||
gives a measure of the spread between the values <span class="math notranslate nohighlight">\(y_i\)</span> (which
|
||
represent the output values we want to reproduce) and the parametrized
|
||
values <span class="math notranslate nohighlight">\(\tilde{y}_i\)</span>, namely the so-called cost/loss function.</p>
|
||
</section>
|
||
<section id="the-cost-loss-function">
|
||
<h2>The cost/loss function<a class="headerlink" href="#the-cost-loss-function" title="Link to this heading">#</a></h2>
|
||
<p>We used the mean squared error to define the way we measure the quality of our model</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\theta})=\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)^T\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)\right\},
|
||
\]</div>
|
||
<p>or using the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and in a more compact matrix-vector notation as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\theta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)\right\}.
|
||
\]</div>
|
||
<p>This function represents one of many possible ways to define the so-called cost function.</p>
|
||
<p>It is also common to define
|
||
the function <span class="math notranslate nohighlight">\(C\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\theta})=\frac{1}{2n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2,
|
||
\]</div>
|
||
<p>since when taking the first derivative with respect to the unknown parameters <span class="math notranslate nohighlight">\(\theta\)</span>, the factor of <span class="math notranslate nohighlight">\(2\)</span> cancels out.</p>
|
||
</section>
|
||
<section id="interpretations-and-optimizing-our-parameters">
|
||
<h2>Interpretations and optimizing our parameters<a class="headerlink" href="#interpretations-and-optimizing-our-parameters" title="Link to this heading">#</a></h2>
|
||
<p>The function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\theta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)\right\},
|
||
\]</div>
|
||
<p>can be linked to the variance of the quantity <span class="math notranslate nohighlight">\(y_i\)</span> if we interpret the latter as the mean value.
|
||
When linking (see the discussions next week) with the maximum likelihood approach below, we will indeed interpret <span class="math notranslate nohighlight">\(y_i\)</span> as a mean value</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y_{i}=\langle y_i \rangle = \theta_0x_{i,0}+\theta_1x_{i,1}+\theta_2x_{i,2}+\dots+\theta_{n-1}x_{i,n-1}+\epsilon_i,
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\langle y_i \rangle\)</span> is the mean value. Keep in mind also that
|
||
till now we have treated <span class="math notranslate nohighlight">\(y_i\)</span> as the exact value. Normally, the
|
||
output (response, target, dependent or outcome) variable <span class="math notranslate nohighlight">\(y_i\)</span> is the outcome of a
|
||
numerical experiment or another type of experiment and could thus be treated itself as an
|
||
approximation to the true value. It is then always accompanied by an
|
||
error estimate, often limited to a statistical error estimate given by
|
||
the standard deviation discussed earlier. In the discussion here we
|
||
will treat <span class="math notranslate nohighlight">\(y_i\)</span> as our exact value for the output variable.</p>
|
||
<p>In order to find the parameters <span class="math notranslate nohighlight">\(\theta_i\)</span> we will then minimize the spread of <span class="math notranslate nohighlight">\(C(\boldsymbol{\theta})\)</span>, that is we are going to solve the problem</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\theta}}={\displaystyle \min_{\boldsymbol{\theta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)\right\}.
|
||
\]</div>
|
||
<p>In practical terms it means we will require</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\theta})}{\partial \theta_j} = \frac{\partial }{\partial \theta_j}\left[ \frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\theta_0x_{i,0}-\theta_1x_{i,1}-\theta_2x_{i,2}-\dots-\theta_{n-1}x_{i,n-1}\right)^2\right]=0,
|
||
\]</div>
|
||
<p>which results in</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\theta})}{\partial \theta_j} = -\frac{2}{n}\left[ \sum_{i=0}^{n-1}x_{ij}\left(y_i-\theta_0x_{i,0}-\theta_1x_{i,1}-\theta_2x_{i,2}-\dots-\theta_{n-1}x_{i,n-1}\right)\right]=0,
|
||
\]</div>
|
||
<p>or in a matrix-vector form as (multiplying away the factor <span class="math notranslate nohighlight">\(-2/n\)</span>, see derivation below)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}^T} = 0 = \boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right).
|
||
\]</div>
|
||
</section>
|
||
<section id="id1">
|
||
<h2>Interpretations and optimizing our parameters<a class="headerlink" href="#id1" title="Link to this heading">#</a></h2>
|
||
<p>We can rewrite, see the derivations below,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}^T} = 0 = \boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right),
|
||
\]</div>
|
||
<p>as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{y} = \boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\theta},
|
||
\]</div>
|
||
<p>and if the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> is invertible we have the solution</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\theta}} =\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>We note also that since our design matrix is defined as <span class="math notranslate nohighlight">\(\boldsymbol{X}\in
|
||
{\mathbb{R}}^{n\times p}\)</span>, the product <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X} \in
|
||
{\mathbb{R}}^{p\times p}\)</span>. In most cases we have that <span class="math notranslate nohighlight">\(p \ll n\)</span>. In our example case below we have <span class="math notranslate nohighlight">\(p=5\)</span> meaning. We end up with inverting a small
|
||
<span class="math notranslate nohighlight">\(5\times 5\)</span> matrix. This is a rather common situation, in many cases we end up with low-dimensional
|
||
matrices to invert. The methods discussed here and for many other
|
||
supervised learning algorithms like classification with logistic
|
||
regression or support vector machines, exhibit dimensionalities which
|
||
allow for the usage of direct linear algebra methods such as <strong>LU</strong> decomposition or <strong>Singular Value Decomposition</strong> (SVD) for finding the inverse of the matrix
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>.</p>
|
||
<p><strong>Small question</strong>: When inverting the matrix $\boldsymbol{X}^T\boldsymbol{X}, what kind of problems can we expect?</p>
|
||
</section>
|
||
<section id="some-useful-matrix-and-vector-expressions">
|
||
<h2>Some useful matrix and vector expressions<a class="headerlink" href="#some-useful-matrix-and-vector-expressions" title="Link to this heading">#</a></h2>
|
||
<p>The following matrix and vector relation will be useful here and for
|
||
the rest of the course. Vectors are always written as boldfaced lower
|
||
case letters and matrices as upper case boldfaced letters. In the
|
||
following we will discuss how to calculate derivatives of various
|
||
matrices relevant for machine learning. We will often represent our
|
||
data in terms of matrices and vectors.</p>
|
||
<p>Let us introduce first some conventions. We assume that <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is a
|
||
vector of length <span class="math notranslate nohighlight">\(m\)</span>, that is it has <span class="math notranslate nohighlight">\(m\)</span> elements <span class="math notranslate nohighlight">\(y_0,y_1,\dots,
|
||
y_{m-1}\)</span>. By convention we start labeling vectors with the zeroth
|
||
element, as are arrays in Python and C++/C, for example. Similarly, we
|
||
have a vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> of length <span class="math notranslate nohighlight">\(n\)</span>, that is
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{x}^T=[x_0,x_1,\dots, x_{n-1}]\)</span>.</p>
|
||
<p>We assume also that <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is a function of <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> through some
|
||
given function <span class="math notranslate nohighlight">\(f\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{y}=f(\boldsymbol{x}).
|
||
\]</div>
|
||
</section>
|
||
<section id="the-jacobian">
|
||
<h2>The Jacobian<a class="headerlink" href="#the-jacobian" title="Link to this heading">#</a></h2>
|
||
<p>We define the partial derivatives of the various components of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> as functions of <span class="math notranslate nohighlight">\(x_i\)</span> in terms of the so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Jacobian_matrix_and_determinant">Jacobian matrix</a></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{J}=\frac{\partial \boldsymbol{y}}{\partial \boldsymbol{x}}=\begin{bmatrix} \frac{\partial y_0}{\partial x_0} & \frac{\partial y_0}{\partial x_1} & \frac{\partial y_0}{\partial x_2} & \dots & \dots & \frac{\partial y_0}{\partial x_{n-1}} \\ \frac{\partial y_1}{\partial x_0} & \frac{\partial y_1}{\partial x_1} & \frac{\partial y_1}{\partial x_2} & \dots & \dots & \frac{\partial y_1}{\partial x_{n-1}} \\
|
||
\frac{\partial y_2}{\partial x_0} & \frac{\partial y_2}{\partial x_1} & \frac{\partial y_2}{\partial x_2} & \dots & \dots & \frac{\partial y_2}{\partial x_{n-1}} \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\frac{\partial y_{m-1}}{\partial x_0} & \frac{\partial y_{m-1}}{\partial x_1} & \frac{\partial y_{m-1}}{\partial x_2} & \dots & \dots & \frac{\partial y_{m-1}}{\partial x_{n-1}} \end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which is an <span class="math notranslate nohighlight">\(m\times n\)</span> matrix. If <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> is a scalar, then the
|
||
Jacobian is only a single-column vector, or an <span class="math notranslate nohighlight">\(m\times 1\)</span> matrix. If
|
||
on the other hand <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is a scalar, the Jacobian becomes a
|
||
<span class="math notranslate nohighlight">\(1\times n\)</span> matrix.</p>
|
||
<p>When this matrix is a square matrix <span class="math notranslate nohighlight">\(m=n\)</span>, its determinant is often referred to as the Jacobian
|
||
determinant. Both the matrix and (if <span class="math notranslate nohighlight">\(m=n\)</span>) the determinant are
|
||
often referred to simply as the Jacobian. The Jacobian matrix represents the differential of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> at every point where the
|
||
vector is differentiable.</p>
|
||
</section>
|
||
<section id="derivatives-example-1">
|
||
<h2>Derivatives, example 1<a class="headerlink" href="#derivatives-example-1" title="Link to this heading">#</a></h2>
|
||
<p>Let now <span class="math notranslate nohighlight">\(\boldsymbol{y}=\boldsymbol{A}\boldsymbol{x}\)</span>, where <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is an <span class="math notranslate nohighlight">\(m\times n\)</span> matrix and the matrix does not depend on <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span>. If we write out the vector <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> compoment by component we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y_i = \sum_{j=0}^{n-1}a_{ij}x_j,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\forall i=0,1,2,\dots,m-1\)</span>. The individual matrix elements of <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> are given by the symbol <span class="math notranslate nohighlight">\(a_{ij}\)</span>.
|
||
It follows that the partial derivatives of <span class="math notranslate nohighlight">\(y_i\)</span> with respect to <span class="math notranslate nohighlight">\(x_k\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial y_i }{\partial x_k}= a_{ik} \forall i=0,1,2,\dots,m-1.
|
||
\]</div>
|
||
<p>From this we have, using the definition of the Jacobian</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \boldsymbol{y} }{\partial \boldsymbol{x}}= \boldsymbol{A}.
|
||
\]</div>
|
||
</section>
|
||
<section id="example-2">
|
||
<h2>Example 2<a class="headerlink" href="#example-2" title="Link to this heading">#</a></h2>
|
||
<p>We define a scalar (our cost/loss functions are in general also scalars,
|
||
just think of the mean squared error) as the result of some matrix vector
|
||
multiplications</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{y}^T\boldsymbol{A}\boldsymbol{x},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> a vector of length <span class="math notranslate nohighlight">\(m\)</span>, <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> an <span class="math notranslate nohighlight">\(m\times n\)</span> matrix and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> a vector of length <span class="math notranslate nohighlight">\(n\)</span>. We assume also that <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> does not depend on any of the two vectors.
|
||
In order to find the derivative of <span class="math notranslate nohighlight">\(\alpha\)</span> with respect to the two vectors, we define an intermediate vector <span class="math notranslate nohighlight">\(\boldsymbol{z}\)</span>. We define first
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{z}^T=\boldsymbol{y}^T\boldsymbol{A}\)</span>, a vector of length <span class="math notranslate nohighlight">\(n\)</span>. We have then, using the definition of the Jacobian,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{z}^T\boldsymbol{x},
|
||
\]</div>
|
||
<p>which means that (using our previous example and keeping track of our definition of the derivative of a scalar) we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{x}} = \frac{\partial \boldsymbol{z}^T\boldsymbol{x}}{\partial \boldsymbol{x}}=\boldsymbol{z}^T.
|
||
\]</div>
|
||
<p>Note that the resulting vector elements are the same for <span class="math notranslate nohighlight">\(\boldsymbol{z}^T\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{z}\)</span>, the only difference is that one is just the transpose of the other. We have the transposed here since we have used that the inner product of two vectors is a scalar.</p>
|
||
<p>Since <span class="math notranslate nohighlight">\(\alpha\)</span> is a scalar we have <span class="math notranslate nohighlight">\(\alpha =\alpha^T=\boldsymbol{x}^T\boldsymbol{A}^T\boldsymbol{y}\)</span>. Defining now <span class="math notranslate nohighlight">\(\boldsymbol{z}^T=\boldsymbol{x}^T\boldsymbol{A}^T\)</span> we find that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{y}} = \boldsymbol{z}^T=\boldsymbol{x}^T\boldsymbol{A}^T.
|
||
\]</div>
|
||
</section>
|
||
<section id="example-3">
|
||
<h2>Example 3<a class="headerlink" href="#example-3" title="Link to this heading">#</a></h2>
|
||
<p>We start with a new scalar but where now the vector <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is
|
||
replaced by a vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> and the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is a square
|
||
matrix with dimension <span class="math notranslate nohighlight">\(n\times n\)</span>.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{x}^T\boldsymbol{A}\boldsymbol{x},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> a vector of length <span class="math notranslate nohighlight">\(n\)</span>.</p>
|
||
<p>We write out the specific sums involved in the calculation of <span class="math notranslate nohighlight">\(\alpha\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \sum_{i=0}^{n-1}\sum_{j=0}^{n-1}x_i a_{ij}x_j,
|
||
\]</div>
|
||
<p>taking the derivative of <span class="math notranslate nohighlight">\(\alpha\)</span> with respect to a given component <span class="math notranslate nohighlight">\(x_k\)</span> we get the two sums</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial x_k} = \sum_{i=0}^{n-1}a_{ik}x_i+\sum_{j=0}^{n-1}a_{kj}x_j,
|
||
\]</div>
|
||
<p>for <span class="math notranslate nohighlight">\(\forall k =0,1,2,\dots,n-1\)</span>. We identify these sums as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{x}} = \boldsymbol{x}^T\left(\boldsymbol{A}^T+\boldsymbol{A}\right).
|
||
\]</div>
|
||
<p>If the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is symmetric, that is <span class="math notranslate nohighlight">\(\boldsymbol{A}=\boldsymbol{A}^T\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{x}} = 2\boldsymbol{x}^T\boldsymbol{A}.
|
||
\]</div>
|
||
</section>
|
||
<section id="example-4">
|
||
<h2>Example 4<a class="headerlink" href="#example-4" title="Link to this heading">#</a></h2>
|
||
<p>We let the scalar <span class="math notranslate nohighlight">\(\alpha\)</span> be defined by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{y}^T\boldsymbol{x},
|
||
\]</div>
|
||
<p>where both <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> have the same length <span class="math notranslate nohighlight">\(n\)</span>, or if we
|
||
wish to think of them as column vectors, they have dimensions <span class="math notranslate nohighlight">\(n\times
|
||
1\)</span>. We assume that both <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> depend on a vector
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{z}\)</span> of the same length. To calculate the derivative of <span class="math notranslate nohighlight">\(\alpha\)</span>
|
||
with respect to a given component <span class="math notranslate nohighlight">\(z_k\)</span> we need first to write out the
|
||
inner product that defines <span class="math notranslate nohighlight">\(\alpha\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \sum_{i=0}^{n-1}y_ix_i,
|
||
\]</div>
|
||
<p>and the partial derivative</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial z_k} = \sum_{i=0}^{n-1}\left(x_i\frac{\partial y_i}{\partial z_k}+y_i\frac{\partial x_i}{\partial z_k}\right),
|
||
\]</div>
|
||
<p>for <span class="math notranslate nohighlight">\(\forall k =0,1,2,\dots,n-1\)</span>. We can rewrite the partial derivative in a more compact form as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{z}} = \boldsymbol{x}^T\frac{\partial \boldsymbol{y}}{\partial \boldsymbol{z}}+\boldsymbol{y}^T\frac{\partial \boldsymbol{x}}{\partial \boldsymbol{z}},
|
||
\]</div>
|
||
<p>and if <span class="math notranslate nohighlight">\(\boldsymbol{y}=\boldsymbol{x}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{z}} = 2\boldsymbol{x}^T\frac{\partial \boldsymbol{x}}{\partial \boldsymbol{z}}.
|
||
\]</div>
|
||
</section>
|
||
<section id="the-mean-squared-error-and-its-derivative">
|
||
<h2>The mean squared error and its derivative<a class="headerlink" href="#the-mean-squared-error-and-its-derivative" title="Link to this heading">#</a></h2>
|
||
<p>We defined earlier a possible cost function using the mean squared error</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\theta})=\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)^T\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)\right\},
|
||
\]</div>
|
||
<p>or using the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> we have the more compact matrix-vector</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\theta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)\right\}.
|
||
\]</div>
|
||
<p>We note that the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> does not depend on the unknown parameters defined by the vector <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>.
|
||
We are now interested in minimizing the cost function with respect to the unknown parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>.</p>
|
||
<p>The mean squared error is a scalar and if we use the results from example three above, we can define a new vector</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{w}=\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta},
|
||
\]</div>
|
||
<p>which depends on <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. We rewrite the cost function as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\theta})=\frac{1}{n}\boldsymbol{w}^T\boldsymbol{w},
|
||
\]</div>
|
||
<p>with partial derivative</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}}=\frac{2}{n}\boldsymbol{w}^T\frac{\partial \boldsymbol{w}}{\partial \boldsymbol{\theta}},
|
||
\]</div>
|
||
<p>and using that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \boldsymbol{w}}{\partial \boldsymbol{\theta}}=-\boldsymbol{X},
|
||
\]</div>
|
||
<p>where we used the result from example two above. Inserting the last expression we obtain</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}}=-\frac{2}{n}\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\boldsymbol{X},
|
||
\]</div>
|
||
<p>or as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}^T}=-\frac{2}{n}\boldsymbol{X}^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right).
|
||
\]</div>
|
||
</section>
|
||
<section id="meet-the-hessian-matrix">
|
||
<h2>Meet the Hessian Matrix<a class="headerlink" href="#meet-the-hessian-matrix" title="Link to this heading">#</a></h2>
|
||
<p>A very important matrix we will meet again and again in machine
|
||
learning is the Hessian. It is given by the second derivative of the
|
||
cost function with respect to the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. Using the above
|
||
expression for derivatives of vectors and matrices, we find that the
|
||
second derivative of the mean squared error as cost function is,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial}{\partial \boldsymbol{\theta}}\frac{\partial C(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}^T} =\frac{\partial}{\partial \boldsymbol{\theta}}\left[-\frac{2}{n}\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)\right]=\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>The Hessian matrix plays an important role and is defined here as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>For ordinary least squares, it is inversely proportional (derivation
|
||
next week) with the variance of the optimal parameters
|
||
<span class="math notranslate nohighlight">\(\hat{\boldsymbol{\theta}}\)</span>. Furthermore, we will see next week that it is
|
||
(aside the factor <span class="math notranslate nohighlight">\(1/n\)</span>) equal to the covariance matrix. It plays also a very
|
||
important role in optmization algorithms and Principal Component
|
||
Analysis as a way to reduce the dimensionality of a machine learning/data analysis
|
||
problem. We will discuss this in greater detail next week when we introduce gradient methods.</p>
|
||
<p><strong>Linear algebra question:</strong> Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).</p>
|
||
</section>
|
||
<section id="id2">
|
||
<h2>Interpretations and optimizing our parameters<a class="headerlink" href="#id2" title="Link to this heading">#</a></h2>
|
||
<p>The residuals <span class="math notranslate nohighlight">\(\boldsymbol{\epsilon}\)</span> are in turn given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\epsilon} = \boldsymbol{y}-\boldsymbol{\tilde{y}} = \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta},
|
||
\]</div>
|
||
<p>and with</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)= 0,
|
||
\]</div>
|
||
<p>we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{\epsilon}=\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)= 0,
|
||
\]</div>
|
||
<p>meaning that the solution for <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> is the one which minimizes the residuals.</p>
|
||
</section>
|
||
<section id="example-relevant-for-the-exercises">
|
||
<h2>Example relevant for the exercises<a class="headerlink" href="#example-relevant-for-the-exercises" title="Link to this heading">#</a></h2>
|
||
<p>In order to understand the relation among the predictors <span class="math notranslate nohighlight">\(p\)</span>, the set of data <span class="math notranslate nohighlight">\(n\)</span> and the target (outcome, output etc) <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span>,
|
||
we condiser a simple polynomial fit.
|
||
We assume our data can represented by a fourth-order polynomial. For the <span class="math notranslate nohighlight">\(i\)</span>th component we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_i = \theta_0+\theta_1x_i+\theta_2x_i^2+\theta_3x_i^3+\theta_4x_i^4.
|
||
\]</div>
|
||
<p>we have five predictors/features. The first is the intercept <span class="math notranslate nohighlight">\(\theta_0\)</span>. The other terms are <span class="math notranslate nohighlight">\(\theta_i\)</span> with <span class="math notranslate nohighlight">\(i=1,2,3,4\)</span>. Furthermore we have <span class="math notranslate nohighlight">\(n\)</span> entries for each predictor. It means that our design matrix is an
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</section>
|
||
<section id="own-code-for-ordinary-least-squares">
|
||
<h2>Own code for Ordinary Least Squares<a class="headerlink" href="#own-code-for-ordinary-least-squares" title="Link to this heading">#</a></h2>
|
||
<p>It is rather straightforward to implement the matrix inversion and obtain the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. After having defined the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and the outputs <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> we have</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># matrix inversion to find theta</span>
|
||
<span class="c1"># First we set up the data</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">5</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mf">0.1</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="c1"># and then the design matrix X including the intercept</span>
|
||
<span class="c1"># The design matrix now as function of a fourth-order polynomial</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span><span class="mi">5</span><span class="p">))</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">3</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">3</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">4</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">4</span>
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">inv</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span><span class="p">)</span> <span class="o">@</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="p">)</span> <span class="o">@</span> <span class="n">y</span>
|
||
<span class="c1"># and then make the prediction</span>
|
||
<span class="n">ytilde</span> <span class="o">=</span> <span class="n">X</span> <span class="o">@</span> <span class="n">theta</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Alternatively, you can use the least squares functionality in <strong>Numpy</strong> as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fit</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">lstsq</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">rcond</span> <span class="o">=</span><span class="kc">None</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||
<span class="n">ytildenp</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">fit</span><span class="p">,</span><span class="n">X</span><span class="o">.</span><span class="n">T</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="adding-error-analysis-and-training-set-up">
|
||
<h2>Adding error analysis and training set up<a class="headerlink" href="#adding-error-analysis-and-training-set-up" title="Link to this heading">#</a></h2>
|
||
<p>We can easily test our fit by computing the <span class="math notranslate nohighlight">\(R2\)</span> score that we discussed in connection with the functionality of <strong>Scikit-Learn</strong> in the introductory slides.
|
||
Since we are not using <strong>Scikit-Learn</strong> here we can define our own <span class="math notranslate nohighlight">\(R2\)</span> function as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">R2</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span> <span class="n">y_model</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">y_model</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_data</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>and we would be using it as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="n">R2</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We can easily add our <strong>MSE</strong> score as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>and finally the relative error as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">RelativeError</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="nb">abs</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">/</span><span class="n">y_data</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">RelativeError</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">ytilde</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="splitting-our-data-in-training-and-test-data">
|
||
<h2>Splitting our Data in Training and Test data<a class="headerlink" href="#splitting-our-data-in-training-and-test-data" title="Link to this heading">#</a></h2>
|
||
<p>It is normal in essentially all Machine Learning studies to split the
|
||
data in a training set and a test set (sometimes also an additional
|
||
validation set). <strong>Scikit-Learn</strong> has an own function for this. There
|
||
is no explicit recipe for how much data should be included as training
|
||
data and say test data. An accepted rule of thumb is to use
|
||
approximately <span class="math notranslate nohighlight">\(2/3\)</span> to <span class="math notranslate nohighlight">\(4/5\)</span> of the data as training data. We will
|
||
postpone a discussion of this splitting to the end of these notes and
|
||
our discussion of the so-called <strong>bias-variance</strong> tradeoff. Here we
|
||
limit ourselves to repeat the above equation of state fitting example
|
||
but now splitting the data into a training set and a test set.</p>
|
||
</section>
|
||
<section id="the-complete-code-with-a-simple-data-set">
|
||
<h2>The complete code with a simple data set<a class="headerlink" href="#the-complete-code-with-a-simple-data-set" title="Link to this heading">#</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
|
||
|
||
<span class="kn">import</span> <span class="nn">os</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">R2</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span> <span class="n">y_model</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">y_model</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_data</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">5</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mf">0.1</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># The design matrix now as function of a fourth-order polynomial</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span><span class="mi">5</span><span class="p">))</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">3</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">3</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">4</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">4</span>
|
||
<span class="c1"># We split the data in test and training data</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
<span class="c1"># matrix inversion to find theta</span>
|
||
<span class="n">theta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">inv</span><span class="p">(</span><span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X_train</span><span class="p">)</span> <span class="o">@</span> <span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y_train</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">theta</span><span class="p">)</span>
|
||
<span class="c1"># and then make the prediction</span>
|
||
<span class="n">ytilde</span> <span class="o">=</span> <span class="n">X_train</span> <span class="o">@</span> <span class="n">theta</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Training R2"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">R2</span><span class="p">(</span><span class="n">y_train</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Training MSE"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y_train</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
<span class="n">ypredict</span> <span class="o">=</span> <span class="n">X_test</span> <span class="o">@</span> <span class="n">theta</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Test R2"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">R2</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredict</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Test MSE"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredict</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="making-your-own-test-train-splitting">
|
||
<h2>Making your own test-train splitting<a class="headerlink" href="#making-your-own-test-train-splitting" title="Link to this heading">#</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># equivalently in numpy</span>
|
||
<span class="k">def</span> <span class="nf">train_test_split_numpy</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">train_size</span><span class="p">,</span> <span class="n">test_size</span><span class="p">):</span>
|
||
<span class="n">n_inputs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
|
||
<span class="n">inputs_shuffled</span> <span class="o">=</span> <span class="n">inputs</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="n">labels_shuffled</span> <span class="o">=</span> <span class="n">labels</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">inputs_shuffled</span><span class="p">)</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">labels_shuffled</span><span class="p">)</span>
|
||
|
||
<span class="n">train_end</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n_inputs</span><span class="o">*</span><span class="n">train_size</span><span class="p">)</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span> <span class="o">=</span> <span class="n">inputs_shuffled</span><span class="p">[:</span><span class="n">train_end</span><span class="p">],</span> <span class="n">inputs_shuffled</span><span class="p">[</span><span class="n">train_end</span><span class="p">:]</span>
|
||
<span class="n">Y_train</span><span class="p">,</span> <span class="n">Y_test</span> <span class="o">=</span> <span class="n">labels_shuffled</span><span class="p">[:</span><span class="n">train_end</span><span class="p">],</span> <span class="n">labels_shuffled</span><span class="p">[</span><span class="n">train_end</span><span class="p">:]</span>
|
||
|
||
<span class="k">return</span> <span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">,</span> <span class="n">Y_test</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>But since <strong>scikit-learn</strong> has its own function for doing this and since
|
||
it interfaces easily with <strong>tensorflow</strong> and other libraries, we
|
||
normally recommend using the latter functionality.</p>
|
||
</section>
|
||
<section id="reducing-the-number-of-degrees-of-freedom-overarching-view">
|
||
<h2>Reducing the number of degrees of freedom, overarching view<a class="headerlink" href="#reducing-the-number-of-degrees-of-freedom-overarching-view" title="Link to this heading">#</a></h2>
|
||
<p>Many Machine Learning problems involve thousands or even millions of
|
||
features for each training instance. Not only does this make training
|
||
extremely slow, it can also make it much harder to find a good
|
||
solution, as we will see. This problem is often referred to as the
|
||
curse of dimensionality. Fortunately, in real-world problems, it is
|
||
often possible to reduce the number of features considerably, turning
|
||
an intractable problem into a tractable one.</p>
|
||
<p>Later we will discuss some of the most popular dimensionality reduction
|
||
techniques: the principal component analysis (PCA), Kernel PCA, and
|
||
Locally Linear Embedding (LLE).</p>
|
||
<p>Principal component analysis and its various variants deal with the
|
||
problem of fitting a low-dimensional <a class="reference external" href="https://en.wikipedia.org/wiki/Affine_space">affine
|
||
subspace</a> to a set of of
|
||
data points in a high-dimensional space. With its family of methods it
|
||
is one of the most used tools in data modeling, compression and
|
||
visualization.</p>
|
||
</section>
|
||
<section id="preprocessing-our-data">
|
||
<h2>Preprocessing our data<a class="headerlink" href="#preprocessing-our-data" title="Link to this heading">#</a></h2>
|
||
<p>Before we proceed however, we will discuss how to preprocess our
|
||
data. Till now and in connection with our previous examples we have
|
||
not met so many cases where we are too sensitive to the scaling of our
|
||
data. Normally the data may need a rescaling and/or may be sensitive
|
||
to extreme values. Scaling the data renders our inputs much more
|
||
suitable for the algorithms we want to employ.</p>
|
||
<p>For data sets gathered for real world applications, it is rather normal that
|
||
different features have very different units and
|
||
numerical scales. For example, a data set detailing health habits may include
|
||
features such as <strong>age</strong> in the range <span class="math notranslate nohighlight">\(0-80\)</span>, and <strong>caloric intake</strong> of order <span class="math notranslate nohighlight">\(2000\)</span>.
|
||
Many machine learning methods sensitive to the scales of the features and may perform poorly if they
|
||
are very different scales. Therefore, it is typical to scale
|
||
the features in a way to avoid such outlier values.</p>
|
||
</section>
|
||
<section id="functionality-in-scikit-learn">
|
||
<h2>Functionality in Scikit-Learn<a class="headerlink" href="#functionality-in-scikit-learn" title="Link to this heading">#</a></h2>
|
||
<p><strong>Scikit-Learn</strong> has several functions which allow us to rescale the
|
||
data, normally resulting in much better results in terms of various
|
||
accuracy scores. The <strong>StandardScaler</strong> function in <strong>Scikit-Learn</strong>
|
||
ensures that for each feature/predictor we study the mean value is
|
||
zero and the variance is one (every column in the design/feature
|
||
matrix). This scaling has the drawback that it does not ensure that
|
||
we have a particular maximum or minimum in our data set. Another
|
||
function included in <strong>Scikit-Learn</strong> is the <strong>MinMaxScaler</strong> which
|
||
ensures that all features are exactly between <span class="math notranslate nohighlight">\(0\)</span> and <span class="math notranslate nohighlight">\(1\)</span>. The</p>
|
||
</section>
|
||
<section id="more-preprocessing">
|
||
<h2>More preprocessing<a class="headerlink" href="#more-preprocessing" title="Link to this heading">#</a></h2>
|
||
<p>The <strong>Normalizer</strong> scales each data
|
||
point such that the feature vector has a euclidean length of one. In other words, it
|
||
projects a data point on the circle (or sphere in the case of higher dimensions) with a
|
||
radius of 1. This means every data point is scaled by a different number (by the
|
||
inverse of it’s length).
|
||
This normalization is often used when only the direction (or angle) of the data matters,
|
||
not the length of the feature vector.</p>
|
||
<p>The <strong>RobustScaler</strong> works similarly to the StandardScaler in that it
|
||
ensures statistical properties for each feature that guarantee that
|
||
they are on the same scale. However, the RobustScaler uses the median
|
||
and quartiles, instead of mean and variance. This makes the
|
||
RobustScaler ignore data points that are very different from the rest
|
||
(like measurement errors). These odd data points are also called
|
||
outliers, and might often lead to trouble for other scaling
|
||
techniques.</p>
|
||
</section>
|
||
<section id="frequently-used-scaling-functions">
|
||
<h2>Frequently used scaling functions<a class="headerlink" href="#frequently-used-scaling-functions" title="Link to this heading">#</a></h2>
|
||
<p>Many features are often scaled using standardization to improve performance. In <strong>Scikit-Learn</strong> this is given by the <strong>StandardScaler</strong> function as discussed above. It is easy however to write your own.
|
||
Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
x_j^{(i)} \rightarrow \frac{x_j^{(i)} - \overline{x}_j}{\sigma(x_j)},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\overline{x}_j\)</span> and <span class="math notranslate nohighlight">\(\sigma(x_j)\)</span> are the mean and standard deviation, respectively, of the feature <span class="math notranslate nohighlight">\(x_j\)</span>.
|
||
This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don’t wish to calculate it, it is then common to simply set it to one.</p>
|
||
</section>
|
||
<section id="example-of-own-standard-scaling">
|
||
<h2>Example of own Standard scaling<a class="headerlink" href="#example-of-own-standard-scaling" title="Link to this heading">#</a></h2>
|
||
<p>Let us consider the following vanilla example where we use both
|
||
<strong>Scikit-Learn</strong> and write our own function as well. We produce a
|
||
simple test design matrix with random numbers. Each column could then
|
||
represent a specific feature whose mean value is subracted.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">sklearn.linear_model</span> <span class="k">as</span> <span class="nn">skl</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_error</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">MinMaxScaler</span><span class="p">,</span> <span class="n">StandardScaler</span><span class="p">,</span> <span class="n">Normalizer</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">from</span> <span class="nn">IPython.display</span> <span class="kn">import</span> <span class="n">display</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="c1"># setting up a 10 x 5 matrix</span>
|
||
<span class="n">rows</span> <span class="o">=</span> <span class="mi">10</span>
|
||
<span class="n">cols</span> <span class="o">=</span> <span class="mi">5</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">rows</span><span class="p">,</span><span class="n">cols</span><span class="p">)</span>
|
||
<span class="n">XPandas</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">display</span><span class="p">(</span><span class="n">XPandas</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">XPandas</span><span class="o">.</span><span class="n">mean</span><span class="p">())</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">XPandas</span><span class="o">.</span><span class="n">std</span><span class="p">())</span>
|
||
<span class="n">XPandas</span> <span class="o">=</span> <span class="p">(</span><span class="n">XPandas</span> <span class="o">-</span><span class="n">XPandas</span><span class="o">.</span><span class="n">mean</span><span class="p">())</span>
|
||
<span class="n">display</span><span class="p">(</span><span class="n">XPandas</span><span class="p">)</span>
|
||
<span class="c1"># This option does not include the standard deviation</span>
|
||
<span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="n">with_std</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
<span class="n">scaler</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">Xscaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">display</span><span class="p">(</span><span class="n">XPandas</span><span class="o">-</span><span class="n">Xscaled</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.</p>
|
||
</section>
|
||
<section id="min-max-scaling">
|
||
<h2>Min-Max Scaling<a class="headerlink" href="#min-max-scaling" title="Link to this heading">#</a></h2>
|
||
<p>Another commonly used scaling method is min-max scaling. This is very
|
||
useful for when we want the features to lie in a certain interval. To
|
||
scale the feature <span class="math notranslate nohighlight">\(x_j\)</span> to the interval <span class="math notranslate nohighlight">\([a, b]\)</span>, we can apply the
|
||
transformation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
x_j^{(i)} \rightarrow (b-a)\frac{x_j^{(i)} - \min(x_j)}{\max(x_j) - \min(x_j)} - a
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\min(x_j)\)</span> and <span class="math notranslate nohighlight">\(\max(x_j)\)</span> return the minimum and maximum value of <span class="math notranslate nohighlight">\(x_j\)</span> over the data set, respectively.</p>
|
||
</section>
|
||
<section id="testing-the-means-squared-error-as-function-of-complexity">
|
||
<h2>Testing the Means Squared Error as function of Complexity<a class="headerlink" href="#testing-the-means-squared-error-as-function-of-complexity" title="Link to this heading">#</a></h2>
|
||
<p>One of
|
||
the aims is to reproduce Figure 2.11 of <a class="reference external" href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf">Hastie et al</a>.</p>
|
||
<p>Our data is defined by <span class="math notranslate nohighlight">\(x\in [-3,3]\)</span> with a total of for example <span class="math notranslate nohighlight">\(100\)</span> data points.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">()</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">maxdegree</span> <span class="o">=</span> <span class="mi">14</span>
|
||
<span class="c1"># Make data set.</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>where <span class="math notranslate nohighlight">\(y\)</span> is the function we want to fit with a given polynomial.</p>
|
||
<p>Write a first code which sets up a design matrix <span class="math notranslate nohighlight">\(X\)</span> defined by a fourth-order polynomial. Scale your data and split it in training and test data.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">PolynomialFeatures</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.pipeline</span> <span class="kn">import</span> <span class="n">make_pipeline</span>
|
||
|
||
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">2018</span><span class="p">)</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">maxdegree</span> <span class="o">=</span> <span class="mi">5</span>
|
||
<span class="c1"># Make data set.</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
<span class="n">TestError</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">)</span>
|
||
<span class="n">TrainError</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">)</span>
|
||
<span class="n">polydegree</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">)</span>
|
||
<span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
<span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">()</span>
|
||
<span class="n">scaler</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">x_train</span><span class="p">)</span>
|
||
<span class="n">x_train_scaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">x_train</span><span class="p">)</span>
|
||
<span class="n">x_test_scaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">x_test</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">degree</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">):</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">make_pipeline</span><span class="p">(</span><span class="n">PolynomialFeatures</span><span class="p">(</span><span class="n">degree</span><span class="o">=</span><span class="n">degree</span><span class="p">),</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
|
||
<span class="n">clf</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">x_train_scaled</span><span class="p">,</span><span class="n">y_train</span><span class="p">)</span>
|
||
<span class="n">y_fit</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">x_train_scaled</span><span class="p">)</span>
|
||
<span class="n">y_pred</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">x_test_scaled</span><span class="p">)</span>
|
||
<span class="n">polydegree</span><span class="p">[</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">degree</span>
|
||
<span class="n">TestError</span><span class="p">[</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">((</span><span class="n">y_test</span> <span class="o">-</span> <span class="n">y_pred</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="p">)</span>
|
||
<span class="n">TrainError</span><span class="p">[</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">((</span><span class="n">y_train</span> <span class="o">-</span> <span class="n">y_fit</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">polydegree</span><span class="p">,</span> <span class="n">TestError</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">'Test Error'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">polydegree</span><span class="p">,</span> <span class="n">TrainError</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">'Train Error'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="mathematical-interpretation-of-ordinary-least-squares">
|
||
<h2>Mathematical Interpretation of Ordinary Least Squares<a class="headerlink" href="#mathematical-interpretation-of-ordinary-least-squares" title="Link to this heading">#</a></h2>
|
||
<p>What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).</p>
|
||
<p>We have shown that in ordinary least squares the optimal parameters <span class="math notranslate nohighlight">\(\theta\)</span> are given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\theta}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The <strong>hat</strong> over <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> means we have the optimal parameters after minimization of the cost function.</p>
|
||
<p>This means that our best model is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\theta}} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>We now define a matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T.
|
||
\]</div>
|
||
<p>We can rewrite</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\theta}} = \boldsymbol{A}\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> has the important property that <span class="math notranslate nohighlight">\(\boldsymbol{A}^2=\boldsymbol{A}\)</span>. This is the definition of a projection matrix.
|
||
We can then interpret our optimal model <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{y}}\)</span> as being represented by an orthogonal projection of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> onto a space defined by the column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>. In our case here the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is a square matrix. If it is a general rectangular matrix we have an oblique projection matrix.</p>
|
||
</section>
|
||
<section id="residual-error">
|
||
<h2>Residual Error<a class="headerlink" href="#residual-error" title="Link to this heading">#</a></h2>
|
||
<p>We have defined the residual error as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=\left[\boldsymbol{I}-\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\right]\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The residual errors are then the projections of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> onto the orthogonal component of the space defined by the column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</section>
|
||
<section id="simple-case">
|
||
<h2>Simple case<a class="headerlink" href="#simple-case" title="Link to this heading">#</a></h2>
|
||
<p>If the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is an orthogonal (or unitary in case of complex values) matrix, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{X}\boldsymbol{X}^T = \boldsymbol{I}.
|
||
\]</div>
|
||
<p>In this case the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> becomes</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T)=\boldsymbol{I},
|
||
\]</div>
|
||
<p>and we have the obvious case</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=0.
|
||
\]</div>
|
||
<p>This serves also as a useful test of our codes.</p>
|
||
</section>
|
||
<section id="the-singular-value-decomposition">
|
||
<h2>The singular value decomposition<a class="headerlink" href="#the-singular-value-decomposition" title="Link to this heading">#</a></h2>
|
||
<p>The examples we have looked at so far are cases where we normally can
|
||
invert the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. Using a polynomial expansion where we fit of various functions leads to
|
||
row vectors of the design matrix which are essentially orthogonal due
|
||
to the polynomial character of our model. Obtaining the inverse of the
|
||
design matrix is then often done via a so-called LU, QR or Cholesky
|
||
decomposition.</p>
|
||
<p>As we will also see in the first project,
|
||
this may
|
||
however not the be case in general and a standard matrix inversion
|
||
algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below and in other examples.</p>
|
||
<p>There is however a way to circumvent this problem and also
|
||
gain some insights about the ordinary least squares approach, and
|
||
later shrinkage methods like Ridge and Lasso regressions.</p>
|
||
<p>This is given by the <strong>Singular Value Decomposition</strong> (SVD) algorithm,
|
||
perhaps the most powerful linear algebra algorithm. The SVD provides
|
||
a numerically stable matrix decomposition that is used in a large
|
||
swath oc applications and the decomposition is always stable
|
||
numerically.</p>
|
||
<p>In machine learning it plays a central role in dealing with for
|
||
example design matrices that may be near singular or singular.
|
||
Furthermore, as we will see here, the singular values can be related
|
||
to the covariance matrix (and thereby the correlation matrix) and in
|
||
turn the variance of a given quantity. It plays also an important role
|
||
in the principal component analysis where high-dimensional data can be
|
||
reduced to the statistically relevant features.</p>
|
||
</section>
|
||
<section id="linear-regression-problems">
|
||
<h2>Linear Regression Problems<a class="headerlink" href="#linear-regression-problems" title="Link to this heading">#</a></h2>
|
||
<p>One of the typical problems we encounter with linear regression, in particular
|
||
when the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> (our so-called design matrix) is high-dimensional,
|
||
are problems with near singular or singular matrices. The column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>
|
||
may be linearly dependent, normally referred to as super-collinearity.<br />
|
||
This means that the matrix may be rank deficient and it is basically impossible to
|
||
to model the data using linear regression. As an example, consider the matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
\mathbf{X} & = \left[
|
||
\begin{array}{rrr}
|
||
1 & -1 & 2
|
||
\\
|
||
1 & 0 & 1
|
||
\\
|
||
1 & 2 & -1
|
||
\\
|
||
1 & 1 & 0
|
||
\end{array} \right]
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>The columns of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> are linearly dependent. We see this easily since the
|
||
the first column is the row-wise sum of the other two columns. The rank (more correct,
|
||
the column rank) of a matrix is the dimension of the space spanned by the
|
||
column vectors. Hence, the rank of <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> is equal to the number
|
||
of linearly independent columns. In this particular case the matrix has rank 2.</p>
|
||
<p>Super-collinearity of an <span class="math notranslate nohighlight">\((n \times p)\)</span>-dimensional design matrix <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> implies
|
||
that the inverse of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
\boldsymbol{X} & = \left[
|
||
\begin{array}{rr}
|
||
1 & -1
|
||
\\
|
||
1 & -1
|
||
\end{array} \right].
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>We see easily that <span class="math notranslate nohighlight">\(\mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0\)</span>. Hence, <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> is singular and its inverse is undefined.
|
||
This is equivalent to saying that the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has at least an eigenvalue which is zero.</p>
|
||
</section>
|
||
<section id="fixing-the-singularity">
|
||
<h2>Fixing the singularity<a class="headerlink" href="#fixing-the-singularity" title="Link to this heading">#</a></h2>
|
||
<p>If our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> which enters the linear regression problem</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\boldsymbol{\theta} = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y},
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>has linearly dependent column vectors, we will not be able to compute the inverse
|
||
of <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> and we cannot find the parameters (estimators) <span class="math notranslate nohighlight">\(\theta_i\)</span>.
|
||
The estimators are only well-defined if <span class="math notranslate nohighlight">\((\boldsymbol{X}^{T}\boldsymbol{X})^{-1}\)</span> exits.
|
||
This is more likely to happen when the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is high-dimensional. In this case it is likely to encounter a situation where
|
||
the regression parameters <span class="math notranslate nohighlight">\(\theta_i\)</span> cannot be estimated.</p>
|
||
<p>A cheap <em>ad hoc</em> approach is simply to add a small diagonal component to the matrix to invert, that is we change</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{I}\)</span> is the identity matrix. When we discuss <strong>Ridge</strong> regression this is actually what we end up evaluating. The parameter <span class="math notranslate nohighlight">\(\lambda\)</span> is called a hyperparameter. More about this later.</p>
|
||
</section>
|
||
<section id="ridge-and-lasso-regression">
|
||
<h2>Ridge and LASSO Regression<a class="headerlink" href="#ridge-and-lasso-regression" title="Link to this heading">#</a></h2>
|
||
<p>Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
|
||
our optimization problem is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)\right\}.
|
||
\]</div>
|
||
<p>or we can state it as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2,
|
||
\]</div>
|
||
<p>where we have used the definition of a norm-2 vector, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
||
\]</div>
|
||
<p>By minimizing the above equation with respect to the parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> we could then obtain an analytical expression for the
|
||
parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. We can add a regularization parameter <span class="math notranslate nohighlight">\(\lambda\)</span> by
|
||
defining a new cost function to be optimized, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\theta}\vert\vert_2^2
|
||
\]</div>
|
||
<p>which leads to the Ridge regression minimization problem where we
|
||
require that <span class="math notranslate nohighlight">\(\vert\vert \boldsymbol{\theta}\vert\vert_2^2\le t\)</span>, where <span class="math notranslate nohighlight">\(t\)</span> is
|
||
a finite number larger than zero. By defining</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\theta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\theta}\vert\vert_1,
|
||
\]</div>
|
||
<p>we have a new optimization equation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\theta}\vert\vert_1
|
||
\]</div>
|
||
<p>which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator.</p>
|
||
<p>Here we have defined the norm-1 as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
|
||
\]</div>
|
||
</section>
|
||
<section id="deriving-the-ridge-regression-equations">
|
||
<h2>Deriving the Ridge Regression Equations<a class="headerlink" href="#deriving-the-ridge-regression-equations" title="Link to this heading">#</a></h2>
|
||
<p>Using the matrix-vector expression for Ridge regression and dropping the parameter <span class="math notranslate nohighlight">\(1/n\)</span> in front of the standard means squared error equation, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\theta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta})\right\}+\lambda\boldsymbol{\theta}^T\boldsymbol{\theta},
|
||
\]</div>
|
||
<p>and
|
||
taking the derivatives with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> we obtain then
|
||
a slightly modified matrix inversion problem which for finite values
|
||
of <span class="math notranslate nohighlight">\(\lambda\)</span> does not suffer from singularity problems. We obtain
|
||
the optimal parameters</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\theta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{I}\)</span> being a <span class="math notranslate nohighlight">\(p\times p\)</span> identity matrix with the constraint that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sum_{i=0}^{p-1} \theta_i^2 \leq t,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(t\)</span> a finite positive number.</p>
|
||
<p>If we keep the <span class="math notranslate nohighlight">\(1/n\)</span> factor, the equation for the optimal <span class="math notranslate nohighlight">\(\theta\)</span> changes to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\theta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+n\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>In many textbooks the <span class="math notranslate nohighlight">\(1/n\)</span> term is often omitted. Note that a library like <strong>Scikit-Learn</strong> does not include the <span class="math notranslate nohighlight">\(1/n\)</span> factor in the setup of the cost function.</p>
|
||
<p>When we compare this with the ordinary least squares result we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\theta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>.</p>
|
||
<p>We see that Ridge regression is nothing but the standard OLS with a
|
||
modified diagonal term added to <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. The consequences, in
|
||
particular for our discussion of the bias-variance tradeoff are rather
|
||
interesting. We will see that for specific values of <span class="math notranslate nohighlight">\(\lambda\)</span>, we may
|
||
even reduce the variance of the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.</p>
|
||
<p>When we have discussed the singular value decomposition of the design
|
||
matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>, we will in turn perform a more rigorous mathematical
|
||
discussion of Ridge regression.</p>
|
||
<p>The code here is a simple demonstration of how to implement Ridge regression with our own code and compare this with scikit-learn.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">linear_model</span>
|
||
|
||
<span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
|
||
<span class="c1"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||
<span class="c1"># Useful for eventual debugging.</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">3155</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">Maxpolydegree</span> <span class="o">=</span> <span class="mi">20</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="n">Maxpolydegree</span><span class="p">))</span>
|
||
<span class="c1">#We include explicitely the intercept column</span>
|
||
<span class="k">for</span> <span class="n">degree</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Maxpolydegree</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="n">degree</span>
|
||
<span class="c1"># We split the data in test and training data</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">Maxpolydegree</span>
|
||
<span class="n">I</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">p</span><span class="p">,</span><span class="n">p</span><span class="p">)</span>
|
||
<span class="c1"># Decide which values of lambda to use</span>
|
||
<span class="n">nlambdas</span> <span class="o">=</span> <span class="mi">6</span>
|
||
<span class="n">MSEOwnRidgePredict</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="n">MSERidgePredict</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="n">lambdas</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">logspace</span><span class="p">(</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">):</span>
|
||
<span class="n">lmb</span> <span class="o">=</span> <span class="n">lambdas</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
<span class="n">OwnRidgeTheta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X_train</span><span class="o">+</span><span class="n">lmb</span><span class="o">*</span><span class="n">I</span><span class="p">)</span> <span class="o">@</span> <span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y_train</span>
|
||
<span class="c1"># Note: we include the intercept column and no scaling</span>
|
||
<span class="n">RegRidge</span> <span class="o">=</span> <span class="n">linear_model</span><span class="o">.</span><span class="n">Ridge</span><span class="p">(</span><span class="n">lmb</span><span class="p">,</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
<span class="n">RegRidge</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span><span class="n">y_train</span><span class="p">)</span>
|
||
<span class="c1"># and then make the prediction</span>
|
||
<span class="n">ytildeOwnRidge</span> <span class="o">=</span> <span class="n">X_train</span> <span class="o">@</span> <span class="n">OwnRidgeTheta</span>
|
||
<span class="n">ypredictOwnRidge</span> <span class="o">=</span> <span class="n">X_test</span> <span class="o">@</span> <span class="n">OwnRidgeTheta</span>
|
||
<span class="n">ytildeRidge</span> <span class="o">=</span> <span class="n">RegRidge</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_train</span><span class="p">)</span>
|
||
<span class="n">ypredictRidge</span> <span class="o">=</span> <span class="n">RegRidge</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
|
||
<span class="n">MSEOwnRidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredictOwnRidge</span><span class="p">)</span>
|
||
<span class="n">MSERidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredictRidge</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Theta values for own Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">OwnRidgeTheta</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Theta values for Scikit-Learn Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">RegRidge</span><span class="o">.</span><span class="n">coef_</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE values for own Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSEOwnRidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE values for Scikit-Learn Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSERidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
|
||
|
||
<span class="c1"># Now plot the results</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">lambdas</span><span class="p">),</span> <span class="n">MSEOwnRidgePredict</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'MSE own Ridge Test'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">lambdas</span><span class="p">),</span> <span class="n">MSERidgePredict</span><span class="p">,</span> <span class="s1">'g'</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'MSE Ridge Test'</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">'log10(lambda)'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">'MSE'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The results here agree when we force <strong>Scikit-Learn</strong>’s Ridge function to include the first column in our design matrix.
|
||
We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.
|
||
What happens if we do not include the intercept in our fit? We will discuss this in more detail next week.</p>
|
||
</section>
|
||
<section id="basic-math-of-the-svd">
|
||
<h2>Basic math of the SVD<a class="headerlink" href="#basic-math-of-the-svd" title="Link to this heading">#</a></h2>
|
||
<p>From standard linear algebra we know that a square matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be diagonalized if and only if it is
|
||
a so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Normal_matrix">normal matrix</a>, that is if <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times n}\)</span>
|
||
we have <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X}\)</span> or if <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{C}}^{n\times n}\)</span> we have <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X}\)</span>.
|
||
The matrix has then a set of eigenpairs</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n),
|
||
\]</div>
|
||
<p>and the eigenvalues are given by the diagonal matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n).
|
||
\]</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be written in terms of an orthogonal/unitary transformation <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I}\)</span> or <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I}\)</span>.</p>
|
||
<p>Not all square matrices are diagonalizable. A matrix like the one discussed above</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X} = \begin{bmatrix}
|
||
1& -1 \\
|
||
1& -1\\
|
||
\end{bmatrix}
|
||
\end{split}\]</div>
|
||
<p>is not diagonalizable, it is a so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Defective_matrix">defective matrix</a>. It is easy to see that the condition
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X}\)</span> is not fulfilled.</p>
|
||
</section>
|
||
<section id="the-svd-a-fantastic-algorithm">
|
||
<h2>The SVD, a Fantastic Algorithm<a class="headerlink" href="#the-svd-a-fantastic-algorithm" title="Link to this heading">#</a></h2>
|
||
<p>However, and this is the strength of the SVD algorithm, any general
|
||
matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be decomposed in terms of a diagonal matrix and
|
||
two orthogonal/unitary matrices. The <a class="reference external" href="https://en.wikipedia.org/wiki/Singular_value_decomposition">Singular Value Decompostion
|
||
(SVD) theorem</a>
|
||
states that a general <span class="math notranslate nohighlight">\(m\times n\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be written in
|
||
terms of a diagonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> of dimensionality <span class="math notranslate nohighlight">\(m\times n\)</span>
|
||
and two orthognal matrices <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>, where the first has
|
||
dimensionality <span class="math notranslate nohighlight">\(m \times m\)</span> and the last dimensionality <span class="math notranslate nohighlight">\(n\times n\)</span>.
|
||
We have then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T
|
||
\]</div>
|
||
<p>As an example, the above defective matrix can be decomposed as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\end{split}\]</div>
|
||
<p>with eigenvalues <span class="math notranslate nohighlight">\(\sigma_1=2\)</span> and <span class="math notranslate nohighlight">\(\sigma_2=0\)</span>.
|
||
The SVD exits always!</p>
|
||
<p>The SVD
|
||
decomposition (singular values) gives eigenvalues
|
||
<span class="math notranslate nohighlight">\(\sigma_i\geq\sigma_{i+1}\)</span> for all <span class="math notranslate nohighlight">\(i\)</span> and for dimensions larger than <span class="math notranslate nohighlight">\(i=p\)</span>, the
|
||
eigenvalues (singular values) are zero.</p>
|
||
<p>In the general case, where our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has dimension
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span>, the matrix is thus decomposed into an <span class="math notranslate nohighlight">\(n\times n\)</span>
|
||
orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>, a <span class="math notranslate nohighlight">\(p\times p\)</span> orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>
|
||
and a diagonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> with <span class="math notranslate nohighlight">\(r=\mathrm{min}(n,p)\)</span>
|
||
singular values <span class="math notranslate nohighlight">\(\sigma_i\geq 0\)</span> on the main diagonal and zeros filling
|
||
the rest of the matrix. There are at most <span class="math notranslate nohighlight">\(p\)</span> singular values
|
||
assuming that <span class="math notranslate nohighlight">\(n > p\)</span>. In our regression examples for the nuclear
|
||
masses and the equation of state this is indeed the case, while for
|
||
the Ising model we have <span class="math notranslate nohighlight">\(p > n\)</span>. These are often cases that lead to
|
||
near singular or singular matrices.</p>
|
||
<p>The columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> are called the left singular vectors while the columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are the right singular vectors.</p>
|
||
</section>
|
||
<section id="economy-size-svd">
|
||
<h2>Economy-size SVD<a class="headerlink" href="#economy-size-svd" title="Link to this heading">#</a></h2>
|
||
<p>If we assume that <span class="math notranslate nohighlight">\(n > p\)</span>, then our matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> has dimension <span class="math notranslate nohighlight">\(n
|
||
\times n\)</span>. The last <span class="math notranslate nohighlight">\(n-p\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> become however
|
||
irrelevant in our calculations since they are multiplied with the
|
||
zeros in <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span>.</p>
|
||
<p>The economy-size decomposition removes extra rows or columns of zeros
|
||
from the diagonal matrix of singular values, <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span>, along with the columns
|
||
in either <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> or <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> that multiply those zeros in the expression.
|
||
Removing these zeros and columns can improve execution time
|
||
and reduce storage requirements without compromising the accuracy of
|
||
the decomposition.</p>
|
||
<p>If <span class="math notranslate nohighlight">\(n > p\)</span>, we keep only the first <span class="math notranslate nohighlight">\(p\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> has dimension <span class="math notranslate nohighlight">\(p\times p\)</span>.
|
||
If <span class="math notranslate nohighlight">\(p > n\)</span>, then only the first <span class="math notranslate nohighlight">\(n\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are computed and <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> has dimension <span class="math notranslate nohighlight">\(n\times n\)</span>.
|
||
The <span class="math notranslate nohighlight">\(n=p\)</span> case is obvious, we retain the full SVD.
|
||
In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy.</p>
|
||
</section>
|
||
<section id="codes-for-the-svd">
|
||
<h2>Codes for the SVD<a class="headerlink" href="#codes-for-the-svd" title="Link to this heading">#</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="c1"># SVD inversion</span>
|
||
<span class="k">def</span> <span class="nf">SVD</span><span class="p">(</span><span class="n">A</span><span class="p">):</span>
|
||
<span class="w"> </span><span class="sd">''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).</span>
|
||
<span class="sd"> SVD is numerically more stable than the inversion algorithms provided by</span>
|
||
<span class="sd"> numpy and scipy.linalg at the cost of being slower.</span>
|
||
<span class="sd"> '''</span>
|
||
<span class="n">U</span><span class="p">,</span> <span class="n">S</span><span class="p">,</span> <span class="n">VT</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">svd</span><span class="p">(</span><span class="n">A</span><span class="p">,</span><span class="n">full_matrices</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'test U'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">U</span><span class="p">)</span> <span class="o">@</span> <span class="n">U</span> <span class="o">-</span> <span class="n">U</span> <span class="nd">@np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">U</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'test VT'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">VT</span><span class="p">)</span> <span class="o">@</span> <span class="n">VT</span> <span class="o">-</span> <span class="n">VT</span> <span class="nd">@np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">VT</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">U</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">S</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">VT</span><span class="p">)</span>
|
||
|
||
<span class="n">D</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">U</span><span class="p">),</span><span class="nb">len</span><span class="p">(</span><span class="n">VT</span><span class="p">)))</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="nb">len</span><span class="p">(</span><span class="n">VT</span><span class="p">)):</span>
|
||
<span class="n">D</span><span class="p">[</span><span class="n">i</span><span class="p">,</span><span class="n">i</span><span class="p">]</span><span class="o">=</span><span class="n">S</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
<span class="k">return</span> <span class="n">U</span> <span class="o">@</span> <span class="n">D</span> <span class="o">@</span> <span class="n">VT</span>
|
||
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span><span class="o">-</span><span class="mf">1.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span><span class="o">-</span><span class="mf">1.0</span><span class="p">]])</span>
|
||
<span class="c1">#X = np.array([[1, 2], [3, 4], [5, 6]])</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">SVD</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="c1"># Print the difference between the original matrix and the SVD one</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="o">-</span><span class="n">X</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has columns that are linearly dependent. The first
|
||
column is the row-wise sum of the other two columns. The rank of a
|
||
matrix (the column rank) is the dimension of space spanned by the
|
||
column vectors. The rank of the matrix is the number of linearly
|
||
independent columns, in this case just <span class="math notranslate nohighlight">\(2\)</span>. We see this from the
|
||
singular values when running the above code. Running the standard
|
||
inversion algorithm for matrix inversion with <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> results
|
||
in the program terminating due to a singular matrix.</p>
|
||
</section>
|
||
<section id="note-about-svd-calculations">
|
||
<h2>Note about SVD Calculations<a class="headerlink" href="#note-about-svd-calculations" title="Link to this heading">#</a></h2>
|
||
<p>The <span class="math notranslate nohighlight">\(U\)</span>, <span class="math notranslate nohighlight">\(S\)</span>, and <span class="math notranslate nohighlight">\(V\)</span> matrices returned from the <strong>svd()</strong> function
|
||
cannot be multiplied directly.</p>
|
||
<p>As you can see from the code, the <span class="math notranslate nohighlight">\(S\)</span> vector must be converted into a
|
||
diagonal matrix. This may cause a problem as the size of the matrices
|
||
do not fit the rules of matrix multiplication, where the number of
|
||
columns in a matrix must match the number of rows in the subsequent
|
||
matrix.</p>
|
||
<p>If you wish to include the zero singular values, you will need to
|
||
resize the matrices and set up a diagonal matrix as done in the above
|
||
example</p>
|
||
</section>
|
||
<section id="mathematics-of-the-svd-and-implications">
|
||
<h2>Mathematics of the SVD and implications<a class="headerlink" href="#mathematics-of-the-svd-and-implications" title="Link to this heading">#</a></h2>
|
||
<p>Let us take a closer look at the mathematics of the SVD and the various implications for machine learning studies.</p>
|
||
<p>Our starting point is our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> of dimension <span class="math notranslate nohighlight">\(n\times p\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\
|
||
x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\
|
||
x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\
|
||
\dots & \dots & \dots & \dots \dots & \dots \\
|
||
x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\
|
||
x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>We can SVD decompose our matrix as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> is an orthogonal matrix of dimension <span class="math notranslate nohighlight">\(n\times n\)</span>, meaning that <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{U}^T\boldsymbol{U}=\boldsymbol{I}_n\)</span>. Here <span class="math notranslate nohighlight">\(\boldsymbol{I}_n\)</span> is the unit matrix of dimension <span class="math notranslate nohighlight">\(n \times n\)</span>.</p>
|
||
<p>Similarly, <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> is an orthogonal matrix of dimension <span class="math notranslate nohighlight">\(p\times p\)</span>, meaning that <span class="math notranslate nohighlight">\(\boldsymbol{V}\boldsymbol{V}^T=\boldsymbol{V}^T\boldsymbol{V}=\boldsymbol{I}_p\)</span>. Here <span class="math notranslate nohighlight">\(\boldsymbol{I}_n\)</span> is the unit matrix of dimension <span class="math notranslate nohighlight">\(p \times p\)</span>.</p>
|
||
<p>Finally <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> contains the singular values <span class="math notranslate nohighlight">\(\sigma_i\)</span>. This matrix has dimension <span class="math notranslate nohighlight">\(n\times p\)</span> and the singular values <span class="math notranslate nohighlight">\(\sigma_i\)</span> are all positive. The non-zero values are ordered in descending order, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sigma_0 > \sigma_1 > \sigma_2 > \dots > \sigma_{p-1} > 0.
|
||
\]</div>
|
||
<p>All values beyond <span class="math notranslate nohighlight">\(p-1\)</span> are all zero.</p>
|
||
</section>
|
||
<section id="example-matrix">
|
||
<h2>Example Matrix<a class="headerlink" href="#example-matrix" title="Link to this heading">#</a></h2>
|
||
<p>As an example, consider the following <span class="math notranslate nohighlight">\(3\times 2\)</span> example for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}=
|
||
\begin{bmatrix}
|
||
2& 0 \\
|
||
0 & 1 \\
|
||
0 & 0 \\
|
||
\end{bmatrix}
|
||
\end{split}\]</div>
|
||
<p>The singular values are <span class="math notranslate nohighlight">\(\sigma_0=2\)</span> and <span class="math notranslate nohighlight">\(\sigma_1=1\)</span>. It is common to rewrite the matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}=
|
||
\begin{bmatrix}
|
||
\boldsymbol{\tilde{\Sigma}}\\
|
||
\boldsymbol{0}\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\tilde{\Sigma}}=
|
||
\begin{bmatrix}
|
||
2& 0 \\
|
||
0 & 1 \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>contains only the singular values. Note also (and we will use this below) that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}=
|
||
\begin{bmatrix}
|
||
4& 0 \\
|
||
0 & 1 \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which is a <span class="math notranslate nohighlight">\(2\times 2 \)</span> matrix while</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T=
|
||
\begin{bmatrix}
|
||
4& 0 & 0\\
|
||
0 & 1 & 0\\
|
||
0 & 0 & 0\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>is a <span class="math notranslate nohighlight">\(3\times 3 \)</span> matrix. The last row and column of this last matrix
|
||
contain only zeros. This will have important consequences for our SVD
|
||
decomposition of the design matrix.</p>
|
||
</section>
|
||
<section id="setting-up-the-matrix-to-be-inverted">
|
||
<h2>Setting up the Matrix to be inverted<a class="headerlink" href="#setting-up-the-matrix-to-be-inverted" title="Link to this heading">#</a></h2>
|
||
<p>The matrix that may cause problems for us is <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. Using the SVD we can rewrite this matrix as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>and using the orthogonality of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>We define <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}=\tilde{\boldsymbol{\Sigma}}^2\)</span> which is a diagonal matrix containing only the singular values squared. It has dimensionality <span class="math notranslate nohighlight">\(p \times p\)</span>.</p>
|
||
<p>We can now insert the result for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> into our equation for ordinary least squares where</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_{\mathrm{OLS}}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>and using our SVD decomposition of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\left(\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^{2}(\boldsymbol{V}^T\right)^{-1}\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>which gives us, using the orthogonality of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_{\mathrm{OLS}}=\sum_{i=0}^{p-1}\boldsymbol{u}_i\boldsymbol{u}^T_i\boldsymbol{y},
|
||
\]</div>
|
||
<p>which is not the same as <span class="math notranslate nohighlight">\(\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}\)</span>, which due to the orthogonality of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> would have given us that the model equals the output.</p>
|
||
<p>It means that the ordinary least square model (with the optimal
|
||
parameters) <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}}\)</span>, corresponds to an orthogonal
|
||
transformation of the output (or target) vector <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> by the
|
||
vectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>. <strong>Note that the summation ends at</strong>
|
||
<span class="math notranslate nohighlight">\(p-1\)</span>, that is <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}}\ne \boldsymbol{y}\)</span>. We can thus not use the
|
||
orthogonality relation for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>.</p>
|
||
</section>
|
||
<section id="further-properties-important-for-our-analyses-later">
|
||
<h2>Further properties (important for our analyses later)<a class="headerlink" href="#further-properties-important-for-our-analyses-later" title="Link to this heading">#</a></h2>
|
||
<p>Let us study again <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> in terms of our SVD,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>If we now multiply from the right with <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> (using the orthogonality of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>) we get</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}.
|
||
\]</div>
|
||
<p>This means the vectors <span class="math notranslate nohighlight">\(\boldsymbol{v}_i\)</span> of the orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are the eigenvectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>
|
||
with eigenvalues given by the singular values squared, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2.
|
||
\]</div>
|
||
<p>Similarly, if we use the SVD decomposition for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T\boldsymbol{U}^T.
|
||
\]</div>
|
||
<p>If we now multiply from the right with <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> (using the orthogonality of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>) we get</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{U}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T.
|
||
\]</div>
|
||
<p>This means the vectors <span class="math notranslate nohighlight">\(\boldsymbol{u}_i\)</span> of the orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> are the eigenvectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span>
|
||
with eigenvalues given by the singular values squared, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{u}_i=\boldsymbol{u}_i\sigma_i^2.
|
||
\]</div>
|
||
<p><strong>Important note</strong>: we have defined our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> to be an
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span> matrix. In most supervised learning cases we have that <span class="math notranslate nohighlight">\(n
|
||
\ge p\)</span>, and quite often we have <span class="math notranslate nohighlight">\(n >> p\)</span>. For linear algebra based methods like ordinary least squares or Ridge regression, this leads to a matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> which is small and thereby easier to handle from a computational point of view (in terms of number of floating point operations).</p>
|
||
<p>In our lectures, the number of columns will
|
||
always refer to the number of features in our data set, while the
|
||
number of rows represents the number of data inputs. Note that in
|
||
other texts you may find the opposite notation. This has consequences
|
||
for the definition of for example the covariance matrix and its relation to the SVD.</p>
|
||
</section>
|
||
<section id="meet-the-covariance-matrix">
|
||
<h2>Meet the Covariance Matrix<a class="headerlink" href="#meet-the-covariance-matrix" title="Link to this heading">#</a></h2>
|
||
<p>Before we move on to a discussion of Ridge and Lasso regression, we want to show an important example of the above.</p>
|
||
<p>We have already noted that the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> in ordinary
|
||
least squares is proportional to the second derivative of the cost
|
||
function, that is we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial^2 C(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}\partial \boldsymbol{\theta}^T} =\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>This quantity defines was what is called the Hessian matrix (the second derivative of a function we want to optimize).</p>
|
||
<p>The Hessian matrix plays an important role and is defined in this course as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>The Hessian matrix for ordinary least squares is also proportional to
|
||
the covariance matrix. This means also that we can use the SVD to find
|
||
the eigenvalues of the covariance matrix and the Hessian matrix in
|
||
terms of the singular values. Let us develop these arguments, as they will play an important role in our machine learning studies.</p>
|
||
</section>
|
||
<section id="introducing-the-covariance-and-correlation-functions">
|
||
<h2>Introducing the Covariance and Correlation functions<a class="headerlink" href="#introducing-the-covariance-and-correlation-functions" title="Link to this heading">#</a></h2>
|
||
<p>Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about
|
||
the definition of the covariance and the correlation function. These are quantities that play a central role in machine learning methods.</p>
|
||
<p>Suppose we have defined two vectors
|
||
<span class="math notranslate nohighlight">\(\hat{x}\)</span> and <span class="math notranslate nohighlight">\(\hat{y}\)</span> with <span class="math notranslate nohighlight">\(n\)</span> elements each. The covariance matrix <span class="math notranslate nohighlight">\(\boldsymbol{C}\)</span> is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where for example</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}).
|
||
\]</div>
|
||
<p>With this definition and recalling that the variance is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2,
|
||
\]</div>
|
||
<p>we can rewrite the covariance matrix as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p><strong>Note:</strong> we have used <span class="math notranslate nohighlight">\(1/n\)</span> in the above definitions of the <em>sample</em> variance and covariance. We assume then that we can calculate the exact mean value.
|
||
What you will find in essentially all statistics texts are equations
|
||
with a factor <span class="math notranslate nohighlight">\(1/(n-1)\)</span>. This is called <a class="reference external" href="https://mathworld.wolfram.com/BesselsCorrection.html">Bessel’s correction</a>. This
|
||
method corrects the bias in the estimation of the population variance
|
||
and covariance. It also partially corrects the bias in the estimation
|
||
of the population standard deviation. If you use a library like
|
||
<strong>Scikit-Learn</strong> or <strong>nunmpy’s</strong> function to calculate the covariance, this
|
||
quantity will be computed with a factor <span class="math notranslate nohighlight">\(1/(n-1)\)</span>.</p>
|
||
</section>
|
||
<section id="covariance-and-correlation-matrix">
|
||
<h2>Covariance and Correlation Matrix<a class="headerlink" href="#covariance-and-correlation-matrix" title="Link to this heading">#</a></h2>
|
||
<p>The covariance takes values between zero and infinity and may thus
|
||
lead to problems with loss of numerical precision for particularly
|
||
large values. It is common to scale the covariance matrix by
|
||
introducing instead the correlation matrix defined via the so-called
|
||
correlation function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}.
|
||
\]</div>
|
||
<p>The correlation function is then given by values <span class="math notranslate nohighlight">\(\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]
|
||
\in [-1,1]\)</span>. This avoids eventual problems with too large values. We
|
||
can then define the correlation matrix for the two vectors <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span>
|
||
and <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>In the above example this is the function we constructed using <strong>pandas</strong>.</p>
|
||
</section>
|
||
<section id="correlation-function-and-design-feature-matrix">
|
||
<h2>Correlation Function and Design/Feature Matrix<a class="headerlink" href="#correlation-function-and-design-feature-matrix" title="Link to this heading">#</a></h2>
|
||
<p>In our derivation of the various regression algorithms like <strong>Ordinary Least Squares</strong> or <strong>Ridge regression</strong>
|
||
we defined the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\
|
||
x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\
|
||
x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\
|
||
\dots & \dots & \dots & \dots \dots & \dots \\
|
||
x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\
|
||
x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>, with the predictors/features <span class="math notranslate nohighlight">\(p\)</span> refering to the column numbers and the
|
||
entries <span class="math notranslate nohighlight">\(n\)</span> being the row elements.
|
||
We can rewrite the design/feature matrix in terms of its column vectors as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}=\begin{bmatrix} \boldsymbol{x}_0 & \boldsymbol{x}_1 & \boldsymbol{x}_2 & \dots & \dots & \boldsymbol{x}_{p-1}\end{bmatrix},
|
||
\]</div>
|
||
<p>with a given vector</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}.
|
||
\]</div>
|
||
<p>With these definitions, we can now rewrite our <span class="math notranslate nohighlight">\(2\times 2\)</span>
|
||
correlation/covariance matrix in terms of a moe general design/feature
|
||
matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>. This leads to a <span class="math notranslate nohighlight">\(p\times p\)</span>
|
||
covariance matrix for the vectors <span class="math notranslate nohighlight">\(\boldsymbol{x}_i\)</span> with <span class="math notranslate nohighlight">\(i=0,1,\dots,p-1\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix}
|
||
\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>and the correlation matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix}
|
||
1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
</section>
|
||
<section id="covariance-matrix-examples">
|
||
<h2>Covariance Matrix Examples<a class="headerlink" href="#covariance-matrix-examples" title="Link to this heading">#</a></h2>
|
||
<p>The Numpy function <strong>np.cov</strong> calculates the covariance elements using
|
||
the factor <span class="math notranslate nohighlight">\(1/(n-1)\)</span> instead of <span class="math notranslate nohighlight">\(1/n\)</span> since it assumes we do not have
|
||
the exact mean values. The following simple function uses the
|
||
<strong>np.vstack</strong> function which takes each vector of dimension <span class="math notranslate nohighlight">\(1\times n\)</span>
|
||
and produces a <span class="math notranslate nohighlight">\(2\times n\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{W}\)</span></p>
|
||
<p>Note that this assumes you have the features as the rows, and the inputs as columns, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{W} = \begin{bmatrix} x_0 & x_1 & x_2 & \dots & x_{n-2} & x_{n-1} \\
|
||
y_0 & y_1 & y_2 & \dots & y_{n-2} & y_{n-1} \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which in turn is converted into into the <span class="math notranslate nohighlight">\(2\times 2\)</span> covariance matrix
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{C}\)</span> via the Numpy function <strong>np.cov()</strong>. We note that we can also calculate
|
||
the mean value of each set of samples <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> etc using the Numpy
|
||
function <strong>np.mean(x)</strong>. We can also extract the eigenvalues of the
|
||
covariance matrix through the <strong>np.linalg.eig()</strong> function.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Importing various packages</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">))</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">((</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">))</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">cov</span><span class="p">(</span><span class="n">W</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="correlation-matrix">
|
||
<h2>Correlation Matrix<a class="headerlink" href="#correlation-matrix" title="Link to this heading">#</a></h2>
|
||
<p>The previous example can be converted into the correlation matrix by
|
||
simply scaling the matrix elements with the variances. We should also
|
||
subtract the mean values for each column. This leads to the following
|
||
code which sets up the correlations matrix for the previous example in
|
||
a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the <span class="math notranslate nohighlight">\(2\times 2\)</span> correlation matrix (since we have only two vectors).</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="c1"># define two vectors </span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="c1">#scaling the x and y vectors </span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">y</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="n">variance_x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@x</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">variance_y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">y</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">variance_x</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">variance_y</span><span class="p">)</span>
|
||
<span class="n">cov_xy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">cov_xx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@x</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">cov_yy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">y</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_xx</span><span class="o">/</span><span class="n">variance_x</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_yy</span><span class="o">/</span><span class="n">variance_y</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_xy</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">variance_y</span><span class="o">*</span><span class="n">variance_x</span><span class="p">)</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">=</span> <span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We see that the matrix elements along the diagonal are one as they
|
||
should be and that the matrix is symmetric. Furthermore, diagonalizing
|
||
this matrix we easily see that it is a positive definite matrix.</p>
|
||
<p>The above procedure with <strong>numpy</strong> can be made more compact if we use <strong>pandas</strong>.</p>
|
||
</section>
|
||
<section id="correlation-matrix-with-pandas">
|
||
<h2>Correlation Matrix with Pandas<a class="headerlink" href="#correlation-matrix-with-pandas" title="Link to this heading">#</a></h2>
|
||
<p>We whow here how we can set up the correlation matrix using <strong>pandas</strong>, as done in this simple code</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">10</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">y</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="c1"># Note that we transpose the matrix in order to stay with our ordering n x p</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">((</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)))</span><span class="o">.</span><span class="n">T</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">Xpd</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">Xpd</span><span class="p">)</span>
|
||
<span class="n">correlation_matrix</span> <span class="o">=</span> <span class="n">Xpd</span><span class="o">.</span><span class="n">corr</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">correlation_matrix</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="rewriting-the-covariance-and-or-correlation-matrix">
|
||
<h2>Rewriting the Covariance and/or Correlation Matrix<a class="headerlink" href="#rewriting-the-covariance-and-or-correlation-matrix" title="Link to this heading">#</a></h2>
|
||
<p>We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{C}[\boldsymbol{x}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}= \mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}].
|
||
\]</div>
|
||
<p>To see this let us simply look at a design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{2\times 2}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{00} & x_{01}\\
|
||
x_{10} & x_{11}\\
|
||
\end{bmatrix}=\begin{bmatrix}
|
||
\boldsymbol{x}_{0} & \boldsymbol{x}_{1}\\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>If we then compute the expectation value (note the <span class="math notranslate nohighlight">\(1/n\)</span> factor instead of <span class="math notranslate nohighlight">\(1/(n-1)\)</span>)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}=\frac{1}{n}\begin{bmatrix}
|
||
x_{00}^2+x_{10}^2 & x_{00}x_{01}+x_{10}x_{11}\\
|
||
x_{01}x_{00}+x_{11}x_{10} & x_{01}^2+x_{11}^2\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which is just</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]=\begin{bmatrix} \mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] \\
|
||
\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where we wrote $<span class="math notranslate nohighlight">\(\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]\)</span><span class="math notranslate nohighlight">\( to indicate that this is the covariance of the vectors \)</span>\boldsymbol{x}<span class="math notranslate nohighlight">\( of the design/feature matrix \)</span>\boldsymbol{X}$.</p>
|
||
<p>It is easy to generalize this to a matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>.</p>
|
||
</section>
|
||
<section id="linking-with-the-svd">
|
||
<h2>Linking with the SVD<a class="headerlink" href="#linking-with-the-svd" title="Link to this heading">#</a></h2>
|
||
<p>We saw earlier that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>Since the matrices here have dimension <span class="math notranslate nohighlight">\(p\times p\)</span>, with <span class="math notranslate nohighlight">\(p\)</span> corresponding to the singular values, we defined earlier the matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}^T\boldsymbol{\Sigma} = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where the tilde-matrix <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{\Sigma}}\)</span> is a matrix of dimension <span class="math notranslate nohighlight">\(p\times p\)</span> containing only the singular values <span class="math notranslate nohighlight">\(\sigma_i\)</span>, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\tilde{\boldsymbol{\Sigma}}=\begin{bmatrix} \sigma_0 & 0 & 0 & \dots & 0 & 0 \\
|
||
0 & \sigma_1 & 0 & \dots & 0 & 0 \\
|
||
0 & 0 & \sigma_2 & \dots & 0 & 0 \\
|
||
0 & 0 & 0 & \dots & \sigma_{p-2} & 0 \\
|
||
0 & 0 & 0 & \dots & 0 & \sigma_{p-1} \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>meaning we can write</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>Multiplying from the right with <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> (using the orthogonality of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>) we get</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2.
|
||
\]</div>
|
||
</section>
|
||
<section id="what-does-it-mean">
|
||
<h2>What does it mean?<a class="headerlink" href="#what-does-it-mean" title="Link to this heading">#</a></h2>
|
||
<p>This means the vectors <span class="math notranslate nohighlight">\(\boldsymbol{v}_i\)</span> of the orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>
|
||
are the eigenvectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> with eigenvalues
|
||
given by the singular values squared, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2.
|
||
\]</div>
|
||
<p>In other words, each non-zero singular value of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is a positive
|
||
square root of an eigenvalue of <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. It means also that
|
||
the columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are the eigenvectors of
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. Since we have ordered the singular values of
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> in a descending order, it means that the column vectors
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{v}_i\)</span> are hierarchically ordered by how much correlation they
|
||
encode from the columns of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
<p>Note that these are also the eigenvectors and eigenvalues of the
|
||
Hessian matrix. Note also that the Hessian matrix we are discussing here is from a cost function defined by the mean squared error only.</p>
|
||
<p>If we now recall the definition of the covariance matrix (not using
|
||
Bessel’s correction) we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{C}[\boldsymbol{X}]=\frac{1}{n}\boldsymbol{X}^T\boldsymbol{X},
|
||
\]</div>
|
||
<p>meaning that every squared non-singular value of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> divided by <span class="math notranslate nohighlight">\(n\)</span> (
|
||
the number of samples) are the eigenvalues of the covariance
|
||
matrix. Every singular value of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is thus a positive square
|
||
root of an eigenvalue of <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. If the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is
|
||
self-adjoint, the singular values of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> are equal to the
|
||
absolute value of the eigenvalues of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</section>
|
||
<section id="and-finally-boldsymbol-x-boldsymbol-x-t">
|
||
<h2>And finally <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span><a class="headerlink" href="#and-finally-boldsymbol-x-boldsymbol-x-t" title="Link to this heading">#</a></h2>
|
||
<p>For <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span> we found</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{U}^T.
|
||
\]</div>
|
||
<p>Since the matrices here have dimension <span class="math notranslate nohighlight">\(n\times n\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \boldsymbol{0}\\ \end{bmatrix}=\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>leading to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}\boldsymbol{U}^T.
|
||
\end{split}\]</div>
|
||
<p>Multiplying with <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> from the right gives us the eigenvalue problem</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U}=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>It means that the eigenvalues of <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span> are again given by
|
||
the non-zero singular values plus now a series of zeros. The column
|
||
vectors of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> are the eigenvectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span> and
|
||
measure how much correlations are contained in the rows of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
<p>Since we will mainly be interested in the correlations among the features
|
||
of our data (the columns of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>, the quantity of interest for us are the non-zero singular
|
||
values and the column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>.</p>
|
||
</section>
|
||
<section id="back-to-ridge-and-lasso-regression">
|
||
<h2>Back to Ridge and LASSO Regression<a class="headerlink" href="#back-to-ridge-and-lasso-regression" title="Link to this heading">#</a></h2>
|
||
<p>Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
|
||
our optimization problem is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\right)\right\}.
|
||
\]</div>
|
||
<p>or we can state it as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2,
|
||
\]</div>
|
||
<p>where we have used the definition of a norm-2 vector, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
||
\]</div>
|
||
<p>By minimizing the above equation with respect to the parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> we could then obtain an analytical expression for the
|
||
parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. We can add a regularization parameter <span class="math notranslate nohighlight">\(\lambda\)</span> by
|
||
defining a new cost function to be optimized, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\theta}\vert\vert_2^2
|
||
\]</div>
|
||
<p>which leads to the Ridge regression minimization problem where we
|
||
require that <span class="math notranslate nohighlight">\(\vert\vert \boldsymbol{\theta}\vert\vert_2^2\le t\)</span>, where <span class="math notranslate nohighlight">\(t\)</span> is
|
||
a finite number larger than zero. By defining</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\theta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\theta}\vert\vert_1,
|
||
\]</div>
|
||
<p>we have a new optimization equation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\theta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\theta}\vert\vert_1
|
||
\]</div>
|
||
<p>which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator.</p>
|
||
<p>Here we have defined the norm-1 as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
|
||
\]</div>
|
||
<p>Ridge regression, as discussed above, is nothing but the standard OLS with a
|
||
modified diagonal term added to <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. The consequences, in
|
||
particular for our discussion of the bias-variance tradeoff are rather
|
||
interesting. We will see that for specific values of <span class="math notranslate nohighlight">\(\lambda\)</span>, we may
|
||
even reduce the variance of the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span>. These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.</p>
|
||
<p>Using our insights about the SVD of the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>
|
||
We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}_{\mathrm{OLS}}=\boldsymbol{X}\boldsymbol{\theta} =\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>For Ridge regression this becomes</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}_{\mathrm{Ridge}}=\boldsymbol{X}\boldsymbol{\theta}_{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{\Sigma}^2\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y},
|
||
\]</div>
|
||
<p>with the vectors <span class="math notranslate nohighlight">\(\boldsymbol{u}_j\)</span> being the columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> from the SVD of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</section>
|
||
<section id="interpreting-the-ridge-results">
|
||
<h2>Interpreting the Ridge results<a class="headerlink" href="#interpreting-the-ridge-results" title="Link to this heading">#</a></h2>
|
||
<p>Since <span class="math notranslate nohighlight">\(\lambda \geq 0\)</span>, it means that compared to OLS, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1.
|
||
\]</div>
|
||
<p>Ridge regression finds the coordinates of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> with respect to the
|
||
orthonormal basis <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>, it then shrinks the coordinates by
|
||
<span class="math notranslate nohighlight">\(\frac{\sigma_j^2}{\sigma_j^2+\lambda}\)</span>. Recall that the SVD has
|
||
eigenvalues ordered in a descending way, that is <span class="math notranslate nohighlight">\(\sigma_i \geq
|
||
\sigma_{i+1}\)</span>.</p>
|
||
<p>For small eigenvalues <span class="math notranslate nohighlight">\(\sigma_i\)</span> it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.</p>
|
||
</section>
|
||
<section id="more-interpretations">
|
||
<h2>More interpretations<a class="headerlink" href="#more-interpretations" title="Link to this heading">#</a></h2>
|
||
<p>For the sake of simplicity, let us assume that the design matrix is orthonormal, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}.
|
||
\]</div>
|
||
<p>In this case the standard OLS results in</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\theta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{n-1}\boldsymbol{u}_i\boldsymbol{u}_i^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\theta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\theta}^{\mathrm{OLS}},
|
||
\]</div>
|
||
<p>that is the Ridge estimator scales the OLS estimator by the inverse of a factor <span class="math notranslate nohighlight">\(1+\lambda\)</span>, and
|
||
the Ridge estimator converges to zero when the hyperparameter goes to
|
||
infinity.</p>
|
||
<p>We will come back to more interpreations after we have gone through some of the statistical analysis part.</p>
|
||
<p>For more discussions of Ridge and Lasso regression, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.
|
||
Similarly, <a class="reference external" href="https://arxiv.org/abs/1803.08823">Mehta et al’s article</a> is also recommended.</p>
|
||
</section>
|
||
<section id="deriving-the-lasso-regression-equations">
|
||
<h2>Deriving the Lasso Regression Equations<a class="headerlink" href="#deriving-the-lasso-regression-equations" title="Link to this heading">#</a></h2>
|
||
<p>Using the matrix-vector expression for Lasso regression, we have the following <strong>cost</strong> function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\theta})=\frac{1}{n}\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta})\right\}+\lambda\vert\vert\boldsymbol{\theta}\vert\vert_1,
|
||
\]</div>
|
||
<p>Taking the derivative with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\frac{d \vert \theta\vert}{d \theta}=\mathrm{sgn}(\theta)=\left\{\begin{array}{cc} 1 & \theta > 0 \\-1 & \theta < 0, \end{array}\right.
|
||
\end{split}\]</div>
|
||
<p>we have that the derivative of the cost function is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{X},\boldsymbol{\theta})}{\partial \boldsymbol{\theta}}=-\frac{2}{n}\boldsymbol{X}^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\theta})+\lambda sgn(\boldsymbol{\theta})=0,
|
||
\]</div>
|
||
<p>and reordering we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\theta}+\frac{n}{2}\lambda sgn(\boldsymbol{\theta})=2\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>We can redefine <span class="math notranslate nohighlight">\(\lambda\)</span> to absorb the constant <span class="math notranslate nohighlight">\(n/2\)</span> and we rewrite the last equation as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\theta}+\lambda sgn(\boldsymbol{\theta})=2\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package <a class="reference external" href="https://cvxopt.org/">CVXOPT</a>. We will discuss how to code LASSO regression next week, when we have introduced gradient methods.</p>
|
||
</section>
|
||
<section id="material-for-exercises-week-35">
|
||
<h2>Material for exercises week 35<a class="headerlink" href="#material-for-exercises-week-35" title="Link to this heading">#</a></h2>
|
||
</section>
|
||
<section id="important-technicalities-more-on-rescaling-data">
|
||
<h2>Important technicalities: More on Rescaling data<a class="headerlink" href="#important-technicalities-more-on-rescaling-data" title="Link to this heading">#</a></h2>
|
||
<p>When you are comparing your own code with for example <strong>Scikit-Learn</strong>’s
|
||
library, there are some technicalities to keep in mind. The examples
|
||
here demonstrate some of these aspects with potential pitfalls.</p>
|
||
<p>The discussion here focuses on the role of the intercept, how we can
|
||
set up the design matrix, what scaling we should use and other topics
|
||
which tend confuse us.</p>
|
||
<p>The intercept can be interpreted as the expected value of our
|
||
target/output variables when all other predictors are set to zero.
|
||
Thus, if we cannot assume that the expected outputs/targets are zero
|
||
when all predictors are zero (the columns in the design matrix), it
|
||
may be a bad idea to implement a model which penalizes the intercept.
|
||
Furthermore, in for example Ridge and Lasso regression, the default solutions
|
||
from the library <strong>Scikit-Learn</strong> (when not shrinking <span class="math notranslate nohighlight">\(\beta_0\)</span>) for the unknown parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>, are derived under the assumption that both <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> and
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> are zero centered, that is we subtract the mean values.</p>
|
||
<p>If our predictors represent different scales, then it is important to
|
||
standardize the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> by subtracting the mean of each
|
||
column from the corresponding column and dividing the column with its
|
||
standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library,
|
||
the results may differ.</p>
|
||
<p>The
|
||
<a class="reference external" href="https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html">Standardscaler</a>
|
||
function in <strong>Scikit-Learn</strong> does this for us. For the data sets we
|
||
have been studying in our various examples, the data are in many cases
|
||
already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a
|
||
survey of your data, with a critical assessment of them in case you need to scale the data.</p>
|
||
<p>If you need to scale the data, not doing so will give an <em>unfair</em>
|
||
penalization of the parameters since their magnitude depends on the
|
||
scale of their corresponding predictor.</p>
|
||
<p>The <strong>Scikit-Learn</strong> site <a class="reference external" href="https://scikit-learn.org/stable/auto_examples/preprocessing/plot_all_scaling.html#plot-all-scaling-standard-scaler-section">https://scikit-learn.org/stable/auto_examples/preprocessing/plot_all_scaling.html#plot-all-scaling-standard-scaler-section</a> has a good discussion of different ways of preprocessing data.</p>
|
||
<p>Suppose as an example that you
|
||
you have an input variable given by the heights of different persons.
|
||
Human height might be measured in inches or meters or
|
||
kilometers. If measured in kilometers, a standard linear regression
|
||
model with this predictor would probably give a much bigger
|
||
coefficient term, than if measured in millimeters.
|
||
This can clearly lead to problems in evaluating the cost/loss functions.</p>
|
||
<p>Keep in mind that when you transform your data set before training a model, the same transformation needs to be done
|
||
on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="sd">"""</span>
|
||
<span class="sd">#Model training, we compute the mean value of y and X</span>
|
||
<span class="sd">y_train_mean = np.mean(y_train)</span>
|
||
<span class="sd">X_train_mean = np.mean(X_train,axis=0)</span>
|
||
<span class="sd">X_train = X_train - X_train_mean</span>
|
||
<span class="sd">y_train = y_train - y_train_mean</span>
|
||
|
||
<span class="sd"># The we fit our model with the training data</span>
|
||
<span class="sd">trained_model = some_model.fit(X_train,y_train)</span>
|
||
|
||
|
||
<span class="sd">#Model prediction, we need also to transform our data set used for the prediction.</span>
|
||
<span class="sd">X_test = X_test - X_train_mean #Use mean from training data</span>
|
||
<span class="sd">y_pred = trained_model(X_test)</span>
|
||
<span class="sd">y_pred = y_pred + y_train_mean</span>
|
||
<span class="sd">"""</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Let us try to understand what this may imply mathematically when we
|
||
subtract the mean values, also known as <em>zero centering</em>. For
|
||
simplicity, we will focus on ordinary regression, as done in the above example.</p>
|
||
<p>The cost/loss function for regression is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,.
|
||
\]</div>
|
||
<p>Recall also that we use the squared value. This expression can lead to an
|
||
increased penalty for higher differences between predicted and
|
||
output/target values.</p>
|
||
<p>What we have done is to single out the <span class="math notranslate nohighlight">\(\beta_0\)</span> term in the
|
||
definition of the mean squared error (MSE). The design matrix <span class="math notranslate nohighlight">\(X\)</span>
|
||
does in this case not contain any intercept column. When we take the
|
||
derivative with respect to <span class="math notranslate nohighlight">\(\beta_0\)</span>, we want the derivative to obey</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C}{\partial \beta_j} = 0,
|
||
\]</div>
|
||
<p>for all <span class="math notranslate nohighlight">\(j\)</span>. For <span class="math notranslate nohighlight">\(\beta_0\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right).
|
||
\]</div>
|
||
<p>Multiplying away the constant <span class="math notranslate nohighlight">\(2/n\)</span>, we obtain</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j.
|
||
\]</div>
|
||
<p>Let us specialize first to the case where we have only two parameters <span class="math notranslate nohighlight">\(\beta_0\)</span> and <span class="math notranslate nohighlight">\(\beta_1\)</span>.
|
||
Our result for <span class="math notranslate nohighlight">\(\beta_0\)</span> simplifies then to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1.
|
||
\]</div>
|
||
<p>We obtain then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}.
|
||
\]</div>
|
||
<p>If we define</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mu_{\boldsymbol{x}_1}=\frac{1}{n}\sum_{i=0}^{n-1} X_{i1},
|
||
\]</div>
|
||
<p>and the mean value of the outputs as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i,
|
||
\]</div>
|
||
<p>we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_0 = \mu_y - \beta_1\mu_{\boldsymbol{x}_1}.
|
||
\]</div>
|
||
<p>In the general case with more parameters than <span class="math notranslate nohighlight">\(\beta_0\)</span> and <span class="math notranslate nohighlight">\(\beta_1\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j.
|
||
\]</div>
|
||
<p>We can rewrite the latter equation as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \sum_{j=1}^{p-1} \mu_{\boldsymbol{x}_j}\beta_j,
|
||
\]</div>
|
||
<p>where we have defined</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mu_{\boldsymbol{x}_j}=\frac{1}{n}\sum_{i=0}^{n-1} X_{ij},
|
||
\]</div>
|
||
<p>the mean value for all elements of the column vector <span class="math notranslate nohighlight">\(\boldsymbol{x}_j\)</span>.</p>
|
||
<p>Replacing <span class="math notranslate nohighlight">\(y_i\)</span> with <span class="math notranslate nohighlight">\(y_i - y_i - \overline{\boldsymbol{y}}\)</span> and centering also our design matrix results in a cost function (in vector-matrix disguise)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}).
|
||
\]</div>
|
||
<p>If we minimize with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> we have then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}}\)</span>
|
||
and <span class="math notranslate nohighlight">\(\tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj}\)</span>.</p>
|
||
<p>For Ridge regression we need to add <span class="math notranslate nohighlight">\(\lambda \boldsymbol{\beta}^T\boldsymbol{\beta}\)</span> to the cost function and get then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}.
|
||
\]</div>
|
||
<p>What does this mean? And why do we insist on all this? Let us look at some examples.</p>
|
||
<p>This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (<em>code example thanks to Øyvind Sigmundson Schøyen</em>). Here our scaling of the data is done by subtracting the mean values only.
|
||
Note also that we do not split the data into training and test.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
|
||
|
||
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">2021</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">fit_beta</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span><span class="p">)</span> <span class="o">@</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span>
|
||
|
||
|
||
<span class="n">true_beta</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">3.7</span><span class="p">]</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">11</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">([</span><span class="n">x</span> <span class="o">**</span> <span class="n">p</span> <span class="o">*</span> <span class="n">b</span> <span class="k">for</span> <span class="n">p</span><span class="p">,</span> <span class="n">b</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">true_beta</span><span class="p">)]),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span>
|
||
<span class="p">)</span> <span class="o">+</span> <span class="mf">0.1</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
|
||
|
||
<span class="n">degree</span> <span class="o">=</span> <span class="mi">3</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">degree</span><span class="p">))</span>
|
||
|
||
<span class="c1"># Include the intercept in the design matrix</span>
|
||
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">degree</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span> <span class="n">p</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="n">p</span>
|
||
|
||
<span class="n">beta</span> <span class="o">=</span> <span class="n">fit_beta</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Intercept is included in the design matrix</span>
|
||
<span class="n">skl</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"True beta: </span><span class="si">{</span><span class="n">true_beta</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Fitted beta: </span><span class="si">{</span><span class="n">beta</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sklearn fitted beta: </span><span class="si">{</span><span class="n">skl</span><span class="o">.</span><span class="n">coef_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="n">ypredictOwn</span> <span class="o">=</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span>
|
||
<span class="n">ypredictSKL</span> <span class="o">=</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with intercept column"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictOwn</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with intercept column from SKL"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictSKL</span><span class="p">))</span>
|
||
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Data"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Fit"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Sklearn (fit_intercept=False)"</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Do not include the intercept in the design matrix</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">degree</span> <span class="o">-</span> <span class="mi">1</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">degree</span> <span class="o">-</span> <span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span> <span class="n">p</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="p">(</span><span class="n">p</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Intercept is not included in the design matrix</span>
|
||
<span class="n">skl</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Use centered values for X and y when computing coefficients</span>
|
||
<span class="n">y_offset</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">average</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">X_offset</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">average</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
|
||
<span class="n">beta</span> <span class="o">=</span> <span class="n">fit_beta</span><span class="p">(</span><span class="n">X</span> <span class="o">-</span> <span class="n">X_offset</span><span class="p">,</span> <span class="n">y</span> <span class="o">-</span> <span class="n">y_offset</span><span class="p">)</span>
|
||
<span class="n">intercept</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_offset</span> <span class="o">-</span> <span class="n">X_offset</span> <span class="o">@</span> <span class="n">beta</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Manual intercept: </span><span class="si">{</span><span class="n">intercept</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Fitted beta (without intercept): </span><span class="si">{</span><span class="n">beta</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sklearn intercept: </span><span class="si">{</span><span class="n">skl</span><span class="o">.</span><span class="n">intercept_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sklearn fitted beta (without intercept): </span><span class="si">{</span><span class="n">skl</span><span class="o">.</span><span class="n">coef_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="n">ypredictOwn</span> <span class="o">=</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span>
|
||
<span class="n">ypredictSKL</span> <span class="o">=</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with Manual intercept"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictOwn</span><span class="o">+</span><span class="n">intercept</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with Sklearn intercept"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictSKL</span><span class="p">))</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span> <span class="o">+</span> <span class="n">intercept</span><span class="p">,</span> <span class="s2">"--"</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Fit (manual intercept)"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">),</span> <span class="s2">"--"</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Sklearn (fit_intercept=True)"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">grid</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>True beta: [2, 0.5, 3.7]
|
||
Fitted beta: [2.08376632 0.19569961 3.97898392]
|
||
Sklearn fitted beta: [2.08376632 0.19569961 3.97898392]
|
||
MSE with intercept column
|
||
0.004113634617443139
|
||
MSE with intercept column from SKL
|
||
0.004113634617443147
|
||
Manual intercept: 2.083766322923899
|
||
Fitted beta (without intercept): [0.19569961 3.97898392]
|
||
Sklearn intercept: 2.0837663229239043
|
||
Sklearn fitted beta (without intercept): [0.19569961 3.97898392]
|
||
MSE with Manual intercept
|
||
0.00411363461744314
|
||
MSE with Sklearn intercept
|
||
0.004113634617443131
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/10da0f5e683788ca6e620272ac457ee7b57be28e1f84c126da008b0cb32f6b2f.png" src="_images/10da0f5e683788ca6e620272ac457ee7b57be28e1f84c126da008b0cb32f6b2f.png" />
|
||
</div>
|
||
</div>
|
||
<p>The intercept is the value of our output/target variable
|
||
when all our features are zero and our function crosses the <span class="math notranslate nohighlight">\(y\)</span>-axis (for a one-dimensional case).</p>
|
||
<p>Printing the MSE, we see first that both methods give the same MSE, as
|
||
they should. However, when we move to for example Ridge regression,
|
||
the way we treat the intercept may give a larger or smaller MSE,
|
||
meaning that the MSE can be penalized by the value of the
|
||
intercept. Not including the intercept in the fit, means that the
|
||
regularization term does not include <span class="math notranslate nohighlight">\(\beta_0\)</span>. For different values
|
||
of <span class="math notranslate nohighlight">\(\lambda\)</span>, this may lead to different MSE values.</p>
|
||
<p>To remind the reader, the regularization term, with the intercept in Ridge regression, is given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2,
|
||
\]</div>
|
||
<p>but when we take out the intercept, this equation becomes</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2.
|
||
\]</div>
|
||
<p>For Lasso regression we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert.
|
||
\]</div>
|
||
<p>It means that, when scaling the design matrix and the outputs/targets,
|
||
by subtracting the mean values, we have an optimization problem which
|
||
is not penalized by the intercept. The MSE value can then be smaller
|
||
since it focuses only on the remaining quantities. If we however bring
|
||
back the intercept, we will get a MSE which then contains the
|
||
intercept.</p>
|
||
<p>Armed with this wisdom, we attempt first to simply set the intercept equal to <strong>False</strong> in our implementation of Ridge regression for our well-known vanilla data set.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">linear_model</span>
|
||
|
||
<span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
|
||
<span class="c1"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||
<span class="c1"># Useful for eventual debugging.</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">3155</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">Maxpolydegree</span> <span class="o">=</span> <span class="mi">20</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="n">Maxpolydegree</span><span class="p">))</span>
|
||
<span class="c1">#We include explicitely the intercept column</span>
|
||
<span class="k">for</span> <span class="n">degree</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">Maxpolydegree</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="n">degree</span>
|
||
<span class="c1"># We split the data in test and training data</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">Maxpolydegree</span>
|
||
<span class="n">I</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">p</span><span class="p">,</span><span class="n">p</span><span class="p">)</span>
|
||
<span class="c1"># Decide which values of lambda to use</span>
|
||
<span class="n">nlambdas</span> <span class="o">=</span> <span class="mi">6</span>
|
||
<span class="n">MSEOwnRidgePredict</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="n">MSERidgePredict</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="n">lambdas</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">logspace</span><span class="p">(</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">):</span>
|
||
<span class="n">lmb</span> <span class="o">=</span> <span class="n">lambdas</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
<span class="n">OwnRidgeBeta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X_train</span><span class="o">+</span><span class="n">lmb</span><span class="o">*</span><span class="n">I</span><span class="p">)</span> <span class="o">@</span> <span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y_train</span>
|
||
<span class="c1"># Note: we include the intercept column and no scaling</span>
|
||
<span class="n">RegRidge</span> <span class="o">=</span> <span class="n">linear_model</span><span class="o">.</span><span class="n">Ridge</span><span class="p">(</span><span class="n">lmb</span><span class="p">,</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
<span class="n">RegRidge</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span><span class="n">y_train</span><span class="p">)</span>
|
||
<span class="c1"># and then make the prediction</span>
|
||
<span class="n">ytildeOwnRidge</span> <span class="o">=</span> <span class="n">X_train</span> <span class="o">@</span> <span class="n">OwnRidgeBeta</span>
|
||
<span class="n">ypredictOwnRidge</span> <span class="o">=</span> <span class="n">X_test</span> <span class="o">@</span> <span class="n">OwnRidgeBeta</span>
|
||
<span class="n">ytildeRidge</span> <span class="o">=</span> <span class="n">RegRidge</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_train</span><span class="p">)</span>
|
||
<span class="n">ypredictRidge</span> <span class="o">=</span> <span class="n">RegRidge</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
|
||
<span class="n">MSEOwnRidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredictOwnRidge</span><span class="p">)</span>
|
||
<span class="n">MSERidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredictRidge</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Beta values for own Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">OwnRidgeBeta</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Beta values for Scikit-Learn Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">RegRidge</span><span class="o">.</span><span class="n">coef_</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE values for own Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSEOwnRidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE values for Scikit-Learn Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSERidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
|
||
|
||
<span class="c1"># Now plot the results</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">lambdas</span><span class="p">),</span> <span class="n">MSEOwnRidgePredict</span><span class="p">,</span> <span class="s1">'r'</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'MSE own Ridge Test'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">lambdas</span><span class="p">),</span> <span class="n">MSERidgePredict</span><span class="p">,</span> <span class="s1">'g'</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'MSE Ridge Test'</span><span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">'log10(lambda)'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">'MSE'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The results here agree when we force <strong>Scikit-Learn</strong>’s Ridge function to include the first column in our design matrix.
|
||
We see that the results agree very well. Here we have thus explicitely included the intercept column in the design matrix.
|
||
What happens if we do not include the intercept in our fit?
|
||
Let us see how we can change this code by zero centering.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">linear_model</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">StandardScaler</span>
|
||
|
||
<span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="c1"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||
<span class="c1"># Useful for eventual debugging.</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">315</span><span class="p">)</span>
|
||
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
|
||
<span class="n">Maxpolydegree</span> <span class="o">=</span> <span class="mi">20</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="n">n</span><span class="p">,</span><span class="n">Maxpolydegree</span><span class="o">-</span><span class="mi">1</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">degree</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="n">Maxpolydegree</span><span class="p">):</span> <span class="c1">#No intercept column</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="n">degree</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="p">(</span><span class="n">degree</span><span class="p">)</span>
|
||
|
||
<span class="c1"># We split the data in test and training data</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
|
||
<span class="c1">#For our own implementation, we will need to deal with the intercept by centering the design matrix and the target variable</span>
|
||
<span class="n">X_train_mean</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="c1">#Center by removing mean from each feature</span>
|
||
<span class="n">X_train_scaled</span> <span class="o">=</span> <span class="n">X_train</span> <span class="o">-</span> <span class="n">X_train_mean</span>
|
||
<span class="n">X_test_scaled</span> <span class="o">=</span> <span class="n">X_test</span> <span class="o">-</span> <span class="n">X_train_mean</span>
|
||
<span class="c1">#The model intercept (called y_scaler) is given by the mean of the target variable (IF X is centered)</span>
|
||
<span class="c1">#Remove the intercept from the training data.</span>
|
||
<span class="n">y_scaler</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_train</span><span class="p">)</span>
|
||
<span class="n">y_train_scaled</span> <span class="o">=</span> <span class="n">y_train</span> <span class="o">-</span> <span class="n">y_scaler</span>
|
||
|
||
<span class="n">p</span> <span class="o">=</span> <span class="n">Maxpolydegree</span><span class="o">-</span><span class="mi">1</span>
|
||
<span class="n">I</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">p</span><span class="p">,</span><span class="n">p</span><span class="p">)</span>
|
||
<span class="c1"># Decide which values of lambda to use</span>
|
||
<span class="n">nlambdas</span> <span class="o">=</span> <span class="mi">6</span>
|
||
<span class="n">MSEOwnRidgePredict</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="n">MSERidgePredict</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">)</span>
|
||
|
||
<span class="n">lambdas</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">logspace</span><span class="p">(</span><span class="o">-</span><span class="mi">4</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="n">nlambdas</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">):</span>
|
||
<span class="n">lmb</span> <span class="o">=</span> <span class="n">lambdas</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
<span class="n">OwnRidgeBeta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">X_train_scaled</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X_train_scaled</span><span class="o">+</span><span class="n">lmb</span><span class="o">*</span><span class="n">I</span><span class="p">)</span> <span class="o">@</span> <span class="n">X_train_scaled</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="p">(</span><span class="n">y_train_scaled</span><span class="p">)</span>
|
||
<span class="n">intercept_</span> <span class="o">=</span> <span class="n">y_scaler</span> <span class="o">-</span> <span class="n">X_train_mean</span><span class="nd">@OwnRidgeBeta</span> <span class="c1">#The intercept can be shifted so the model can predict on uncentered data</span>
|
||
<span class="c1">#Add intercept to prediction</span>
|
||
<span class="n">ypredictOwnRidge</span> <span class="o">=</span> <span class="n">X_test_scaled</span> <span class="o">@</span> <span class="n">OwnRidgeBeta</span> <span class="o">+</span> <span class="n">y_scaler</span>
|
||
<span class="n">RegRidge</span> <span class="o">=</span> <span class="n">linear_model</span><span class="o">.</span><span class="n">Ridge</span><span class="p">(</span><span class="n">lmb</span><span class="p">)</span>
|
||
<span class="n">RegRidge</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span><span class="n">y_train</span><span class="p">)</span>
|
||
<span class="n">ypredictRidge</span> <span class="o">=</span> <span class="n">RegRidge</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
|
||
<span class="n">MSEOwnRidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredictOwnRidge</span><span class="p">)</span>
|
||
<span class="n">MSERidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredictRidge</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Beta values for own Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">OwnRidgeBeta</span><span class="p">)</span> <span class="c1">#Intercept is given by mean of target variable</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Beta values for Scikit-Learn Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">RegRidge</span><span class="o">.</span><span class="n">coef_</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Intercept from own implementation:'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">intercept_</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'Intercept from Scikit-Learn Ridge implementation'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">RegRidge</span><span class="o">.</span><span class="n">intercept_</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE values for own Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSEOwnRidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE values for Scikit-Learn Ridge implementation"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSERidgePredict</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
|
||
|
||
|
||
<span class="c1"># Now plot the results</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">lambdas</span><span class="p">),</span> <span class="n">MSEOwnRidgePredict</span><span class="p">,</span> <span class="s1">'b--'</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'MSE own Ridge Test'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">lambdas</span><span class="p">),</span> <span class="n">MSERidgePredict</span><span class="p">,</span> <span class="s1">'g--'</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'MSE SL Ridge Test'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">'log10(lambda)'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">'MSE'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We see here, when compared to the code which includes explicitely the
|
||
intercept column, that our MSE value is actually smaller. This is
|
||
because the regularization term does not include the intercept value
|
||
<span class="math notranslate nohighlight">\(\beta_0\)</span> in the fitting. This applies to Lasso regularization as
|
||
well. It means that our optimization is now done only with the
|
||
centered matrix and/or vector that enter the fitting procedure.</p>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="exercisesweek35.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Exercises week 35</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="exercisesweek36.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Exercises week 36</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#plans-for-week-35">Plans for week 35</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#reading-recommendations">Reading recommendations:</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#reminder-from-last-week">Reminder from last week</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-equations-for-ordinary-least-squares">The equations for ordinary least squares</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-cost-loss-function">The cost/loss function</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#interpretations-and-optimizing-our-parameters">Interpretations and optimizing our parameters</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id1">Interpretations and optimizing our parameters</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#some-useful-matrix-and-vector-expressions">Some useful matrix and vector expressions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-jacobian">The Jacobian</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#derivatives-example-1">Derivatives, example 1</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-2">Example 2</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-3">Example 3</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-4">Example 4</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-mean-squared-error-and-its-derivative">The mean squared error and its derivative</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#meet-the-hessian-matrix">Meet the Hessian Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id2">Interpretations and optimizing our parameters</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-relevant-for-the-exercises">Example relevant for the exercises</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#own-code-for-ordinary-least-squares">Own code for Ordinary Least Squares</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#adding-error-analysis-and-training-set-up">Adding error analysis and training set up</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#splitting-our-data-in-training-and-test-data">Splitting our Data in Training and Test data</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-complete-code-with-a-simple-data-set">The complete code with a simple data set</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#making-your-own-test-train-splitting">Making your own test-train splitting</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#reducing-the-number-of-degrees-of-freedom-overarching-view">Reducing the number of degrees of freedom, overarching view</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#preprocessing-our-data">Preprocessing our data</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#functionality-in-scikit-learn">Functionality in Scikit-Learn</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-preprocessing">More preprocessing</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#frequently-used-scaling-functions">Frequently used scaling functions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-of-own-standard-scaling">Example of own Standard scaling</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#min-max-scaling">Min-Max Scaling</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#testing-the-means-squared-error-as-function-of-complexity">Testing the Means Squared Error as function of Complexity</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematical-interpretation-of-ordinary-least-squares">Mathematical Interpretation of Ordinary Least Squares</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#residual-error">Residual Error</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#simple-case">Simple case</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-singular-value-decomposition">The singular value decomposition</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#linear-regression-problems">Linear Regression Problems</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#fixing-the-singularity">Fixing the singularity</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#ridge-and-lasso-regression">Ridge and LASSO Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#deriving-the-ridge-regression-equations">Deriving the Ridge Regression Equations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#basic-math-of-the-svd">Basic math of the SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-svd-a-fantastic-algorithm">The SVD, a Fantastic Algorithm</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#economy-size-svd">Economy-size SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#codes-for-the-svd">Codes for the SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#note-about-svd-calculations">Note about SVD Calculations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#mathematics-of-the-svd-and-implications">Mathematics of the SVD and implications</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#example-matrix">Example Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#setting-up-the-matrix-to-be-inverted">Setting up the Matrix to be inverted</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#further-properties-important-for-our-analyses-later">Further properties (important for our analyses later)</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#meet-the-covariance-matrix">Meet the Covariance Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#introducing-the-covariance-and-correlation-functions">Introducing the Covariance and Correlation functions</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#covariance-and-correlation-matrix">Covariance and Correlation Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#correlation-function-and-design-feature-matrix">Correlation Function and Design/Feature Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#covariance-matrix-examples">Covariance Matrix Examples</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#correlation-matrix">Correlation Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#correlation-matrix-with-pandas">Correlation Matrix with Pandas</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#rewriting-the-covariance-and-or-correlation-matrix">Rewriting the Covariance and/or Correlation Matrix</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#linking-with-the-svd">Linking with the SVD</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#what-does-it-mean">What does it mean?</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#and-finally-boldsymbol-x-boldsymbol-x-t">And finally <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span></a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#back-to-ridge-and-lasso-regression">Back to Ridge and LASSO Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#interpreting-the-ridge-results">Interpreting the Ridge results</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#more-interpretations">More interpretations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#deriving-the-lasso-regression-equations">Deriving the Lasso Regression Equations</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#material-for-exercises-week-35">Material for exercises week 35</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#important-technicalities-more-on-rescaling-data">Important technicalities: More on Rescaling data</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |