Morten Hjorth-Jensen
4255 lines
294 KiB
HTML
4255 lines
294 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
<html>
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression — Applied Data Analysis and Machine Learning</title>
|
||
|
||
<link href="_static/css/theme.css" rel="stylesheet">
|
||
<link href="_static/css/index.ff1ffe594081f20da1ef19478df9384b.css" rel="stylesheet">
|
||
|
||
|
||
<link rel="stylesheet"
|
||
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin
|
||
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-book-theme.css?digest=c3fdc42140077d1ad13ad2f1588a4309" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-main.c949a650a448cc0ae9fd3441c0e17fb0.css" />
|
||
<link rel="stylesheet" type="text/css" href="_static/panels-variables.06eb56fa6e07937060861dad626602ad.css" />
|
||
|
||
<link rel="preload" as="script" href="_static/js/index.be7d3bbb2ef33a8344ce.js">
|
||
|
||
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
|
||
<script src="_static/jquery.js"></script>
|
||
<script src="_static/underscore.js"></script>
|
||
<script src="_static/doctools.js"></script>
|
||
<script src="_static/clipboard.min.js"></script>
|
||
<script src="_static/copybutton.js"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
|
||
<script src="_static/sphinx-book-theme.d59cb220de22ca1c485ebbdc042f0030.js"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
|
||
const thebe_selector = ".thebe,.cell"
|
||
const thebe_selector_input = "pre"
|
||
const thebe_selector_output = ".output, .cell_output"
|
||
</script>
|
||
<script async="async" src="_static/sphinx-thebe.js"></script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Exercises week 36" href="exercisesweek36.html" />
|
||
<link rel="prev" title="Exercises week 35" href="exercisesweek35.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<meta name="docsearch:language" content="None">
|
||
|
||
|
||
<!-- Google Analytics -->
|
||
|
||
</head>
|
||
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="80">
|
||
|
||
<div class="container-fluid" id="banner"></div>
|
||
|
||
|
||
|
||
<div class="container-xl">
|
||
<div class="row">
|
||
|
||
<div class="col-12 col-md-3 bd-sidebar site-navigation show" id="site-navigation">
|
||
|
||
<div class="navbar-brand-box">
|
||
<a class="navbar-brand text-wrap" href="index.html">
|
||
|
||
<!-- `logo` is deprecated in Sphinx 4.0, so remove this when we stop supporting 3 -->
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo" alt="logo">
|
||
|
||
|
||
<h1 class="site-logo" id="site-title">Applied Data Analysis and Machine Learning</h1>
|
||
|
||
</a>
|
||
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
|
||
<i class="icon fas fa-search"></i>
|
||
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search this book..." aria-label="Search this book..." autocomplete="off" >
|
||
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item active">
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
About the course
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="schedule.html">
|
||
Teaching schedule with links to material
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="teachers.html">
|
||
Teachers and Grading
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="textbooks.html">
|
||
Textbooks
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Review of Statistics with Resampling Techniques and Linear Algebra
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="statistics.html">
|
||
1. Elements of Probability Theory and Statistical Data Analysis
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="linalg.html">
|
||
2. Linear Algebra, Handling of Arrays and more Python Features
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
From Regression to Support Vector Machines
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter1.html">
|
||
3. Linear Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter2.html">
|
||
4. Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter3.html">
|
||
5. Resampling Methods
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter4.html">
|
||
6. Logistic Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapteroptimization.html">
|
||
7. Optimization, the central part of any Machine Learning algortithm
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter5.html">
|
||
8. Support Vector Machines, overarching aims
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Decision Trees, Ensemble Methods and Boosting
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter6.html">
|
||
9. Decision trees, overarching aims
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter7.html">
|
||
10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Dimensionality Reduction
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter8.html">
|
||
11. Basic ideas of the Principal Component Analysis (PCA)
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="clustering.html">
|
||
12. Clustering and Unsupervised Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Deep Learning Methods
|
||
</span>
|
||
</p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter9.html">
|
||
13. Neural networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter10.html">
|
||
14. Building a Feed Forward Neural Network
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter11.html">
|
||
15. Solving Differential Equations with Deep Learning
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter12.html">
|
||
16. Convolutional Neural Networks
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="chapter13.html">
|
||
17. Recurrent neural networks: Overarching view
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading">
|
||
<span class="caption-text">
|
||
Weekly material, notes and exercises
|
||
</span>
|
||
</p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek34.html">
|
||
Exercises week 34
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week34.html">
|
||
Week 34: Introduction to the course, Logistics and Practicalities
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek35.html">
|
||
Exercises week 35
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1 current active">
|
||
<a class="current reference internal" href="#">
|
||
Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="exercisesweek36.html">
|
||
Exercises week 36
|
||
</a>
|
||
</li>
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="week36.html">
|
||
Week 36: Statistical interpretation of Linear Regression and Resampling techniques
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav> <!-- To handle the deprecated key -->
|
||
|
||
<div class="navbar_extra_footer">
|
||
Powered by <a href="https://jupyterbook.org">Jupyter Book</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<main class="col py-md-3 pl-md-4 bd-content overflow-auto" role="main">
|
||
|
||
<div class="topbar container-xl fixed-top">
|
||
<div class="topbar-contents row">
|
||
<div class="col-12 col-md-3 bd-topbar-whitespace site-navigation show"></div>
|
||
<div class="col pl-md-4 topbar-main">
|
||
|
||
<button id="navbar-toggler" class="navbar-toggler ml-0" type="button" data-toggle="collapse"
|
||
data-toggle="tooltip" data-placement="bottom" data-target=".site-navigation" aria-controls="navbar-menu"
|
||
aria-expanded="true" aria-label="Toggle navigation" aria-controls="site-navigation"
|
||
title="Toggle navigation" data-toggle="tooltip" data-placement="left">
|
||
<i class="fas fa-bars"></i>
|
||
<i class="fas fa-arrow-left"></i>
|
||
<i class="fas fa-arrow-up"></i>
|
||
</button>
|
||
|
||
|
||
<div class="dropdown-buttons-trigger">
|
||
<button id="dropdown-buttons-trigger" class="btn btn-secondary topbarbtn" aria-label="Download this page"><i
|
||
class="fas fa-download"></i></button>
|
||
|
||
<div class="dropdown-buttons">
|
||
<!-- ipynb file if we had a myst markdown file -->
|
||
|
||
<!-- Download raw file -->
|
||
<a class="dropdown-buttons" href="_sources/week35.ipynb"><button type="button"
|
||
class="btn btn-secondary topbarbtn" title="Download source file" data-toggle="tooltip"
|
||
data-placement="left">.ipynb</button></a>
|
||
<!-- Download PDF via print -->
|
||
<button type="button" id="download-print" class="btn btn-secondary topbarbtn" title="Print to PDF"
|
||
onclick="printPdf(this)" data-toggle="tooltip" data-placement="left">.pdf</button>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Source interaction buttons -->
|
||
|
||
<!-- Full screen (wrap in <a> to have style consistency -->
|
||
|
||
<a class="full-screen-button"><button type="button" class="btn btn-secondary topbarbtn" data-toggle="tooltip"
|
||
data-placement="bottom" onclick="toggleFullScreen()" aria-label="Fullscreen mode"
|
||
title="Fullscreen mode"><i
|
||
class="fas fa-expand"></i></button></a>
|
||
|
||
<!-- Launch buttons -->
|
||
|
||
</div>
|
||
|
||
<!-- Table of contents -->
|
||
<div class="d-none d-md-block col-md-2 bd-toc show noprint">
|
||
|
||
<div class="tocsection onthispage pt-5 pb-3">
|
||
<i class="fas fa-list"></i> Contents
|
||
</div>
|
||
<nav id="bd-toc-nav" aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#plans-for-week-35">
|
||
Plans for week 35
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#reading-recommendations">
|
||
Reading recommendations:
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week">
|
||
Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-equations-for-ordinary-least-squares">
|
||
The equations for ordinary least squares
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-cost-loss-function">
|
||
The cost/loss function
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#interpretations-and-optimizing-our-parameters">
|
||
Interpretations and optimizing our parameters
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#id1">
|
||
Interpretations and optimizing our parameters
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#some-useful-matrix-and-vector-expressions">
|
||
Some useful matrix and vector expressions
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-jacobian">
|
||
The Jacobian
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#derivatives-example-1">
|
||
Derivatives, example 1
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-2">
|
||
Example 2
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-3">
|
||
Example 3
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-4">
|
||
Example 4
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-mean-squared-error-and-its-derivative">
|
||
The mean squared error and its derivative
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#other-useful-relations">
|
||
Other useful relations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#meet-the-hessian-matrix">
|
||
Meet the Hessian Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#id2">
|
||
Interpretations and optimizing our parameters
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-relevant-for-the-exercises">
|
||
Example relevant for the exercises
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#own-code-for-ordinary-least-squares">
|
||
Own code for Ordinary Least Squares
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#adding-error-analysis-and-training-set-up">
|
||
Adding error analysis and training set up
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#splitting-our-data-in-training-and-test-data">
|
||
Splitting our Data in Training and Test data
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-complete-code-with-a-simple-data-set">
|
||
The complete code with a simple data set
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#making-your-own-test-train-splitting">
|
||
Making your own test-train splitting
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#reducing-the-number-of-degrees-of-freedom-overarching-view">
|
||
Reducing the number of degrees of freedom, overarching view
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#preprocessing-our-data">
|
||
Preprocessing our data
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#functionality-in-scikit-learn">
|
||
Functionality in Scikit-Learn
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-preprocessing">
|
||
More preprocessing
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#frequently-used-scaling-functions">
|
||
Frequently used scaling functions
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-of-own-standard-scaling">
|
||
Example of own Standard scaling
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#min-max-scaling">
|
||
Min-Max Scaling
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#testing-the-means-squared-error-as-function-of-complexity">
|
||
Testing the Means Squared Error as function of Complexity
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-preprocessing-examples-two-dimensional-example-the-franke-function">
|
||
More preprocessing examples, two-dimensional example, the Franke function
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#to-think-about-first-part">
|
||
To think about, first part
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-thinking">
|
||
More thinking
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#still-thinking">
|
||
Still thinking
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#what-does-centering-subtracting-the-mean-values-mean-mathematically">
|
||
What does centering (subtracting the mean values) mean mathematically?
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#further-manipulations">
|
||
Further Manipulations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#wrapping-it-up">
|
||
Wrapping it up
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#linear-regression-code-intercept-handling-first">
|
||
Linear Regression code, Intercept handling first
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-boston-housing-data-example">
|
||
The Boston housing data example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#housing-data-the-code">
|
||
Housing data, the code
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#material-for-lecture-thursday-august-31">
|
||
Material for lecture Thursday, August 31
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mathematical-interpretation-of-ordinary-least-squares">
|
||
Mathematical Interpretation of Ordinary Least Squares
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#residual-error">
|
||
Residual Error
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#simple-case">
|
||
Simple case
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-singular-value-decomposition">
|
||
The singular value decomposition
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#linear-regression-problems">
|
||
Linear Regression Problems
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#fixing-the-singularity">
|
||
Fixing the singularity
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#basic-math-of-the-svd">
|
||
Basic math of the SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-svd-a-fantastic-algorithm">
|
||
The SVD, a Fantastic Algorithm
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#economy-size-svd">
|
||
Economy-size SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#codes-for-the-svd">
|
||
Codes for the SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#note-about-svd-calculations">
|
||
Note about SVD Calculations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mathematics-of-the-svd-and-implications">
|
||
Mathematics of the SVD and implications
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-matrix">
|
||
Example Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#setting-up-the-matrix-to-be-inverted">
|
||
Setting up the Matrix to be inverted
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#further-properties-important-for-our-analyses-later">
|
||
Further properties (important for our analyses later)
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#meet-the-covariance-matrix">
|
||
Meet the Covariance Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#introducing-the-covariance-and-correlation-functions">
|
||
Introducing the Covariance and Correlation functions
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#covariance-and-correlation-matrix">
|
||
Covariance and Correlation Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-function-and-design-feature-matrix">
|
||
Correlation Function and Design/Feature Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#covariance-matrix-examples">
|
||
Covariance Matrix Examples
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-matrix">
|
||
Correlation Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-matrix-with-pandas">
|
||
Correlation Matrix with Pandas
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-matrix-with-pandas-and-the-franke-function">
|
||
Correlation Matrix with Pandas and the Franke function
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#rewriting-the-covariance-and-or-correlation-matrix">
|
||
Rewriting the Covariance and/or Correlation Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#linking-with-the-svd">
|
||
Linking with the SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#what-does-it-mean">
|
||
What does it mean?
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#and-finally-boldsymbol-x-boldsymbol-x-t">
|
||
And finally
|
||
<span class="math notranslate nohighlight">
|
||
\(\boldsymbol{X}\boldsymbol{X}^T\)
|
||
</span>
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#ridge-and-lasso-regression">
|
||
Ridge and LASSO Regression
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#deriving-the-ridge-regression-equations">
|
||
Deriving the Ridge Regression Equations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#interpreting-the-ridge-results">
|
||
Interpreting the Ridge results
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-interpretations">
|
||
More interpretations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#deriving-the-lasso-regression-equations">
|
||
Deriving the Lasso Regression Equations
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div id="main-content" class="row">
|
||
<div class="col-12 col-md-9 pl-md-3 pr-md-0">
|
||
<!-- Table of contents that is only displayed when printing the page -->
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#plans-for-week-35">
|
||
Plans for week 35
|
||
</a>
|
||
<ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#reading-recommendations">
|
||
Reading recommendations:
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week">
|
||
Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-equations-for-ordinary-least-squares">
|
||
The equations for ordinary least squares
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-cost-loss-function">
|
||
The cost/loss function
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#interpretations-and-optimizing-our-parameters">
|
||
Interpretations and optimizing our parameters
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#id1">
|
||
Interpretations and optimizing our parameters
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#some-useful-matrix-and-vector-expressions">
|
||
Some useful matrix and vector expressions
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-jacobian">
|
||
The Jacobian
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#derivatives-example-1">
|
||
Derivatives, example 1
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-2">
|
||
Example 2
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-3">
|
||
Example 3
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-4">
|
||
Example 4
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-mean-squared-error-and-its-derivative">
|
||
The mean squared error and its derivative
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#other-useful-relations">
|
||
Other useful relations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#meet-the-hessian-matrix">
|
||
Meet the Hessian Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#id2">
|
||
Interpretations and optimizing our parameters
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-relevant-for-the-exercises">
|
||
Example relevant for the exercises
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#own-code-for-ordinary-least-squares">
|
||
Own code for Ordinary Least Squares
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#adding-error-analysis-and-training-set-up">
|
||
Adding error analysis and training set up
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#splitting-our-data-in-training-and-test-data">
|
||
Splitting our Data in Training and Test data
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-complete-code-with-a-simple-data-set">
|
||
The complete code with a simple data set
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#making-your-own-test-train-splitting">
|
||
Making your own test-train splitting
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#reducing-the-number-of-degrees-of-freedom-overarching-view">
|
||
Reducing the number of degrees of freedom, overarching view
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#preprocessing-our-data">
|
||
Preprocessing our data
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#functionality-in-scikit-learn">
|
||
Functionality in Scikit-Learn
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-preprocessing">
|
||
More preprocessing
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#frequently-used-scaling-functions">
|
||
Frequently used scaling functions
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-of-own-standard-scaling">
|
||
Example of own Standard scaling
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#min-max-scaling">
|
||
Min-Max Scaling
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#testing-the-means-squared-error-as-function-of-complexity">
|
||
Testing the Means Squared Error as function of Complexity
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-preprocessing-examples-two-dimensional-example-the-franke-function">
|
||
More preprocessing examples, two-dimensional example, the Franke function
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#to-think-about-first-part">
|
||
To think about, first part
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-thinking">
|
||
More thinking
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#still-thinking">
|
||
Still thinking
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#what-does-centering-subtracting-the-mean-values-mean-mathematically">
|
||
What does centering (subtracting the mean values) mean mathematically?
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#further-manipulations">
|
||
Further Manipulations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#wrapping-it-up">
|
||
Wrapping it up
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#linear-regression-code-intercept-handling-first">
|
||
Linear Regression code, Intercept handling first
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-boston-housing-data-example">
|
||
The Boston housing data example
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#housing-data-the-code">
|
||
Housing data, the code
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#material-for-lecture-thursday-august-31">
|
||
Material for lecture Thursday, August 31
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mathematical-interpretation-of-ordinary-least-squares">
|
||
Mathematical Interpretation of Ordinary Least Squares
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#residual-error">
|
||
Residual Error
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#simple-case">
|
||
Simple case
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-singular-value-decomposition">
|
||
The singular value decomposition
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#linear-regression-problems">
|
||
Linear Regression Problems
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#fixing-the-singularity">
|
||
Fixing the singularity
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#basic-math-of-the-svd">
|
||
Basic math of the SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#the-svd-a-fantastic-algorithm">
|
||
The SVD, a Fantastic Algorithm
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#economy-size-svd">
|
||
Economy-size SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#codes-for-the-svd">
|
||
Codes for the SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#note-about-svd-calculations">
|
||
Note about SVD Calculations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#mathematics-of-the-svd-and-implications">
|
||
Mathematics of the SVD and implications
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#example-matrix">
|
||
Example Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#setting-up-the-matrix-to-be-inverted">
|
||
Setting up the Matrix to be inverted
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#further-properties-important-for-our-analyses-later">
|
||
Further properties (important for our analyses later)
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#meet-the-covariance-matrix">
|
||
Meet the Covariance Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#introducing-the-covariance-and-correlation-functions">
|
||
Introducing the Covariance and Correlation functions
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#covariance-and-correlation-matrix">
|
||
Covariance and Correlation Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-function-and-design-feature-matrix">
|
||
Correlation Function and Design/Feature Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#covariance-matrix-examples">
|
||
Covariance Matrix Examples
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-matrix">
|
||
Correlation Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-matrix-with-pandas">
|
||
Correlation Matrix with Pandas
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#correlation-matrix-with-pandas-and-the-franke-function">
|
||
Correlation Matrix with Pandas and the Franke function
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#rewriting-the-covariance-and-or-correlation-matrix">
|
||
Rewriting the Covariance and/or Correlation Matrix
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#linking-with-the-svd">
|
||
Linking with the SVD
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#what-does-it-mean">
|
||
What does it mean?
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#and-finally-boldsymbol-x-boldsymbol-x-t">
|
||
And finally
|
||
<span class="math notranslate nohighlight">
|
||
\(\boldsymbol{X}\boldsymbol{X}^T\)
|
||
</span>
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#ridge-and-lasso-regression">
|
||
Ridge and LASSO Regression
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#deriving-the-ridge-regression-equations">
|
||
Deriving the Ridge Regression Equations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#interpreting-the-ridge-results">
|
||
Interpreting the Ridge results
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#more-interpretations">
|
||
More interpretations
|
||
</a>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry">
|
||
<a class="reference internal nav-link" href="#deriving-the-lasso-regression-equations">
|
||
Deriving the Lasso Regression Equations
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div>
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html week35.do.txt --no_mako -->
|
||
<!-- dom:TITLE: Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression --><div class="tex2jax_ignore mathjax_ignore section" id="week-35-from-ordinary-linear-regression-to-ridge-and-lasso-regression">
|
||
<h1>Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression<a class="headerlink" href="#week-35-from-ordinary-linear-regression-to-ridge-and-lasso-regression" title="Permalink to this headline">¶</a></h1>
|
||
<p><strong>Morten Hjorth-Jensen</strong>, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</p>
|
||
<p>Date: <strong>August 28-September 1</strong></p>
|
||
<div class="section" id="plans-for-week-35">
|
||
<h2>Plans for week 35<a class="headerlink" href="#plans-for-week-35" title="Permalink to this headline">¶</a></h2>
|
||
<p>The main topics are:</p>
|
||
<ol class="simple">
|
||
<li><p>Brief repetition from last week</p></li>
|
||
<li><p>Derivation of the equations for ordinary least squares</p></li>
|
||
<li><p>Discussion on how to prepare data and examples of applications of linear regression</p></li>
|
||
<li><p>Material for the lecture on Thursday: Mathematical interpretations of linear regression</p></li>
|
||
<li><p>Thursday: Ridge and Lasso regression and Singular Value Decomposition</p></li>
|
||
<li><p><a class="reference external" href="https://youtu.be/qBNm-HGSxL4">Video of lecture</a></p></li>
|
||
<li><p><a class="reference external" href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/HandWrittenNotes/2023/NotesAug31.pdf">Whiteboard notes</a></p></li>
|
||
</ol>
|
||
<div class="section" id="reading-recommendations">
|
||
<h3>Reading recommendations:<a class="headerlink" href="#reading-recommendations" title="Permalink to this headline">¶</a></h3>
|
||
<ol class="simple">
|
||
<li><p>See lecture notes for week 35 at <a class="reference external" href="https://compphysics.github.io/MachineLearning/doc/web/course.html">https://compphysics.github.io/MachineLearning/doc/web/course.html</a></p></li>
|
||
<li><p>Goodfellow, Bengio and Courville, Deep Learning, chapter 2 on linear algebra and sections 3.1-3.10 on elements of statistics (background)</p></li>
|
||
<li><p>Hastie, Tibshirani and Friedman, The elements of statistical learning, sections 3.1-3.4 (on relevance for the discussion of linear regression).</p></li>
|
||
</ol>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week">
|
||
<h2>Why Linear Regression (aka Ordinary Least Squares and family), repeat from last week<a class="headerlink" href="#why-linear-regression-aka-ordinary-least-squares-and-family-repeat-from-last-week" title="Permalink to this headline">¶</a></h2>
|
||
<p>We need first a reminder from last week about linear regression.</p>
|
||
<p>Fitting a continuous function with linear parameterization in terms of the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>.</p>
|
||
<ul class="simple">
|
||
<li><p>Method of choice for fitting a continuous function!</p></li>
|
||
<li><p>Gives an excellent introduction to central Machine Learning features with <strong>understandable pedagogical</strong> links to other methods like <strong>Neural Networks</strong>, <strong>Support Vector Machines</strong> etc</p></li>
|
||
<li><p>Analytical expression for the fitting parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span></p></li>
|
||
<li><p>Analytical expressions for statistical propertiers like mean values, variances, confidence intervals and more</p></li>
|
||
<li><p>Analytical relation with probabilistic interpretations</p></li>
|
||
<li><p>Easy to introduce basic concepts like bias-variance tradeoff, cross-validation, resampling and regularization techniques and many other ML topics</p></li>
|
||
<li><p>Easy to code! And links well with classification problems and logistic regression and neural networks</p></li>
|
||
<li><p>Allows for <strong>easy</strong> hands-on understanding of gradient descent methods</p></li>
|
||
<li><p>and many more features</p></li>
|
||
</ul>
|
||
<p>For more discussions of Ridge and Lasso regression, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.
|
||
Similarly, <a class="reference external" href="https://arxiv.org/abs/1803.08823">Mehta et al’s article</a> is also recommended.</p>
|
||
</div>
|
||
<div class="section" id="the-equations-for-ordinary-least-squares">
|
||
<h2>The equations for ordinary least squares<a class="headerlink" href="#the-equations-for-ordinary-least-squares" title="Permalink to this headline">¶</a></h2>
|
||
<p>Our data which we want to apply a machine learning method on, consist
|
||
of a set of inputs <span class="math notranslate nohighlight">\(\boldsymbol{x}^T=[x_0,x_1,x_2,\dots,x_{n-1}]\)</span> and the
|
||
outputs we want to model <span class="math notranslate nohighlight">\(\boldsymbol{y}^T=[y_0,y_1,y_2,\dots,y_{n-1}]\)</span>.
|
||
We assume that the output data can be represented (for a regression case) by a continuous function <span class="math notranslate nohighlight">\(f\)</span>
|
||
through</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y_i=f(x_i)+\epsilon_i,
|
||
\]</div>
|
||
<p>or in general</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{y}=f(\boldsymbol{x})+\boldsymbol{\epsilon},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{\epsilon}\)</span> represents some noise which is normally assumed to
|
||
be distributed via a normal probability distribution with zero mean
|
||
value and a variance <span class="math notranslate nohighlight">\(\sigma^2\)</span>.</p>
|
||
<p>In linear regression we approximate the unknown function with another
|
||
continuous function <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{y}}(\boldsymbol{x})\)</span> which depends linearly on
|
||
some unknown parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\beta}^T=[\beta_0,\beta_1,\beta_2,\dots,\beta_{p-1}]\)</span>.</p>
|
||
<p>Last week we introduced the so-called design matrix in order to define
|
||
the approximation <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}}\)</span> via the unknown quantity
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\tilde{y}}= \boldsymbol{X}\boldsymbol{\beta},
|
||
\]</div>
|
||
<p>and in order to find the optimal parameters <span class="math notranslate nohighlight">\(\beta_i\)</span> we defined a function which
|
||
gives a measure of the spread between the values <span class="math notranslate nohighlight">\(y_i\)</span> (which
|
||
represent the output values we want to reproduce) and the parametrized
|
||
values <span class="math notranslate nohighlight">\(\tilde{y}_i\)</span>, namely the so-called cost/loss function.</p>
|
||
</div>
|
||
<div class="section" id="the-cost-loss-function">
|
||
<h2>The cost/loss function<a class="headerlink" href="#the-cost-loss-function" title="Permalink to this headline">¶</a></h2>
|
||
<p>We used the mean squared error to define the way we measure the quality of our model</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta})=\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)^T\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)\right\},
|
||
\]</div>
|
||
<p>or using the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and in a more compact matrix-vector notation as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
||
\]</div>
|
||
<p>This function represents one of many possible ways to define the so-called cost function.</p>
|
||
<p>It is also common to define
|
||
the function <span class="math notranslate nohighlight">\(C\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta})=\frac{1}{2n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2,
|
||
\]</div>
|
||
<p>since when taking the first derivative with respect to the unknown parameters <span class="math notranslate nohighlight">\(\beta\)</span>, the factor of <span class="math notranslate nohighlight">\(2\)</span> cancels out.</p>
|
||
</div>
|
||
<div class="section" id="interpretations-and-optimizing-our-parameters">
|
||
<h2>Interpretations and optimizing our parameters<a class="headerlink" href="#interpretations-and-optimizing-our-parameters" title="Permalink to this headline">¶</a></h2>
|
||
<p>The function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\},
|
||
\]</div>
|
||
<p>can be linked to the variance of the quantity <span class="math notranslate nohighlight">\(y_i\)</span> if we interpret the latter as the mean value.
|
||
When linking (see the discussions next week) with the maximum likelihood approach below, we will indeed interpret <span class="math notranslate nohighlight">\(y_i\)</span> as a mean value</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y_{i}=\langle y_i \rangle = \beta_0x_{i,0}+\beta_1x_{i,1}+\beta_2x_{i,2}+\dots+\beta_{n-1}x_{i,n-1}+\epsilon_i,
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\langle y_i \rangle\)</span> is the mean value. Keep in mind also that
|
||
till now we have treated <span class="math notranslate nohighlight">\(y_i\)</span> as the exact value. Normally, the
|
||
response (dependent or outcome) variable <span class="math notranslate nohighlight">\(y_i\)</span> is the outcome of a
|
||
numerical experiment or another type of experiment and could thus be treated itself as an
|
||
approximation to the true value. It is then always accompanied by an
|
||
error estimate, often limited to a statistical error estimate given by
|
||
the standard deviation discussed earlier. In the discussion here we
|
||
will treat <span class="math notranslate nohighlight">\(y_i\)</span> as our exact value for the response variable.</p>
|
||
<p>In order to find the parameters <span class="math notranslate nohighlight">\(\beta_i\)</span> we will then minimize the spread of <span class="math notranslate nohighlight">\(C(\boldsymbol{\beta})\)</span>, that is we are going to solve the problem</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
||
\]</div>
|
||
<p>In practical terms it means we will require</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\beta})}{\partial \beta_j} = \frac{\partial }{\partial \beta_j}\left[ \frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)^2\right]=0,
|
||
\]</div>
|
||
<p>which results in</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\beta})}{\partial \beta_j} = -\frac{2}{n}\left[ \sum_{i=0}^{n-1}x_{ij}\left(y_i-\beta_0x_{i,0}-\beta_1x_{i,1}-\beta_2x_{i,2}-\dots-\beta_{n-1}x_{i,n-1}\right)\right]=0,
|
||
\]</div>
|
||
<p>or in a matrix-vector form as (multiplying away the factor <span class="math notranslate nohighlight">\(-2/n\)</span>, see derivation below)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}^T} = 0 = \boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right).
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="id1">
|
||
<h2>Interpretations and optimizing our parameters<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h2>
|
||
<p>We can rewrite, see the derivations below,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}^T} = 0 = \boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right),
|
||
\]</div>
|
||
<p>as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{y} = \boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta},
|
||
\]</div>
|
||
<p>and if the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> is invertible we have the solution</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\beta} =\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>We note also that since our design matrix is defined as <span class="math notranslate nohighlight">\(\boldsymbol{X}\in
|
||
{\mathbb{R}}^{n\times p}\)</span>, the product <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X} \in
|
||
{\mathbb{R}}^{p\times p}\)</span>. In most cases we have that <span class="math notranslate nohighlight">\(p \ll n\)</span>. In our example case below we have <span class="math notranslate nohighlight">\(p=5\)</span> meaning. We end up with inverting a small
|
||
<span class="math notranslate nohighlight">\(5\times 5\)</span> matrix. This is a rather common situation, in many cases we end up with low-dimensional
|
||
matrices to invert. The methods discussed here and for many other
|
||
supervised learning algorithms like classification with logistic
|
||
regression or support vector machines, exhibit dimensionalities which
|
||
allow for the usage of direct linear algebra methods such as <strong>LU</strong> decomposition or <strong>Singular Value Decomposition</strong> (SVD) for finding the inverse of the matrix
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. This is discussed on Thursday this week.</p>
|
||
<p><strong>Small question</strong>: Do you think the example we have at hand here (the nuclear binding energies) can lead to problems in inverting the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>? What kind of problems can we expect?</p>
|
||
</div>
|
||
<div class="section" id="some-useful-matrix-and-vector-expressions">
|
||
<h2>Some useful matrix and vector expressions<a class="headerlink" href="#some-useful-matrix-and-vector-expressions" title="Permalink to this headline">¶</a></h2>
|
||
<p>The following matrix and vector relation will be useful here and for
|
||
the rest of the course. Vectors are always written as boldfaced lower
|
||
case letters and matrices as upper case boldfaced letters. In the
|
||
following we will discuss how to calculate derivatives of various
|
||
matrices relevant for machine learning. We will often represent our
|
||
data in terms of matrices and vectors.</p>
|
||
<p>Let us introduce first some conventions. We assume that <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is a
|
||
vector of length <span class="math notranslate nohighlight">\(m\)</span>, that is it has <span class="math notranslate nohighlight">\(m\)</span> elements <span class="math notranslate nohighlight">\(y_0,y_1,\dots,
|
||
y_{m-1}\)</span>. By convention we start labeling vectors with the zeroth
|
||
element, as are arrays in Python and C++/C, for example. Similarly, we
|
||
have a vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> of length <span class="math notranslate nohighlight">\(n\)</span>, that is
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{x}^T=[x_0,x_1,\dots, x_{n-1}]\)</span>.</p>
|
||
<p>We assume also that <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is a function of <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> through some
|
||
given function <span class="math notranslate nohighlight">\(f\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{y}=f(\boldsymbol{x}).
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="the-jacobian">
|
||
<h2>The Jacobian<a class="headerlink" href="#the-jacobian" title="Permalink to this headline">¶</a></h2>
|
||
<p>We define the partial derivatives of the various components of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> as functions of <span class="math notranslate nohighlight">\(x_i\)</span> in terms of the so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Jacobian_matrix_and_determinant">Jacobian matrix</a></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{J}=\frac{\partial \boldsymbol{y}}{\partial \boldsymbol{x}}=\begin{bmatrix} \frac{\partial y_0}{\partial x_0} & \frac{\partial y_0}{\partial x_1} & \frac{\partial y_0}{\partial x_2} & \dots & \dots & \frac{\partial y_0}{\partial x_{n-1}} \\ \frac{\partial y_1}{\partial x_0} & \frac{\partial y_1}{\partial x_1} & \frac{\partial y_1}{\partial x_2} & \dots & \dots & \frac{\partial y_1}{\partial x_{n-1}} \\
|
||
\frac{\partial y_2}{\partial x_0} & \frac{\partial y_2}{\partial x_1} & \frac{\partial y_2}{\partial x_2} & \dots & \dots & \frac{\partial y_2}{\partial x_{n-1}} \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\frac{\partial y_{m-1}}{\partial x_0} & \frac{\partial y_{m-1}}{\partial x_1} & \frac{\partial y_{m-1}}{\partial x_2} & \dots & \dots & \frac{\partial y_{m-1}}{\partial x_{n-1}} \end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which is an <span class="math notranslate nohighlight">\(m\times n\)</span> matrix. If <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> is a scalar, then the
|
||
Jacobian is only a single-column vector, or an <span class="math notranslate nohighlight">\(m\times 1\)</span> matrix. If
|
||
on the other hand <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is a scalar, the Jacobian becomes a
|
||
<span class="math notranslate nohighlight">\(1\times n\)</span> matrix.</p>
|
||
<p>When this matrix is a square matrix <span class="math notranslate nohighlight">\(m=n\)</span>, its determinant is often referred to as the Jacobian
|
||
determinant. Both the matrix and (if <span class="math notranslate nohighlight">\(m=n\)</span>) the determinant are
|
||
often referred to simply as the Jacobian. The Jacobian matrix represents the differential of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> at every point where the
|
||
vector is differentiable.</p>
|
||
</div>
|
||
<div class="section" id="derivatives-example-1">
|
||
<h2>Derivatives, example 1<a class="headerlink" href="#derivatives-example-1" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let now <span class="math notranslate nohighlight">\(\boldsymbol{y}=\boldsymbol{A}\boldsymbol{x}\)</span>, where <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is an <span class="math notranslate nohighlight">\(m\times n\)</span> matrix and the matrix does not depend on <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span>. If we write out the vector <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> compoment by component we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
y_i = \sum_{j=0}^{n-1}a_{ij}x_j,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\forall i=0,1,2,\dots,m-1\)</span>. The individual matrix elements of <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> are given by the symbol <span class="math notranslate nohighlight">\(a_{ij}\)</span>.
|
||
It follows that the partial derivatives of <span class="math notranslate nohighlight">\(y_i\)</span> with respect to <span class="math notranslate nohighlight">\(x_k\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial y_i }{\partial x_k}= a_{ik} \forall i=0,1,2,\dots,m-1.
|
||
\]</div>
|
||
<p>From this we have, using the definition of the Jacobian</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \boldsymbol{y} }{\partial \boldsymbol{x}}= \boldsymbol{A}.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="example-2">
|
||
<h2>Example 2<a class="headerlink" href="#example-2" title="Permalink to this headline">¶</a></h2>
|
||
<p>We define a scalar (our cost/loss functions are in general also scalars,
|
||
just think of the mean squared error) as the result of some matrix vector
|
||
multiplications</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{y}^T\boldsymbol{A}\boldsymbol{x},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> a vector of length <span class="math notranslate nohighlight">\(m\)</span>, <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> an <span class="math notranslate nohighlight">\(m\times n\)</span> matrix and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> a vector of length <span class="math notranslate nohighlight">\(n\)</span>. We assume also that <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> does not depend on any of the two vectors.
|
||
In order to find the derivative of <span class="math notranslate nohighlight">\(\alpha\)</span> with respect to the two vectors, we define an intermediate vector <span class="math notranslate nohighlight">\(\boldsymbol{z}\)</span>. We define first
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{z}^T=\boldsymbol{y}^T\boldsymbol{A}\)</span>, a vector of length <span class="math notranslate nohighlight">\(n\)</span>. We have then, using the definition of the Jacobian,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{z}^T\boldsymbol{x},
|
||
\]</div>
|
||
<p>which means that (using our previous example) we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{x}} = \boldsymbol{z}=bm{A}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>Note that the resulting vector elements are the same for <span class="math notranslate nohighlight">\(\boldsymbol{z}^T\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{z}\)</span>, the only difference is that one if just the transpose of the other.</p>
|
||
<p>Since <span class="math notranslate nohighlight">\(\alpha\)</span> is a scalar we have <span class="math notranslate nohighlight">\(\alpha =\alpha^T=\boldsymbol{x}^T\boldsymbol{A}^T\boldsymbol{y}\)</span>. Defining now <span class="math notranslate nohighlight">\(\boldsymbol{z}=\boldsymbol{x}^T\boldsymbol{A}^T\)</span> we find that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{y}} = \boldsymbol{z}^T=\boldsymbol{x}^T\boldsymbol{A}^T.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="example-3">
|
||
<h2>Example 3<a class="headerlink" href="#example-3" title="Permalink to this headline">¶</a></h2>
|
||
<p>We start with a new scalar but where now the vector <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is
|
||
replaced by a vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> and the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is a square
|
||
matrix with dimension <span class="math notranslate nohighlight">\(n\times n\)</span>.</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{x}^T\boldsymbol{A}\boldsymbol{x},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> a vector of length <span class="math notranslate nohighlight">\(n\)</span>.</p>
|
||
<p>We write out the specific sums involved in the calculation of <span class="math notranslate nohighlight">\(\alpha\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \sum_{i=0}^{n-1}\sum_{j=0}^{n-1}x_i a_{ij}x_j,
|
||
\]</div>
|
||
<p>taking the derivative of <span class="math notranslate nohighlight">\(\alpha\)</span> with respect to a given component <span class="math notranslate nohighlight">\(x_k\)</span> we get the two sums</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial x_k} = \sum_{i=0}^{n-1}a_{ik}x_i+\sum_{j=0}^{n-1}a_{kj}x_j,
|
||
\]</div>
|
||
<p>for <span class="math notranslate nohighlight">\(\forall k =0,1,2,\dots,n-1\)</span>. We identify these sums as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{x}} = \boldsymbol{x}^T\left(\boldsymbol{A}^T+\boldsymbol{A}\right).
|
||
\]</div>
|
||
<p>If the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is symmetric, that is <span class="math notranslate nohighlight">\(\boldsymbol{A}=\boldsymbol{A}^T\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{x}} = 2\boldsymbol{x}^T\boldsymbol{A}.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="example-4">
|
||
<h2>Example 4<a class="headerlink" href="#example-4" title="Permalink to this headline">¶</a></h2>
|
||
<p>We let the scalar <span class="math notranslate nohighlight">\(\alpha\)</span> be defined by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \boldsymbol{y}^T\boldsymbol{x},
|
||
\]</div>
|
||
<p>where both <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> have the same length <span class="math notranslate nohighlight">\(n\)</span>, or if we
|
||
wish to think of them as column vectors, they have dimensions <span class="math notranslate nohighlight">\(n\times
|
||
1\)</span>. We assume that both <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> depend on a vector
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{z}\)</span> of the same length. To calculate the derivative of <span class="math notranslate nohighlight">\(\alpha\)</span>
|
||
with respect to a given component <span class="math notranslate nohighlight">\(z_k\)</span> we need first to write out the
|
||
inner product that defines <span class="math notranslate nohighlight">\(\alpha\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\alpha = \sum_{i=0}^{n-1}y_ix_i,
|
||
\]</div>
|
||
<p>and the partial derivative</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial z_k} = \sum_{i=0}^{n-1}\left(x_i\frac{\partial y_i}{\partial z_k}+y_i\frac{\partial x_i}{\partial z_k}\right),
|
||
\]</div>
|
||
<p>for <span class="math notranslate nohighlight">\(\forall k =0,1,2,\dots,n-1\)</span>. We can rewrite the partial derivative in a more compact form as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{z}} = \boldsymbol{x}^T\frac{\partial \boldsymbol{y}}{\partial \boldsymbol{z}}+\boldsymbol{y}^T\frac{\partial \boldsymbol{x}}{\partial \boldsymbol{z}},
|
||
\]</div>
|
||
<p>and if <span class="math notranslate nohighlight">\(\boldsymbol{y}=\boldsymbol{x}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \alpha}{\partial \boldsymbol{z}} = 2\boldsymbol{x}^T\frac{\partial \boldsymbol{x}}{\partial \boldsymbol{z}}.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="the-mean-squared-error-and-its-derivative">
|
||
<h2>The mean squared error and its derivative<a class="headerlink" href="#the-mean-squared-error-and-its-derivative" title="Permalink to this headline">¶</a></h2>
|
||
<p>We defined earlier a possible cost function using the mean squared error</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta})=\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)^T\left(\boldsymbol{y}-\boldsymbol{\tilde{y}}\right)\right\},
|
||
\]</div>
|
||
<p>or using the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> we have the more compact matrix-vector</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta})=\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
||
\]</div>
|
||
<p>We note that the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> does not depend on the unknown parameters defined by the vector <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>.
|
||
We are now interested in minimizing the cost function with respect to the unknown parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>.</p>
|
||
<p>The mean squared error is a scalar and if we use the results from example three above, we can define a new vector</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{w}=\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta},
|
||
\]</div>
|
||
<p>which depends on <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>. We rewrite the cost function as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta})=\frac{1}{n}\boldsymbol{w}^T\boldsymbol{w},
|
||
\]</div>
|
||
<p>with partial derivative</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}}=\frac{2}{n}\boldsymbol{w}^T\frac{\partial \boldsymbol{w}}{\partial \boldsymbol{\beta}},
|
||
\]</div>
|
||
<p>and using that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \boldsymbol{w}}{\partial \boldsymbol{\beta}}=-\boldsymbol{X},
|
||
\]</div>
|
||
<p>where we used the result from example two above. Inserting the last expression we obtain</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}}=-\frac{2}{n}\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\boldsymbol{X},
|
||
\]</div>
|
||
<p>or as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}^T}=-\frac{2}{n}\boldsymbol{X}^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right).
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="other-useful-relations">
|
||
<h2>Other useful relations<a class="headerlink" href="#other-useful-relations" title="Permalink to this headline">¶</a></h2>
|
||
<p>We list here some other useful relations we may encounter (recall that vectors are defined by boldfaced low-key letters)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial (\boldsymbol{b}^T\boldsymbol{a})}{\partial \boldsymbol{a}} = \boldsymbol{b},
|
||
\]</div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial tr(\boldsymbol{B}\boldsymbol{A})}{\partial \boldsymbol{A}} = \boldsymbol{B}^T,
|
||
\]</div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \log{\vert\boldsymbol{A}\vert}}{\partial \boldsymbol{A}} = (\boldsymbol{A}^{-1})^T.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="meet-the-hessian-matrix">
|
||
<h2>Meet the Hessian Matrix<a class="headerlink" href="#meet-the-hessian-matrix" title="Permalink to this headline">¶</a></h2>
|
||
<p>A very important matrix we will meet again and again in machine
|
||
learning is the Hessian. It is given by the second derivative of the
|
||
cost function with respect to the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>. Using the above
|
||
expression for derivatives of vectors and matrices, we find that the
|
||
second derivative of the mean squared error as cost function is,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial}{\partial \boldsymbol{\beta}}\frac{\partial C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}^T} =\frac{\partial}{\partial \boldsymbol{\beta}}\left[-\frac{2}{n}\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right]=\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>The Hessian matrix plays an important role and is defined here as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>For ordinary least squares, it is inversely proportional (derivation
|
||
next week) with the variance of the optimal parameters
|
||
<span class="math notranslate nohighlight">\(\hat{\boldsymbol{\beta}}\)</span>. Furthermore, we will see later this week that it is
|
||
(aside the factor <span class="math notranslate nohighlight">\(1/n\)</span>) equal to the covariance matrix. It plays also a very
|
||
important role in optmization algorithms and Principal Component
|
||
Analysis as a way to reduce the dimensionality of a machine learning/data analysis
|
||
problem.</p>
|
||
<p><strong>Linear algebra question:</strong> Can we use the Hessian matrix to say something about properties of the cost function (our optmization problem)? (hint: think about convex or concave problems and how to relate these to a matrix!).</p>
|
||
</div>
|
||
<div class="section" id="id2">
|
||
<h2>Interpretations and optimizing our parameters<a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h2>
|
||
<p>The residuals <span class="math notranslate nohighlight">\(\boldsymbol{\epsilon}\)</span> are in turn given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\epsilon} = \boldsymbol{y}-\boldsymbol{\tilde{y}} = \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta},
|
||
\]</div>
|
||
<p>and with</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)= 0,
|
||
\]</div>
|
||
<p>we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{\epsilon}=\boldsymbol{X}^T\left( \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)= 0,
|
||
\]</div>
|
||
<p>meaning that the solution for <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> is the one which minimizes the residuals.</p>
|
||
</div>
|
||
<div class="section" id="example-relevant-for-the-exercises">
|
||
<h2>Example relevant for the exercises<a class="headerlink" href="#example-relevant-for-the-exercises" title="Permalink to this headline">¶</a></h2>
|
||
<p>In order to understand the relation among the predictors <span class="math notranslate nohighlight">\(p\)</span>, the set of data <span class="math notranslate nohighlight">\(n\)</span> and the target (outcome, output etc) <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span>,
|
||
we condiser a simple polynomial fit.
|
||
We assume our data can represented by a fourth-order polynomial. For the <span class="math notranslate nohighlight">\(i\)</span>th component we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_i = \beta_0+\beta_1x_i+\beta_2x_i^2+\beta_3x_i^3+\beta_4x_i^4.
|
||
\]</div>
|
||
<p>we have five predictors/features. The first is the intercept <span class="math notranslate nohighlight">\(\beta_0\)</span>. The other terms are <span class="math notranslate nohighlight">\(\beta_i\)</span> with <span class="math notranslate nohighlight">\(i=1,2,3,4\)</span>. Furthermore we have <span class="math notranslate nohighlight">\(n\)</span> entries for each predictor. It means that our design matrix is an
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="own-code-for-ordinary-least-squares">
|
||
<h2>Own code for Ordinary Least Squares<a class="headerlink" href="#own-code-for-ordinary-least-squares" title="Permalink to this headline">¶</a></h2>
|
||
<p>It is rather straightforward to implement the matrix inversion and obtain the parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>. After having defined the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and the outputs <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> we have</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># matrix inversion to find beta</span>
|
||
<span class="c1"># First we set up the data</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">5</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mf">0.1</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="c1"># and then the design matrix X including the intercept</span>
|
||
<span class="c1"># The design matrix now as function of a fourth-order polynomial</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span><span class="mi">5</span><span class="p">))</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">3</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">3</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">4</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">4</span>
|
||
<span class="n">beta</span> <span class="o">=</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">inv</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span><span class="p">)</span> <span class="o">@</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="p">)</span> <span class="o">@</span> <span class="n">y</span>
|
||
<span class="c1"># and then make the prediction</span>
|
||
<span class="n">ytilde</span> <span class="o">=</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Alternatively, you can use the least squares functionality in <strong>Numpy</strong> as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">fit</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">lstsq</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">rcond</span> <span class="o">=</span><span class="kc">None</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||
<span class="n">ytildenp</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">dot</span><span class="p">(</span><span class="n">fit</span><span class="p">,</span><span class="n">X</span><span class="o">.</span><span class="n">T</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="adding-error-analysis-and-training-set-up">
|
||
<h2>Adding error analysis and training set up<a class="headerlink" href="#adding-error-analysis-and-training-set-up" title="Permalink to this headline">¶</a></h2>
|
||
<p>We can easily test our fit by computing the <span class="math notranslate nohighlight">\(R2\)</span> score that we discussed in connection with the functionality of <strong>Scikit-Learn</strong> in the introductory slides.
|
||
Since we are not using <strong>Scikit-Learn</strong> here we can define our own <span class="math notranslate nohighlight">\(R2\)</span> function as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">R2</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span> <span class="n">y_model</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">y_model</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_data</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>and we would be using it as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="n">R2</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.9959033816551833
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We can easily add our <strong>MSE</strong> score as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.009290411029763584
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>and finally the relative error as</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">RelativeError</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="nb">abs</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">/</span><span class="n">y_data</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">RelativeError</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">ytilde</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[0.02126934 0.07602623 0.0014937 0.01294617 0.05421908 0.02237669
|
||
0.0026081 0.00564956 0.00399247 0.05763394 0.00040953 0.06861085
|
||
0.0098629 0.01199845 0.0097577 0.03260201 0.020964 0.01933058
|
||
0.02255062 0.01801774 0.04039087 0.00472303 0.01003763 0.01487742
|
||
0.05554042 0.00886682 0.05110883 0.02944194 0.00806407 0.01028231
|
||
0.03613949 0.03352185 0.0512238 0.01525206 0.00660801 0.01073938
|
||
0.06353697 0.00700232 0.0391902 0.08741274 0.01227458 0.01049472
|
||
0.04691549 0.00963223 0.0143088 0.05177527 0.00850988 0.01121347
|
||
0.02768957 0.02259051 0.02233576 0.01322543 0.02143332 0.01400329
|
||
0.00102864 0.01322099 0.00611932 0.01011376 0.13281267 0.00684221
|
||
0.05358851 0.02232779 0.00695738 0.03054765 0.00554475 0.05748797
|
||
0.03507211 0.00563446 0.03123832 0.00033779 0.01122997 0.1098906
|
||
0.07003926 0.03718926 0.0695405 0.00605451 0.0456042 0.00477722
|
||
0.01224109 0.01072866 0.04273116 0.01873409 0.02903947 0.01927709
|
||
0.00819724 0.00628788 0.00086553 0.02341603 0.0525063 0.03546779
|
||
0.03012368 0.05069808 0.00327082 0.00517074 0.00071305 0.01194406
|
||
0.05454172 0.02480935 0.00577016 0.02925853]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="splitting-our-data-in-training-and-test-data">
|
||
<h2>Splitting our Data in Training and Test data<a class="headerlink" href="#splitting-our-data-in-training-and-test-data" title="Permalink to this headline">¶</a></h2>
|
||
<p>It is normal in essentially all Machine Learning studies to split the
|
||
data in a training set and a test set (sometimes also an additional
|
||
validation set). <strong>Scikit-Learn</strong> has an own function for this. There
|
||
is no explicit recipe for how much data should be included as training
|
||
data and say test data. An accepted rule of thumb is to use
|
||
approximately <span class="math notranslate nohighlight">\(2/3\)</span> to <span class="math notranslate nohighlight">\(4/5\)</span> of the data as training data. We will
|
||
postpone a discussion of this splitting to the end of these notes and
|
||
our discussion of the so-called <strong>bias-variance</strong> tradeoff. Here we
|
||
limit ourselves to repeat the above equation of state fitting example
|
||
but now splitting the data into a training set and a test set.</p>
|
||
</div>
|
||
<div class="section" id="the-complete-code-with-a-simple-data-set">
|
||
<h2>The complete code with a simple data set<a class="headerlink" href="#the-complete-code-with-a-simple-data-set" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
|
||
|
||
<span class="kn">import</span> <span class="nn">os</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">R2</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span> <span class="n">y_model</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">y_model</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">/</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_data</span><span class="p">))</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mf">2.0</span><span class="o">+</span><span class="mi">5</span><span class="o">*</span><span class="n">x</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mf">0.1</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># The design matrix now as function of a fourth-order polynomial</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span><span class="mi">5</span><span class="p">))</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="mf">1.0</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">2</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">2</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">3</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">3</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="mi">4</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="o">**</span><span class="mi">4</span>
|
||
<span class="c1"># We split the data in test and training data</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
<span class="c1"># matrix inversion to find beta</span>
|
||
<span class="n">beta</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">inv</span><span class="p">(</span><span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X_train</span><span class="p">)</span> <span class="o">@</span> <span class="n">X_train</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y_train</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">beta</span><span class="p">)</span>
|
||
<span class="c1"># and then make the prediction</span>
|
||
<span class="n">ytilde</span> <span class="o">=</span> <span class="n">X_train</span> <span class="o">@</span> <span class="n">beta</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Training R2"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">R2</span><span class="p">(</span><span class="n">y_train</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Training MSE"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y_train</span><span class="p">,</span><span class="n">ytilde</span><span class="p">))</span>
|
||
<span class="n">ypredict</span> <span class="o">=</span> <span class="n">X_test</span> <span class="o">@</span> <span class="n">beta</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Test R2"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">R2</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredict</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Test MSE"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span><span class="n">ypredict</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[ 2.04860436 -0.39444293 5.97533203 -0.78980112 0.09575221]
|
||
Training R2
|
||
0.9960913291755783
|
||
Training MSE
|
||
0.008823125370709272
|
||
Test R2
|
||
0.9906601091977738
|
||
Test MSE
|
||
0.01969004537138309
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="making-your-own-test-train-splitting">
|
||
<h2>Making your own test-train splitting<a class="headerlink" href="#making-your-own-test-train-splitting" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># equivalently in numpy</span>
|
||
<span class="k">def</span> <span class="nf">train_test_split_numpy</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">labels</span><span class="p">,</span> <span class="n">train_size</span><span class="p">,</span> <span class="n">test_size</span><span class="p">):</span>
|
||
<span class="n">n_inputs</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
|
||
<span class="n">inputs_shuffled</span> <span class="o">=</span> <span class="n">inputs</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
<span class="n">labels_shuffled</span> <span class="o">=</span> <span class="n">labels</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
|
||
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">inputs_shuffled</span><span class="p">)</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="n">labels_shuffled</span><span class="p">)</span>
|
||
|
||
<span class="n">train_end</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">n_inputs</span><span class="o">*</span><span class="n">train_size</span><span class="p">)</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span> <span class="o">=</span> <span class="n">inputs_shuffled</span><span class="p">[:</span><span class="n">train_end</span><span class="p">],</span> <span class="n">inputs_shuffled</span><span class="p">[</span><span class="n">train_end</span><span class="p">:]</span>
|
||
<span class="n">Y_train</span><span class="p">,</span> <span class="n">Y_test</span> <span class="o">=</span> <span class="n">labels_shuffled</span><span class="p">[:</span><span class="n">train_end</span><span class="p">],</span> <span class="n">labels_shuffled</span><span class="p">[</span><span class="n">train_end</span><span class="p">:]</span>
|
||
|
||
<span class="k">return</span> <span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">,</span> <span class="n">Y_test</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>But since <strong>scikit-learn</strong> has its own function for doing this and since
|
||
it interfaces easily with <strong>tensorflow</strong> and other libraries, we
|
||
normally recommend using the latter functionality.</p>
|
||
</div>
|
||
<div class="section" id="reducing-the-number-of-degrees-of-freedom-overarching-view">
|
||
<h2>Reducing the number of degrees of freedom, overarching view<a class="headerlink" href="#reducing-the-number-of-degrees-of-freedom-overarching-view" title="Permalink to this headline">¶</a></h2>
|
||
<p>Many Machine Learning problems involve thousands or even millions of
|
||
features for each training instance. Not only does this make training
|
||
extremely slow, it can also make it much harder to find a good
|
||
solution, as we will see. This problem is often referred to as the
|
||
curse of dimensionality. Fortunately, in real-world problems, it is
|
||
often possible to reduce the number of features considerably, turning
|
||
an intractable problem into a tractable one.</p>
|
||
<p>Later we will discuss some of the most popular dimensionality reduction
|
||
techniques: the principal component analysis (PCA), Kernel PCA, and
|
||
Locally Linear Embedding (LLE).</p>
|
||
<p>Principal component analysis and its various variants deal with the
|
||
problem of fitting a low-dimensional <a class="reference external" href="https://en.wikipedia.org/wiki/Affine_space">affine
|
||
subspace</a> to a set of of
|
||
data points in a high-dimensional space. With its family of methods it
|
||
is one of the most used tools in data modeling, compression and
|
||
visualization.</p>
|
||
</div>
|
||
<div class="section" id="preprocessing-our-data">
|
||
<h2>Preprocessing our data<a class="headerlink" href="#preprocessing-our-data" title="Permalink to this headline">¶</a></h2>
|
||
<p>Before we proceed however, we will discuss how to preprocess our
|
||
data. Till now and in connection with our previous examples we have
|
||
not met so many cases where we are too sensitive to the scaling of our
|
||
data. Normally the data may need a rescaling and/or may be sensitive
|
||
to extreme values. Scaling the data renders our inputs much more
|
||
suitable for the algorithms we want to employ.</p>
|
||
<p>For data sets gathered for real world applications, it is rather normal that
|
||
different features have very different units and
|
||
numerical scales. For example, a data set detailing health habits may include
|
||
features such as <strong>age</strong> in the range <span class="math notranslate nohighlight">\(0-80\)</span>, and <strong>caloric intake</strong> of order <span class="math notranslate nohighlight">\(2000\)</span>.
|
||
Many machine learning methods sensitive to the scales of the features and may perform poorly if they
|
||
are very different scales. Therefore, it is typical to scale
|
||
the features in a way to avoid such outlier values.</p>
|
||
</div>
|
||
<div class="section" id="functionality-in-scikit-learn">
|
||
<h2>Functionality in Scikit-Learn<a class="headerlink" href="#functionality-in-scikit-learn" title="Permalink to this headline">¶</a></h2>
|
||
<p><strong>Scikit-Learn</strong> has several functions which allow us to rescale the
|
||
data, normally resulting in much better results in terms of various
|
||
accuracy scores. The <strong>StandardScaler</strong> function in <strong>Scikit-Learn</strong>
|
||
ensures that for each feature/predictor we study the mean value is
|
||
zero and the variance is one (every column in the design/feature
|
||
matrix). This scaling has the drawback that it does not ensure that
|
||
we have a particular maximum or minimum in our data set. Another
|
||
function included in <strong>Scikit-Learn</strong> is the <strong>MinMaxScaler</strong> which
|
||
ensures that all features are exactly between <span class="math notranslate nohighlight">\(0\)</span> and <span class="math notranslate nohighlight">\(1\)</span>. The</p>
|
||
</div>
|
||
<div class="section" id="more-preprocessing">
|
||
<h2>More preprocessing<a class="headerlink" href="#more-preprocessing" title="Permalink to this headline">¶</a></h2>
|
||
<p>The <strong>Normalizer</strong> scales each data
|
||
point such that the feature vector has a euclidean length of one. In other words, it
|
||
projects a data point on the circle (or sphere in the case of higher dimensions) with a
|
||
radius of 1. This means every data point is scaled by a different number (by the
|
||
inverse of it’s length).
|
||
This normalization is often used when only the direction (or angle) of the data matters,
|
||
not the length of the feature vector.</p>
|
||
<p>The <strong>RobustScaler</strong> works similarly to the StandardScaler in that it
|
||
ensures statistical properties for each feature that guarantee that
|
||
they are on the same scale. However, the RobustScaler uses the median
|
||
and quartiles, instead of mean and variance. This makes the
|
||
RobustScaler ignore data points that are very different from the rest
|
||
(like measurement errors). These odd data points are also called
|
||
outliers, and might often lead to trouble for other scaling
|
||
techniques.</p>
|
||
</div>
|
||
<div class="section" id="frequently-used-scaling-functions">
|
||
<h2>Frequently used scaling functions<a class="headerlink" href="#frequently-used-scaling-functions" title="Permalink to this headline">¶</a></h2>
|
||
<p>Many features are often scaled using standardization to improve performance. In <strong>Scikit-Learn</strong> this is given by the <strong>StandardScaler</strong> function as discussed above. It is easy however to write your own.
|
||
Mathematically, this involves subtracting the mean and divide by the standard deviation over the data set, for each feature:</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
x_j^{(i)} \rightarrow \frac{x_j^{(i)} - \overline{x}_j}{\sigma(x_j)},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\overline{x}_j\)</span> and <span class="math notranslate nohighlight">\(\sigma(x_j)\)</span> are the mean and standard deviation, respectively, of the feature <span class="math notranslate nohighlight">\(x_j\)</span>.
|
||
This ensures that each feature has zero mean and unit standard deviation. For data sets where we do not have the standard deviation or don’t wish to calculate it, it is then common to simply set it to one.</p>
|
||
</div>
|
||
<div class="section" id="example-of-own-standard-scaling">
|
||
<h2>Example of own Standard scaling<a class="headerlink" href="#example-of-own-standard-scaling" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let us consider the following vanilla example where we use both
|
||
<strong>Scikit-Learn</strong> and write our own function as well. We produce a
|
||
simple test design matrix with random numbers. Each column could then
|
||
represent a specific feature whose mean value is subracted.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">sklearn.linear_model</span> <span class="k">as</span> <span class="nn">skl</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_error</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">MinMaxScaler</span><span class="p">,</span> <span class="n">StandardScaler</span><span class="p">,</span> <span class="n">Normalizer</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">from</span> <span class="nn">IPython.display</span> <span class="kn">import</span> <span class="n">display</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">100</span><span class="p">)</span>
|
||
<span class="c1"># setting up a 10 x 5 matrix</span>
|
||
<span class="n">rows</span> <span class="o">=</span> <span class="mi">10</span>
|
||
<span class="n">cols</span> <span class="o">=</span> <span class="mi">5</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">rows</span><span class="p">,</span><span class="n">cols</span><span class="p">)</span>
|
||
<span class="n">XPandas</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">display</span><span class="p">(</span><span class="n">XPandas</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">XPandas</span><span class="o">.</span><span class="n">mean</span><span class="p">())</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">XPandas</span><span class="o">.</span><span class="n">std</span><span class="p">())</span>
|
||
<span class="n">XPandas</span> <span class="o">=</span> <span class="p">(</span><span class="n">XPandas</span> <span class="o">-</span><span class="n">XPandas</span><span class="o">.</span><span class="n">mean</span><span class="p">())</span>
|
||
<span class="n">display</span><span class="p">(</span><span class="n">XPandas</span><span class="p">)</span>
|
||
<span class="c1"># This option does not include the standard deviation</span>
|
||
<span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="n">with_std</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
<span class="n">scaler</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">Xscaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">display</span><span class="p">(</span><span class="n">XPandas</span><span class="o">-</span><span class="n">Xscaled</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output text_html"><div>
|
||
<style scoped>
|
||
.dataframe tbody tr th:only-of-type {
|
||
vertical-align: middle;
|
||
}
|
||
|
||
.dataframe tbody tr th {
|
||
vertical-align: top;
|
||
}
|
||
|
||
.dataframe thead th {
|
||
text-align: right;
|
||
}
|
||
</style>
|
||
<table border="1" class="dataframe">
|
||
<thead>
|
||
<tr style="text-align: right;">
|
||
<th></th>
|
||
<th>0</th>
|
||
<th>1</th>
|
||
<th>2</th>
|
||
<th>3</th>
|
||
<th>4</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<th>0</th>
|
||
<td>-1.749765</td>
|
||
<td>0.342680</td>
|
||
<td>1.153036</td>
|
||
<td>-0.252436</td>
|
||
<td>0.981321</td>
|
||
</tr>
|
||
<tr>
|
||
<th>1</th>
|
||
<td>0.514219</td>
|
||
<td>0.221180</td>
|
||
<td>-1.070043</td>
|
||
<td>-0.189496</td>
|
||
<td>0.255001</td>
|
||
</tr>
|
||
<tr>
|
||
<th>2</th>
|
||
<td>-0.458027</td>
|
||
<td>0.435163</td>
|
||
<td>-0.583595</td>
|
||
<td>0.816847</td>
|
||
<td>0.672721</td>
|
||
</tr>
|
||
<tr>
|
||
<th>3</th>
|
||
<td>-0.104411</td>
|
||
<td>-0.531280</td>
|
||
<td>1.029733</td>
|
||
<td>-0.438136</td>
|
||
<td>-1.118318</td>
|
||
</tr>
|
||
<tr>
|
||
<th>4</th>
|
||
<td>1.618982</td>
|
||
<td>1.541605</td>
|
||
<td>-0.251879</td>
|
||
<td>-0.842436</td>
|
||
<td>0.184519</td>
|
||
</tr>
|
||
<tr>
|
||
<th>5</th>
|
||
<td>0.937082</td>
|
||
<td>0.731000</td>
|
||
<td>1.361556</td>
|
||
<td>-0.326238</td>
|
||
<td>0.055676</td>
|
||
</tr>
|
||
<tr>
|
||
<th>6</th>
|
||
<td>0.222400</td>
|
||
<td>-1.443217</td>
|
||
<td>-0.756352</td>
|
||
<td>0.816454</td>
|
||
<td>0.750445</td>
|
||
</tr>
|
||
<tr>
|
||
<th>7</th>
|
||
<td>-0.455947</td>
|
||
<td>1.189622</td>
|
||
<td>-1.690617</td>
|
||
<td>-1.356399</td>
|
||
<td>-1.232435</td>
|
||
</tr>
|
||
<tr>
|
||
<th>8</th>
|
||
<td>-0.544439</td>
|
||
<td>-0.668172</td>
|
||
<td>0.007315</td>
|
||
<td>-0.612939</td>
|
||
<td>1.299748</td>
|
||
</tr>
|
||
<tr>
|
||
<th>9</th>
|
||
<td>-1.733096</td>
|
||
<td>-0.983310</td>
|
||
<td>0.357508</td>
|
||
<td>-1.613579</td>
|
||
<td>1.470714</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div></div><div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0 -0.175300
|
||
1 0.083527
|
||
2 -0.044334
|
||
3 -0.399836
|
||
4 0.331939
|
||
dtype: float64
|
||
0 1.069584
|
||
1 0.965548
|
||
2 1.018232
|
||
3 0.793167
|
||
4 0.918992
|
||
dtype: float64
|
||
</pre></div>
|
||
</div>
|
||
<div class="output text_html"><div>
|
||
<style scoped>
|
||
.dataframe tbody tr th:only-of-type {
|
||
vertical-align: middle;
|
||
}
|
||
|
||
.dataframe tbody tr th {
|
||
vertical-align: top;
|
||
}
|
||
|
||
.dataframe thead th {
|
||
text-align: right;
|
||
}
|
||
</style>
|
||
<table border="1" class="dataframe">
|
||
<thead>
|
||
<tr style="text-align: right;">
|
||
<th></th>
|
||
<th>0</th>
|
||
<th>1</th>
|
||
<th>2</th>
|
||
<th>3</th>
|
||
<th>4</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<th>0</th>
|
||
<td>-1.574465</td>
|
||
<td>0.259153</td>
|
||
<td>1.197370</td>
|
||
<td>0.147400</td>
|
||
<td>0.649382</td>
|
||
</tr>
|
||
<tr>
|
||
<th>1</th>
|
||
<td>0.689519</td>
|
||
<td>0.137652</td>
|
||
<td>-1.025709</td>
|
||
<td>0.210340</td>
|
||
<td>-0.076938</td>
|
||
</tr>
|
||
<tr>
|
||
<th>2</th>
|
||
<td>-0.282727</td>
|
||
<td>0.351636</td>
|
||
<td>-0.539261</td>
|
||
<td>1.216683</td>
|
||
<td>0.340782</td>
|
||
</tr>
|
||
<tr>
|
||
<th>3</th>
|
||
<td>0.070889</td>
|
||
<td>-0.614808</td>
|
||
<td>1.074067</td>
|
||
<td>-0.038300</td>
|
||
<td>-1.450257</td>
|
||
</tr>
|
||
<tr>
|
||
<th>4</th>
|
||
<td>1.794282</td>
|
||
<td>1.458078</td>
|
||
<td>-0.207545</td>
|
||
<td>-0.442600</td>
|
||
<td>-0.147420</td>
|
||
</tr>
|
||
<tr>
|
||
<th>5</th>
|
||
<td>1.112383</td>
|
||
<td>0.647473</td>
|
||
<td>1.405890</td>
|
||
<td>0.073598</td>
|
||
<td>-0.276263</td>
|
||
</tr>
|
||
<tr>
|
||
<th>6</th>
|
||
<td>0.397700</td>
|
||
<td>-1.526744</td>
|
||
<td>-0.712018</td>
|
||
<td>1.216290</td>
|
||
<td>0.418506</td>
|
||
</tr>
|
||
<tr>
|
||
<th>7</th>
|
||
<td>-0.280647</td>
|
||
<td>1.106095</td>
|
||
<td>-1.646283</td>
|
||
<td>-0.956563</td>
|
||
<td>-1.564374</td>
|
||
</tr>
|
||
<tr>
|
||
<th>8</th>
|
||
<td>-0.369139</td>
|
||
<td>-0.751699</td>
|
||
<td>0.051649</td>
|
||
<td>-0.213103</td>
|
||
<td>0.967809</td>
|
||
</tr>
|
||
<tr>
|
||
<th>9</th>
|
||
<td>-1.557795</td>
|
||
<td>-1.066837</td>
|
||
<td>0.401842</td>
|
||
<td>-1.213743</td>
|
||
<td>1.138775</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div></div><div class="output text_html"><div>
|
||
<style scoped>
|
||
.dataframe tbody tr th:only-of-type {
|
||
vertical-align: middle;
|
||
}
|
||
|
||
.dataframe tbody tr th {
|
||
vertical-align: top;
|
||
}
|
||
|
||
.dataframe thead th {
|
||
text-align: right;
|
||
}
|
||
</style>
|
||
<table border="1" class="dataframe">
|
||
<thead>
|
||
<tr style="text-align: right;">
|
||
<th></th>
|
||
<th>0</th>
|
||
<th>1</th>
|
||
<th>2</th>
|
||
<th>3</th>
|
||
<th>4</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<th>0</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>1</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>2</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>3</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>4</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>5</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>6</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>7</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>8</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
<tr>
|
||
<th>9</th>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
<td>0.0</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div></div></div>
|
||
</div>
|
||
<p>Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.</p>
|
||
</div>
|
||
<div class="section" id="min-max-scaling">
|
||
<h2>Min-Max Scaling<a class="headerlink" href="#min-max-scaling" title="Permalink to this headline">¶</a></h2>
|
||
<p>Another commonly used scaling method is min-max scaling. This is very
|
||
useful for when we want the features to lie in a certain interval. To
|
||
scale the feature <span class="math notranslate nohighlight">\(x_j\)</span> to the interval <span class="math notranslate nohighlight">\([a, b]\)</span>, we can apply the
|
||
transformation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
x_j^{(i)} \rightarrow (b-a)\frac{x_j^{(i)} - \min(x_j)}{\max(x_j) - \min(x_j)} - a
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\min(x_j)\)</span> and <span class="math notranslate nohighlight">\(\max(x_j)\)</span> return the minimum and maximum value of <span class="math notranslate nohighlight">\(x_j\)</span> over the data set, respectively.</p>
|
||
</div>
|
||
<div class="section" id="testing-the-means-squared-error-as-function-of-complexity">
|
||
<h2>Testing the Means Squared Error as function of Complexity<a class="headerlink" href="#testing-the-means-squared-error-as-function-of-complexity" title="Permalink to this headline">¶</a></h2>
|
||
<p>One of
|
||
the aims is to reproduce Figure 2.11 of <a class="reference external" href="https://github.com/CompPhysics/MLErasmus/blob/master/doc/Textbooks/elementsstat.pdf">Hastie et al</a>.</p>
|
||
<p>Our data is defined by <span class="math notranslate nohighlight">\(x\in [-3,3]\)</span> with a total of for example <span class="math notranslate nohighlight">\(100\)</span> data points.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">()</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">maxdegree</span> <span class="o">=</span> <span class="mi">14</span>
|
||
<span class="c1"># Make data set.</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>where <span class="math notranslate nohighlight">\(y\)</span> is the function we want to fit with a given polynomial.</p>
|
||
<p>Write a first code which sets up a design matrix <span class="math notranslate nohighlight">\(X\)</span> defined by a fourth-order polynomial. Scale your data and split it in training and test data.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">PolynomialFeatures</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.pipeline</span> <span class="kn">import</span> <span class="n">make_pipeline</span>
|
||
|
||
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">2018</span><span class="p">)</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">50</span>
|
||
<span class="n">maxdegree</span> <span class="o">=</span> <span class="mi">5</span>
|
||
<span class="c1"># Make data set.</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
<span class="n">TestError</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">)</span>
|
||
<span class="n">TrainError</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">)</span>
|
||
<span class="n">polydegree</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">)</span>
|
||
<span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
<span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">()</span>
|
||
<span class="n">scaler</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">x_train</span><span class="p">)</span>
|
||
<span class="n">x_train_scaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">x_train</span><span class="p">)</span>
|
||
<span class="n">x_test_scaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">x_test</span><span class="p">)</span>
|
||
|
||
<span class="k">for</span> <span class="n">degree</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">maxdegree</span><span class="p">):</span>
|
||
<span class="n">model</span> <span class="o">=</span> <span class="n">make_pipeline</span><span class="p">(</span><span class="n">PolynomialFeatures</span><span class="p">(</span><span class="n">degree</span><span class="o">=</span><span class="n">degree</span><span class="p">),</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
|
||
<span class="n">clf</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">x_train_scaled</span><span class="p">,</span><span class="n">y_train</span><span class="p">)</span>
|
||
<span class="n">y_fit</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">x_train_scaled</span><span class="p">)</span>
|
||
<span class="n">y_pred</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">x_test_scaled</span><span class="p">)</span>
|
||
<span class="n">polydegree</span><span class="p">[</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">degree</span>
|
||
<span class="n">TestError</span><span class="p">[</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">((</span><span class="n">y_test</span> <span class="o">-</span> <span class="n">y_pred</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="p">)</span>
|
||
<span class="n">TrainError</span><span class="p">[</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">((</span><span class="n">y_train</span> <span class="o">-</span> <span class="n">y_fit</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="p">)</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">polydegree</span><span class="p">,</span> <span class="n">TestError</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">'Test Error'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">polydegree</span><span class="p">,</span> <span class="n">TrainError</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s1">'Train Error'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<img alt="_images/week35_146_0.png" src="_images/week35_146_0.png" />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="more-preprocessing-examples-two-dimensional-example-the-franke-function">
|
||
<h2>More preprocessing examples, two-dimensional example, the Franke function<a class="headerlink" href="#more-preprocessing-examples-two-dimensional-example-the-franke-function" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Common imports</span>
|
||
<span class="kn">import</span> <span class="nn">os</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
<span class="kn">import</span> <span class="nn">sklearn.linear_model</span> <span class="k">as</span> <span class="nn">skl</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_error</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">MinMaxScaler</span><span class="p">,</span> <span class="n">StandardScaler</span><span class="p">,</span> <span class="n">Normalizer</span>
|
||
|
||
<span class="c1"># Where to save the figures and data files</span>
|
||
<span class="n">PROJECT_ROOT_DIR</span> <span class="o">=</span> <span class="s2">"Results"</span>
|
||
<span class="n">FIGURE_ID</span> <span class="o">=</span> <span class="s2">"Results/FigureFiles"</span>
|
||
<span class="n">DATA_ID</span> <span class="o">=</span> <span class="s2">"DataFiles/"</span>
|
||
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">PROJECT_ROOT_DIR</span><span class="p">):</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">PROJECT_ROOT_DIR</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">FIGURE_ID</span><span class="p">):</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">FIGURE_ID</span><span class="p">)</span>
|
||
|
||
<span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">DATA_ID</span><span class="p">):</span>
|
||
<span class="n">os</span><span class="o">.</span><span class="n">makedirs</span><span class="p">(</span><span class="n">DATA_ID</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">image_path</span><span class="p">(</span><span class="n">fig_id</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">FIGURE_ID</span><span class="p">,</span> <span class="n">fig_id</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">data_path</span><span class="p">(</span><span class="n">dat_id</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">DATA_ID</span><span class="p">,</span> <span class="n">dat_id</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">save_fig</span><span class="p">(</span><span class="n">fig_id</span><span class="p">):</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">savefig</span><span class="p">(</span><span class="n">image_path</span><span class="p">(</span><span class="n">fig_id</span><span class="p">)</span> <span class="o">+</span> <span class="s2">".png"</span><span class="p">,</span> <span class="nb">format</span><span class="o">=</span><span class="s1">'png'</span><span class="p">)</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">FrankeFunction</span><span class="p">(</span><span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">):</span>
|
||
<span class="n">term1</span> <span class="o">=</span> <span class="mf">0.75</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mf">0.25</span><span class="o">*</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">-</span> <span class="mf">0.25</span><span class="o">*</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">term2</span> <span class="o">=</span> <span class="mf">0.75</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="mf">49.0</span> <span class="o">-</span> <span class="mf">0.1</span><span class="o">*</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">+</span><span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">term3</span> <span class="o">=</span> <span class="mf">0.5</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">7</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="o">/</span><span class="mf">4.0</span> <span class="o">-</span> <span class="mf">0.25</span><span class="o">*</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">3</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">term4</span> <span class="o">=</span> <span class="o">-</span><span class="mf">0.2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">4</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span> <span class="o">-</span> <span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">7</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">term1</span> <span class="o">+</span> <span class="n">term2</span> <span class="o">+</span> <span class="n">term3</span> <span class="o">+</span> <span class="n">term4</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">create_X</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n</span> <span class="p">):</span>
|
||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="n">N</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">l</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">n</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">n</span><span class="o">+</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span> <span class="c1"># Number of elements in beta</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">N</span><span class="p">,</span><span class="n">l</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="n">n</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">q</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">i</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="n">q</span><span class="o">+</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="p">(</span><span class="n">i</span><span class="o">-</span><span class="n">k</span><span class="p">))</span><span class="o">*</span><span class="p">(</span><span class="n">y</span><span class="o">**</span><span class="n">k</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">X</span>
|
||
|
||
|
||
<span class="c1"># Making meshgrid of datapoints and compute Franke's function</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">5</span>
|
||
<span class="n">N</span> <span class="o">=</span> <span class="mi">1000</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">N</span><span class="p">))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">N</span><span class="p">))</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">FrankeFunction</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">create_X</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="c1"># split in training and test data</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span><span class="n">z</span><span class="p">,</span><span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||
|
||
|
||
<span class="n">clf</span> <span class="o">=</span> <span class="n">skl</span><span class="o">.</span><span class="n">LinearRegression</span><span class="p">()</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
|
||
|
||
<span class="c1"># The mean squared error and R2 score</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE before scaling: </span><span class="si">{:.2f}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">),</span> <span class="n">y_test</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"R2 score before scaling </span><span class="si">{:.2f}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span><span class="n">y_test</span><span class="p">)))</span>
|
||
|
||
<span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">()</span>
|
||
<span class="n">scaler</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">)</span>
|
||
<span class="n">X_train_scaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X_train</span><span class="p">)</span>
|
||
<span class="n">X_test_scaled</span> <span class="o">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Feature min values before scaling:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">X_train</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Feature max values before scaling:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">X_train</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)))</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Feature min values after scaling:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">X_train_scaled</span><span class="o">.</span><span class="n">min</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"Feature max values after scaling:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">X_train_scaled</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)))</span>
|
||
|
||
<span class="n">clf</span> <span class="o">=</span> <span class="n">skl</span><span class="o">.</span><span class="n">LinearRegression</span><span class="p">()</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train_scaled</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
|
||
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"MSE after scaling: </span><span class="si">{:.2f}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test_scaled</span><span class="p">),</span> <span class="n">y_test</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"R2 score for scaled data: </span><span class="si">{:.2f}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test_scaled</span><span class="p">,</span><span class="n">y_test</span><span class="p">)))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>MSE before scaling: 0.00
|
||
R2 score before scaling 1.00
|
||
Feature min values before scaling:
|
||
[1.00000000e+00 6.97906022e-03 2.43639284e-03 4.87072815e-05
|
||
1.70037324e-05 5.93601008e-06 3.39931051e-07 1.18670072e-07
|
||
4.14277718e-08 1.44624525e-08 2.37239927e-09 8.28205578e-10
|
||
2.89126914e-10 1.00934327e-10 3.52362157e-11 1.65571174e-11
|
||
5.78009660e-12 2.01783414e-12 7.04426744e-13 2.45915671e-13
|
||
8.58492636e-14]
|
||
Feature max values before scaling:
|
||
[1. 0.99970894 0.99978365 0.99941797 0.99949266 0.99956735
|
||
0.99912709 0.99920175 0.99927642 0.9993511 0.99883628 0.99891093
|
||
0.99898558 0.99906023 0.99913489 0.99854557 0.99862019 0.99869482
|
||
0.99876945 0.99884409 0.99891873]
|
||
Feature min values after scaling:
|
||
[ 0. -1.71761101 -1.75770568 -1.12330033 -1.12591227 -1.12871842
|
||
-0.88613493 -0.884669 -0.88323026 -0.88182591 -0.75269037 -0.75050135
|
||
-0.74829661 -0.74607851 -0.74384949 -0.6652177 -0.66294408 -0.66064822
|
||
-0.65833132 -0.65599456 -0.6536392 ]
|
||
Feature max values after scaling:
|
||
[0. 1.71737253 1.75576555 2.20916295 2.23971032 2.26995402
|
||
2.60543038 2.63162342 2.65743689 2.68286725 2.94273542 2.96631321
|
||
2.98947894 3.01222822 3.034557 3.24159785 3.26297455 3.28391978
|
||
3.30442964 3.32450054 3.34412923]
|
||
MSE after scaling: 0.00
|
||
R2 score for scaled data: 1.00
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="to-think-about-first-part">
|
||
<h2>To think about, first part<a class="headerlink" href="#to-think-about-first-part" title="Permalink to this headline">¶</a></h2>
|
||
<p>When you are comparing your own code with for example <strong>Scikit-Learn</strong>’s
|
||
library, there are some technicalities to keep in mind. The examples
|
||
here demonstrate some of these aspects with potential pitfalls.</p>
|
||
<p>The discussion here focuses on the role of the intercept, how we can
|
||
set up the design matrix, what scaling we should use and other topics
|
||
which tend confuse us.</p>
|
||
<p>The intercept can be interpreted as the expected value of our
|
||
target/output variables when all other predictors are set to zero.
|
||
Thus, if we cannot assume that the expected outputs/targets are zero
|
||
when all predictors are zero (the columns in the design matrix), it
|
||
may be a bad idea to implement a model which penalizes the intercept.
|
||
Furthermore, in for example Ridge and Lasso regression (to be discussed in moe detail next week), the default solutions
|
||
from the library <strong>Scikit-Learn</strong> (when not shrinking <span class="math notranslate nohighlight">\(\beta_0\)</span>) for the unknown parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>, are derived under the assumption that both <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> and
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> are zero centered, that is we subtract the mean values.</p>
|
||
</div>
|
||
<div class="section" id="more-thinking">
|
||
<h2>More thinking<a class="headerlink" href="#more-thinking" title="Permalink to this headline">¶</a></h2>
|
||
<p>If our predictors represent different scales, then it is important to
|
||
standardize the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> by subtracting the mean of each
|
||
column from the corresponding column and dividing the column with its
|
||
standard deviation. Most machine learning libraries do this as a default. This means that if you compare your code with the results from a given library,
|
||
the results may differ.</p>
|
||
<p>The
|
||
<a class="reference external" href="https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html">Standadscaler</a>
|
||
function in <strong>Scikit-Learn</strong> does this for us. For the data sets we
|
||
have been studying in our various examples, the data are in many cases
|
||
already scaled and there is no need to scale them. You as a user of different machine learning algorithms, should always perform a
|
||
survey of your data, with a critical assessment of them in case you need to scale the data.</p>
|
||
<p>If you need to scale the data, not doing so will give an <em>unfair</em>
|
||
penalization of the parameters since their magnitude depends on the
|
||
scale of their corresponding predictor.</p>
|
||
<p>Suppose as an example that you
|
||
you have an input variable given by the heights of different persons.
|
||
Human height might be measured in inches or meters or
|
||
kilometers. If measured in kilometers, a standard linear regression
|
||
model with this predictor would probably give a much bigger
|
||
coefficient term, than if measured in millimeters.
|
||
This can clearly lead to problems in evaluating the cost/loss functions.</p>
|
||
</div>
|
||
<div class="section" id="still-thinking">
|
||
<h2>Still thinking<a class="headerlink" href="#still-thinking" title="Permalink to this headline">¶</a></h2>
|
||
<p>Keep in mind that when you transform your data set before training a model, the same transformation needs to be done
|
||
on your eventual new data set before making a prediction. If we translate this into a Python code, it would could be implemented as follows
|
||
(note that the lines are commented since the model function has not been defined)</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1">#Model training, we compute the mean value of y and X</span>
|
||
<span class="n">y_train_mean</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_train</span><span class="p">)</span>
|
||
<span class="n">X_train_mean</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span><span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">X_train</span> <span class="o">=</span> <span class="n">X_train</span> <span class="o">-</span> <span class="n">X_train_mean</span>
|
||
<span class="n">y_train</span> <span class="o">=</span> <span class="n">y_train</span> <span class="o">-</span> <span class="n">y_train_mean</span>
|
||
|
||
<span class="c1"># The we fit our model with the training data</span>
|
||
<span class="c1">#trained_model = some_model.fit(X_train,y_train)</span>
|
||
|
||
|
||
<span class="c1">#Model prediction, we need also to transform our data set used for the prediction.</span>
|
||
<span class="n">X_test</span> <span class="o">=</span> <span class="n">X_test</span> <span class="o">-</span> <span class="n">X_train_mean</span> <span class="c1">#Use mean from training data</span>
|
||
<span class="c1">#y_pred = trained_model(X_test)</span>
|
||
<span class="n">y_pred</span> <span class="o">=</span> <span class="n">y_pred</span> <span class="o">+</span> <span class="n">y_train_mean</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="what-does-centering-subtracting-the-mean-values-mean-mathematically">
|
||
<h2>What does centering (subtracting the mean values) mean mathematically?<a class="headerlink" href="#what-does-centering-subtracting-the-mean-values-mean-mathematically" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let us try to understand what this may imply mathematically when we
|
||
subtract the mean values, also known as <em>zero centering</em>. For
|
||
simplicity, we will focus on ordinary regression, as done in the above example.</p>
|
||
<p>The cost/loss function for regression is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\beta_0, \beta_1, ... , \beta_{p-1}) = \frac{1}{n}\sum_{i=0}^{n} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij}\beta_j\right)^2,.
|
||
\]</div>
|
||
<p>Recall also that we use the squared value since this leads to an increase of the penalty for higher differences between predicted and output/target values.</p>
|
||
<p>What we have done is to single out the <span class="math notranslate nohighlight">\(\beta_0\)</span> term in the definition of the mean squared error (MSE).
|
||
The design matrix
|
||
<span class="math notranslate nohighlight">\(X\)</span> does in this case not contain any intercept column.
|
||
When we take the derivative with respect to <span class="math notranslate nohighlight">\(\beta_0\)</span>, we want the derivative to obey</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C}{\partial \beta_j} = 0,
|
||
\]</div>
|
||
<p>for all <span class="math notranslate nohighlight">\(j\)</span>. For <span class="math notranslate nohighlight">\(\beta_0\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C}{\partial \beta_0} = -\frac{2}{n}\sum_{i=0}^{n-1} \left(y_i - \beta_0 - \sum_{j=1}^{p-1} X_{ij} \beta_j\right).
|
||
\]</div>
|
||
<p>Multiplying away the constant <span class="math notranslate nohighlight">\(2/n\)</span>, we obtain</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sum_{i=0}^{n-1} \beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} \sum_{j=1}^{p-1} X_{ij} \beta_j.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="further-manipulations">
|
||
<h2>Further Manipulations<a class="headerlink" href="#further-manipulations" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let us special first to the case where we have only two parameters <span class="math notranslate nohighlight">\(\beta_0\)</span> and <span class="math notranslate nohighlight">\(\beta_1\)</span>.
|
||
Our result for <span class="math notranslate nohighlight">\(\beta_0\)</span> simplifies then to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
n\beta_0 = \sum_{i=0}^{n-1}y_i - \sum_{i=0}^{n-1} X_{i1} \beta_1.
|
||
\]</div>
|
||
<p>We obtain then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \beta_1\frac{1}{n}\sum_{i=0}^{n-1} X_{i1}.
|
||
\]</div>
|
||
<p>If we define</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mu_1=\frac{1}{n}\sum_{i=0}^{n-1} (X_{i1},
|
||
\]</div>
|
||
<p>and if we define the mean value of the outputs as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mu_y=\frac{1}{n}\sum_{i=0}^{n-1}y_i,
|
||
\]</div>
|
||
<p>we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_0 = \mu_y - \beta_1\mu_{1}.
|
||
\]</div>
|
||
<p>In the general case, that is we have more parameters than <span class="math notranslate nohighlight">\(\beta_0\)</span> and <span class="math notranslate nohighlight">\(\beta_1\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_0 = \frac{1}{n}\sum_{i=0}^{n-1}y_i - \frac{1}{n}\sum_{i=0}^{n-1}\sum_{j=1}^{p-1} X_{ij}\beta_j.
|
||
\]</div>
|
||
<p>Replacing <span class="math notranslate nohighlight">\(y_i\)</span> with <span class="math notranslate nohighlight">\(y_i - y_i - \overline{\boldsymbol{y}}\)</span> and centering also our design matrix results in a cost function (in vector-matrix disguise)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{\beta}) = (\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta})^T(\boldsymbol{\tilde{y}} - \tilde{X}\boldsymbol{\beta}).
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="wrapping-it-up">
|
||
<h2>Wrapping it up<a class="headerlink" href="#wrapping-it-up" title="Permalink to this headline">¶</a></h2>
|
||
<p>If we minimize with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> we have then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X})^{-1}\tilde{X}^T\boldsymbol{\tilde{y}},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}} = \boldsymbol{y} - \overline{\boldsymbol{y}}\)</span>
|
||
and <span class="math notranslate nohighlight">\(\tilde{X}_{ij} = X_{ij} - \frac{1}{n}\sum_{k=0}^{n-1}X_{kj}\)</span>.</p>
|
||
<p>For Ridge regression we need to add <span class="math notranslate nohighlight">\(\lambda \boldsymbol{\beta}^T\boldsymbol{\beta}\)</span> to the cost function and get then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}} = (\tilde{X}^T\tilde{X} + \lambda I)^{-1}\tilde{X}^T\boldsymbol{\tilde{y}}.
|
||
\]</div>
|
||
<p>What does this mean? And why do we insist on all this? Let us look at some examples.</p>
|
||
</div>
|
||
<div class="section" id="linear-regression-code-intercept-handling-first">
|
||
<h2>Linear Regression code, Intercept handling first<a class="headerlink" href="#linear-regression-code-intercept-handling-first" title="Permalink to this headline">¶</a></h2>
|
||
<p>This code shows a simple first-order fit to a data set using the above transformed data, where we consider the role of the intercept first, by either excluding it or including it (<em>code example thanks to Øyvind Sigmundson Schøyen</em>). Here our scaling of the data is done by subtracting the mean values only.
|
||
Note also that we do not split the data into training and test.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
|
||
|
||
|
||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">2021</span><span class="p">)</span>
|
||
|
||
<span class="k">def</span> <span class="nf">MSE</span><span class="p">(</span><span class="n">y_data</span><span class="p">,</span><span class="n">y_model</span><span class="p">):</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">y_model</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">((</span><span class="n">y_data</span><span class="o">-</span><span class="n">y_model</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">fit_beta</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">):</span>
|
||
<span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">pinv</span><span class="p">(</span><span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">X</span><span class="p">)</span> <span class="o">@</span> <span class="n">X</span><span class="o">.</span><span class="n">T</span> <span class="o">@</span> <span class="n">y</span>
|
||
|
||
|
||
<span class="n">true_beta</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mf">3.7</span><span class="p">]</span>
|
||
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">11</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span>
|
||
<span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">([</span><span class="n">x</span> <span class="o">**</span> <span class="n">p</span> <span class="o">*</span> <span class="n">b</span> <span class="k">for</span> <span class="n">p</span><span class="p">,</span> <span class="n">b</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">true_beta</span><span class="p">)]),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span>
|
||
<span class="p">)</span> <span class="o">+</span> <span class="mf">0.1</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
|
||
|
||
<span class="n">degree</span> <span class="o">=</span> <span class="mi">3</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">degree</span><span class="p">))</span>
|
||
|
||
<span class="c1"># Include the intercept in the design matrix</span>
|
||
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">degree</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span> <span class="n">p</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="n">p</span>
|
||
|
||
<span class="n">beta</span> <span class="o">=</span> <span class="n">fit_beta</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Intercept is included in the design matrix</span>
|
||
<span class="n">skl</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"True beta: </span><span class="si">{</span><span class="n">true_beta</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Fitted beta: </span><span class="si">{</span><span class="n">beta</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sklearn fitted beta: </span><span class="si">{</span><span class="n">skl</span><span class="o">.</span><span class="n">coef_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="n">ypredictOwn</span> <span class="o">=</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span>
|
||
<span class="n">ypredictSKL</span> <span class="o">=</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with intercept column"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictOwn</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with intercept column from SKL"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictSKL</span><span class="p">))</span>
|
||
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Data"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Fit"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">),</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Sklearn (fit_intercept=False)"</span><span class="p">)</span>
|
||
|
||
|
||
<span class="c1"># Do not include the intercept in the design matrix</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">degree</span> <span class="o">-</span> <span class="mi">1</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">degree</span> <span class="o">-</span> <span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span> <span class="n">p</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span> <span class="o">**</span> <span class="p">(</span><span class="n">p</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Intercept is not included in the design matrix</span>
|
||
<span class="n">skl</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="c1"># Use centered values for X and y when computing coefficients</span>
|
||
<span class="n">y_offset</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">average</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
<span class="n">X_offset</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">average</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||
|
||
<span class="n">beta</span> <span class="o">=</span> <span class="n">fit_beta</span><span class="p">(</span><span class="n">X</span> <span class="o">-</span> <span class="n">X_offset</span><span class="p">,</span> <span class="n">y</span> <span class="o">-</span> <span class="n">y_offset</span><span class="p">)</span>
|
||
<span class="n">intercept</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y_offset</span> <span class="o">-</span> <span class="n">X_offset</span> <span class="o">@</span> <span class="n">beta</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Manual intercept: </span><span class="si">{</span><span class="n">intercept</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Fitted beta (wiothout intercept): </span><span class="si">{</span><span class="n">beta</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sklearn intercept: </span><span class="si">{</span><span class="n">skl</span><span class="o">.</span><span class="n">intercept_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Sklearn fitted beta (without intercept): </span><span class="si">{</span><span class="n">skl</span><span class="o">.</span><span class="n">coef_</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||
<span class="n">ypredictOwn</span> <span class="o">=</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span>
|
||
<span class="n">ypredictSKL</span> <span class="o">=</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with Manual intercept"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictOwn</span><span class="o">+</span><span class="n">intercept</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">"MSE with Sklearn intercept"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">MSE</span><span class="p">(</span><span class="n">y</span><span class="p">,</span><span class="n">ypredictSKL</span><span class="p">))</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">X</span> <span class="o">@</span> <span class="n">beta</span> <span class="o">+</span> <span class="n">intercept</span><span class="p">,</span> <span class="s2">"--"</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Fit (manual intercept)"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">skl</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">),</span> <span class="s2">"--"</span><span class="p">,</span> <span class="n">label</span><span class="o">=</span><span class="s2">"Sklearn (fit_intercept=True)"</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">grid</span><span class="p">()</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>True beta: [2, 0.5, 3.7]
|
||
Fitted beta: [2.08376632 0.19569961 3.97898392]
|
||
Sklearn fitted beta: [2.08376632 0.19569961 3.97898392]
|
||
MSE with intercept column
|
||
0.004113634617443139
|
||
MSE with intercept column from SKL
|
||
0.004113634617443147
|
||
Manual intercept: 2.083766322923899
|
||
Fitted beta (wiothout intercept): [0.19569961 3.97898392]
|
||
Sklearn intercept: 2.0837663229239043
|
||
Sklearn fitted beta (without intercept): [0.19569961 3.97898392]
|
||
MSE with Manual intercept
|
||
0.00411363461744314
|
||
MSE with Sklearn intercept
|
||
0.004113634617443131
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/week35_181_1.png" src="_images/week35_181_1.png" />
|
||
</div>
|
||
</div>
|
||
<p>The intercept is the value of our output/target variable
|
||
when all our features are zero and our function crosses the <span class="math notranslate nohighlight">\(y\)</span>-axis (for a one-dimensional case).</p>
|
||
<p>Printing the MSE, we see first that both methods give the same MSE, as
|
||
they should. However, when we move to for example Ridge regression (discussed next week),
|
||
the way we treat the intercept may give a larger or smaller MSE,
|
||
meaning that the MSE can be penalized by the value of the
|
||
intercept. Not including the intercept in the fit, means that the
|
||
regularization term does not include <span class="math notranslate nohighlight">\(\beta_0\)</span>. For different values
|
||
of <span class="math notranslate nohighlight">\(\lambda\)</span>, this may lead to differing MSE values.</p>
|
||
<p>To remind the reader, the regularization term, with the intercept in Ridge regression is given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=0}^{p-1}\beta_j^2,
|
||
\]</div>
|
||
<p>but when we take out the intercept, this equation becomes</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_2^2 = \lambda \sum_{j=1}^{p-1}\beta_j^2.
|
||
\]</div>
|
||
<p>For Lasso regression we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\lambda \vert\vert \boldsymbol{\beta} \vert\vert_1 = \lambda \sum_{j=1}^{p-1}\vert\beta_j\vert.
|
||
\]</div>
|
||
<p>It means that, when scaling the design matrix and the outputs/targets,
|
||
by subtracting the mean values, we have an optimization problem which
|
||
is not penalized by the intercept. The MSE value can then be smaller
|
||
since it focuses only on the remaining quantities. If we however bring
|
||
back the intercept, we will get an MSE which then contains the
|
||
intercept. This becomes more important when we discuss Ridge and Lasso
|
||
regression next week.</p>
|
||
</div>
|
||
<div class="section" id="the-boston-housing-data-example">
|
||
<h2>The Boston housing data example<a class="headerlink" href="#the-boston-housing-data-example" title="Permalink to this headline">¶</a></h2>
|
||
<p>The Boston housing<br />
|
||
data set was originally a part of UCI Machine Learning Repository
|
||
and has been removed now. The data set is now included in <strong>Scikit-Learn</strong>’s
|
||
library. There are 506 samples and 13 feature (predictor) variables
|
||
in this data set. The objective is to predict the value of prices of
|
||
the house using the features (predictors) listed here.</p>
|
||
<p>The features/predictors are</p>
|
||
<ol class="simple">
|
||
<li><p>CRIM: Per capita crime rate by town</p></li>
|
||
<li><p>ZN: Proportion of residential land zoned for lots over 25000 square feet</p></li>
|
||
<li><p>INDUS: Proportion of non-retail business acres per town</p></li>
|
||
<li><p>CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)</p></li>
|
||
<li><p>NOX: Nitric oxide concentration (parts per 10 million)</p></li>
|
||
<li><p>RM: Average number of rooms per dwelling</p></li>
|
||
<li><p>AGE: Proportion of owner-occupied units built prior to 1940</p></li>
|
||
<li><p>DIS: Weighted distances to five Boston employment centers</p></li>
|
||
<li><p>RAD: Index of accessibility to radial highways</p></li>
|
||
<li><p>TAX: Full-value property tax rate per USD10000</p></li>
|
||
<li><p>B: <span class="math notranslate nohighlight">\(1000(Bk - 0.63)^2\)</span>, where <span class="math notranslate nohighlight">\(Bk\)</span> is the proportion of [people of African American descent] by town</p></li>
|
||
<li><p>LSTAT: Percentage of lower status of the population</p></li>
|
||
<li><p>MEDV: Median value of owner-occupied homes in USD 1000s</p></li>
|
||
</ol>
|
||
</div>
|
||
<div class="section" id="housing-data-the-code">
|
||
<h2>Housing data, the code<a class="headerlink" href="#housing-data-the-code" title="Permalink to this headline">¶</a></h2>
|
||
<p>We start by importing the libraries</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="kn">import</span> <span class="nn">seaborn</span> <span class="k">as</span> <span class="nn">sns</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>and load the Boston Housing DataSet from <strong>Scikit-Learn</strong></p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">load_boston</span>
|
||
|
||
<span class="n">boston_dataset</span> <span class="o">=</span> <span class="n">load_boston</span><span class="p">()</span>
|
||
|
||
<span class="c1"># boston_dataset is a dictionary</span>
|
||
<span class="c1"># let's check what it contains</span>
|
||
<span class="n">boston_dataset</span><span class="o">.</span><span class="n">keys</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2.
|
||
|
||
The Boston housing prices dataset has an ethical problem. You can refer to
|
||
the documentation of this function for further details.
|
||
|
||
The scikit-learn maintainers therefore strongly discourage the use of this
|
||
dataset unless the purpose of the code is to study and educate about
|
||
ethical issues in data science and machine learning.
|
||
|
||
In this special case, you can fetch the dataset from the original
|
||
source::
|
||
|
||
import pandas as pd
|
||
import numpy as np
|
||
|
||
|
||
data_url = "http://lib.stat.cmu.edu/datasets/boston"
|
||
raw_df = pd.read_csv(data_url, sep="\s+", skiprows=22, header=None)
|
||
data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])
|
||
target = raw_df.values[1::2, 2]
|
||
|
||
Alternative datasets include the California housing dataset (i.e.
|
||
:func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing
|
||
dataset. You can load the datasets as follows::
|
||
|
||
from sklearn.datasets import fetch_california_housing
|
||
housing = fetch_california_housing()
|
||
|
||
for the California housing dataset and::
|
||
|
||
from sklearn.datasets import fetch_openml
|
||
housing = fetch_openml(name="house_prices", as_frame=True)
|
||
|
||
for the Ames housing dataset.
|
||
|
||
warnings.warn(msg, category=FutureWarning)
|
||
</pre></div>
|
||
</div>
|
||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>dict_keys(['data', 'target', 'feature_names', 'DESCR', 'filename', 'data_module'])
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Then we invoke Pandas</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">boston</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">boston_dataset</span><span class="o">.</span><span class="n">data</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="n">boston_dataset</span><span class="o">.</span><span class="n">feature_names</span><span class="p">)</span>
|
||
<span class="n">boston</span><span class="o">.</span><span class="n">head</span><span class="p">()</span>
|
||
<span class="n">boston</span><span class="p">[</span><span class="s1">'MEDV'</span><span class="p">]</span> <span class="o">=</span> <span class="n">boston_dataset</span><span class="o">.</span><span class="n">target</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>and preprocess the data</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># check for missing values in all the columns</span>
|
||
<span class="n">boston</span><span class="o">.</span><span class="n">isnull</span><span class="p">()</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>CRIM 0
|
||
ZN 0
|
||
INDUS 0
|
||
CHAS 0
|
||
NOX 0
|
||
RM 0
|
||
AGE 0
|
||
DIS 0
|
||
RAD 0
|
||
TAX 0
|
||
PTRATIO 0
|
||
B 0
|
||
LSTAT 0
|
||
MEDV 0
|
||
dtype: int64
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We can then visualize the data</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># set the size of the figure</span>
|
||
<span class="n">sns</span><span class="o">.</span><span class="n">set</span><span class="p">(</span><span class="n">rc</span><span class="o">=</span><span class="p">{</span><span class="s1">'figure.figsize'</span><span class="p">:(</span><span class="mf">11.7</span><span class="p">,</span><span class="mf">8.27</span><span class="p">)})</span>
|
||
|
||
<span class="c1"># plot a histogram showing the distribution of the target values</span>
|
||
<span class="n">sns</span><span class="o">.</span><span class="n">distplot</span><span class="p">(</span><span class="n">boston</span><span class="p">[</span><span class="s1">'MEDV'</span><span class="p">],</span> <span class="n">bins</span><span class="o">=</span><span class="mi">30</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/miniforge3/envs/myenv/lib/python3.9/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
|
||
warnings.warn(msg, FutureWarning)
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/week35_199_1.png" src="_images/week35_199_1.png" />
|
||
</div>
|
||
</div>
|
||
<p>It is now useful to look at the correlation matrix</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># compute the pair wise correlation for all columns </span>
|
||
<span class="n">correlation_matrix</span> <span class="o">=</span> <span class="n">boston</span><span class="o">.</span><span class="n">corr</span><span class="p">()</span><span class="o">.</span><span class="n">round</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="c1"># use the heatmap function from seaborn to plot the correlation matrix</span>
|
||
<span class="c1"># annot = True to print the values inside the square</span>
|
||
<span class="n">sns</span><span class="o">.</span><span class="n">heatmap</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">correlation_matrix</span><span class="p">,</span> <span class="n">annot</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output text_plain highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><AxesSubplot:>
|
||
</pre></div>
|
||
</div>
|
||
<img alt="_images/week35_201_1.png" src="_images/week35_201_1.png" />
|
||
</div>
|
||
</div>
|
||
<p>From the above coorelation plot we can see that <strong>MEDV</strong> is strongly correlated to <strong>LSTAT</strong> and <strong>RM</strong>. We see also that <strong>RAD</strong> and <strong>TAX</strong> are stronly correlated, but we don’t include this in our features together to avoid multi-colinearity</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
|
||
|
||
<span class="n">features</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'LSTAT'</span><span class="p">,</span> <span class="s1">'RM'</span><span class="p">]</span>
|
||
<span class="n">target</span> <span class="o">=</span> <span class="n">boston</span><span class="p">[</span><span class="s1">'MEDV'</span><span class="p">]</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">features</span><span class="p">):</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">features</span><span class="p">)</span> <span class="p">,</span> <span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">boston</span><span class="p">[</span><span class="n">col</span><span class="p">]</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">target</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">marker</span><span class="o">=</span><span class="s1">'o'</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="n">col</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">'MEDV'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<img alt="_images/week35_203_0.png" src="_images/week35_203_0.png" />
|
||
</div>
|
||
</div>
|
||
<p>Now we start training our model</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">X</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">c_</span><span class="p">[</span><span class="n">boston</span><span class="p">[</span><span class="s1">'LSTAT'</span><span class="p">],</span> <span class="n">boston</span><span class="p">[</span><span class="s1">'RM'</span><span class="p">]],</span> <span class="n">columns</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'LSTAT'</span><span class="p">,</span><span class="s1">'RM'</span><span class="p">])</span>
|
||
<span class="n">Y</span> <span class="o">=</span> <span class="n">boston</span><span class="p">[</span><span class="s1">'MEDV'</span><span class="p">]</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We split the data into training and test sets</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
|
||
|
||
<span class="c1"># splits the training and test data set in 80% : 20%</span>
|
||
<span class="c1"># assign random_state to any value.This ensures consistency.</span>
|
||
<span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">,</span> <span class="n">Y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">Y</span><span class="p">,</span> <span class="n">test_size</span> <span class="o">=</span> <span class="mf">0.2</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X_train</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X_test</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">Y_train</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">Y_test</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>(404, 2)
|
||
(102, 2)
|
||
(404,)
|
||
(102,)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Then we use the linear regression functionality from <strong>Scikit-Learn</strong></p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span>
|
||
<span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_error</span><span class="p">,</span> <span class="n">r2_score</span>
|
||
|
||
<span class="n">lin_model</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">()</span>
|
||
<span class="n">lin_model</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">Y_train</span><span class="p">)</span>
|
||
|
||
<span class="c1"># model evaluation for training set</span>
|
||
|
||
<span class="n">y_train_predict</span> <span class="o">=</span> <span class="n">lin_model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_train</span><span class="p">)</span>
|
||
<span class="n">rmse</span> <span class="o">=</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">Y_train</span><span class="p">,</span> <span class="n">y_train_predict</span><span class="p">)))</span>
|
||
<span class="n">r2</span> <span class="o">=</span> <span class="n">r2_score</span><span class="p">(</span><span class="n">Y_train</span><span class="p">,</span> <span class="n">y_train_predict</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"The model performance for training set"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"--------------------------------------"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'RMSE is </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">rmse</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'R2 score is </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">r2</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="p">)</span>
|
||
|
||
<span class="c1"># model evaluation for testing set</span>
|
||
|
||
<span class="n">y_test_predict</span> <span class="o">=</span> <span class="n">lin_model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X_test</span><span class="p">)</span>
|
||
<span class="c1"># root mean square error of the model</span>
|
||
<span class="n">rmse</span> <span class="o">=</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">Y_test</span><span class="p">,</span> <span class="n">y_test_predict</span><span class="p">)))</span>
|
||
|
||
<span class="c1"># r-squared score of the model</span>
|
||
<span class="n">r2</span> <span class="o">=</span> <span class="n">r2_score</span><span class="p">(</span><span class="n">Y_test</span><span class="p">,</span> <span class="n">y_test_predict</span><span class="p">)</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"The model performance for testing set"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s2">"--------------------------------------"</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'RMSE is </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">rmse</span><span class="p">))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'R2 score is </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">r2</span><span class="p">))</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>The model performance for training set
|
||
--------------------------------------
|
||
RMSE is 5.637129335071195
|
||
R2 score is 0.6300745149331701
|
||
|
||
|
||
The model performance for testing set
|
||
--------------------------------------
|
||
RMSE is 5.137400784702911
|
||
R2 score is 0.6628996975186953
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># plotting the y_test vs y_pred</span>
|
||
<span class="c1"># ideally should have been a straight line</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">Y_test</span><span class="p">,</span> <span class="n">y_test_predict</span><span class="p">)</span>
|
||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<img alt="_images/week35_210_0.png" src="_images/week35_210_0.png" />
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="material-for-lecture-thursday-august-31">
|
||
<h2>Material for lecture Thursday, August 31<a class="headerlink" href="#material-for-lecture-thursday-august-31" title="Permalink to this headline">¶</a></h2>
|
||
</div>
|
||
<div class="section" id="mathematical-interpretation-of-ordinary-least-squares">
|
||
<h2>Mathematical Interpretation of Ordinary Least Squares<a class="headerlink" href="#mathematical-interpretation-of-ordinary-least-squares" title="Permalink to this headline">¶</a></h2>
|
||
<p>What is presented here is a mathematical analysis of various regression algorithms (ordinary least squares, Ridge and Lasso Regression). The analysis is based on an important algorithm in linear algebra, the so-called Singular Value Decomposition (SVD).</p>
|
||
<p>We have shown that in ordinary least squares the optimal parameters <span class="math notranslate nohighlight">\(\beta\)</span> are given by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The <strong>hat</strong> over <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> means we have the optimal parameters after minimization of the cost function.</p>
|
||
<p>This means that our best model is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>We now define a matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T.
|
||
\]</div>
|
||
<p>We can rewrite</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}=\boldsymbol{X}\hat{\boldsymbol{\beta}} = \boldsymbol{A}\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> has the important property that <span class="math notranslate nohighlight">\(\boldsymbol{A}^2=\boldsymbol{A}\)</span>. This is the definition of a projection matrix.
|
||
We can then interpret our optimal model <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{y}}\)</span> as being represented by an orthogonal projection of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> onto a space defined by the column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>. In our case here the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> is a square matrix. If it is a general rectangular matrix we have an oblique projection matrix.</p>
|
||
</div>
|
||
<div class="section" id="residual-error">
|
||
<h2>Residual Error<a class="headerlink" href="#residual-error" title="Permalink to this headline">¶</a></h2>
|
||
<p>We have defined the residual error as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=\left[\boldsymbol{I}-\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\right]\boldsymbol{y}.
|
||
\]</div>
|
||
<p>The residual errors are then the projections of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> onto the orthogonal component of the space defined by the column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="simple-case">
|
||
<h2>Simple case<a class="headerlink" href="#simple-case" title="Permalink to this headline">¶</a></h2>
|
||
<p>If the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is an orthogonal (or unitary in case of complex values) matrix, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{X}\boldsymbol{X}^T = \boldsymbol{I}.
|
||
\]</div>
|
||
<p>In this case the matrix <span class="math notranslate nohighlight">\(\boldsymbol{A}\)</span> becomes</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{A}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T)=\boldsymbol{I},
|
||
\]</div>
|
||
<p>and we have the obvious case</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\epsilon}=\boldsymbol{y}-\tilde{\boldsymbol{y}}=0.
|
||
\]</div>
|
||
<p>This serves also as a useful test of our codes.</p>
|
||
</div>
|
||
<div class="section" id="the-singular-value-decomposition">
|
||
<h2>The singular value decomposition<a class="headerlink" href="#the-singular-value-decomposition" title="Permalink to this headline">¶</a></h2>
|
||
<p>The examples we have looked at so far are cases where we normally can
|
||
invert the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. Using a polynomial expansion where we fit of various functions leads to
|
||
row vectors of the design matrix which are essentially orthogonal due
|
||
to the polynomial character of our model. Obtaining the inverse of the
|
||
design matrix is then often done via a so-called LU, QR or Cholesky
|
||
decomposition.</p>
|
||
<p>As we will also see in the first project,
|
||
this may
|
||
however not the be case in general and a standard matrix inversion
|
||
algorithm based on say LU, QR or Cholesky decomposition may lead to singularities. We will see examples of this below.</p>
|
||
<p>There is however a way to circumvent this problem and also
|
||
gain some insights about the ordinary least squares approach, and
|
||
later shrinkage methods like Ridge and Lasso regressions.</p>
|
||
<p>This is given by the <strong>Singular Value Decomposition</strong> (SVD) algorithm,
|
||
perhaps the most powerful linear algebra algorithm. The SVD provides
|
||
a numerically stable matrix decomposition that is used in a large
|
||
swath oc applications and the decomposition is always stable
|
||
numerically.</p>
|
||
<p>In machine learning it plays a central role in dealing with for
|
||
example design matrices that may be near singular or singular.
|
||
Furthermore, as we will see here, the singular values can be related
|
||
to the covariance matrix (and thereby the correlation matrix) and in
|
||
turn the variance of a given quantity. It plays also an important role
|
||
in the principal component analysis where high-dimensional data can be
|
||
reduced to the statistically relevant features.</p>
|
||
</div>
|
||
<div class="section" id="linear-regression-problems">
|
||
<h2>Linear Regression Problems<a class="headerlink" href="#linear-regression-problems" title="Permalink to this headline">¶</a></h2>
|
||
<p>One of the typical problems we encounter with linear regression, in particular
|
||
when the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> (our so-called design matrix) is high-dimensional,
|
||
are problems with near singular or singular matrices. The column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>
|
||
may be linearly dependent, normally referred to as super-collinearity.<br />
|
||
This means that the matrix may be rank deficient and it is basically impossible to
|
||
to model the data using linear regression. As an example, consider the matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
\mathbf{X} & = \left[
|
||
\begin{array}{rrr}
|
||
1 & -1 & 2
|
||
\\
|
||
1 & 0 & 1
|
||
\\
|
||
1 & 2 & -1
|
||
\\
|
||
1 & 1 & 0
|
||
\end{array} \right]
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>The columns of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> are linearly dependent. We see this easily since the
|
||
the first column is the row-wise sum of the other two columns. The rank (more correct,
|
||
the column rank) of a matrix is the dimension of the space spanned by the
|
||
column vectors. Hence, the rank of <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> is equal to the number
|
||
of linearly independent columns. In this particular case the matrix has rank 2.</p>
|
||
<p>Super-collinearity of an <span class="math notranslate nohighlight">\((n \times p)\)</span>-dimensional design matrix <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> implies
|
||
that the inverse of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
\boldsymbol{X} & = \left[
|
||
\begin{array}{rr}
|
||
1 & -1
|
||
\\
|
||
1 & -1
|
||
\end{array} \right].
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>We see easily that <span class="math notranslate nohighlight">\(\mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0\)</span>. Hence, <span class="math notranslate nohighlight">\(\mathbf{X}\)</span> is singular and its inverse is undefined.
|
||
This is equivalent to saying that the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has at least an eigenvalue which is zero.</p>
|
||
</div>
|
||
<div class="section" id="fixing-the-singularity">
|
||
<h2>Fixing the singularity<a class="headerlink" href="#fixing-the-singularity" title="Permalink to this headline">¶</a></h2>
|
||
<p>If our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> which enters the linear regression problem</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\boldsymbol{\beta} = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y},
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>has linearly dependent column vectors, we will not be able to compute the inverse
|
||
of <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> and we cannot find the parameters (estimators) <span class="math notranslate nohighlight">\(\beta_i\)</span>.
|
||
The estimators are only well-defined if <span class="math notranslate nohighlight">\((\boldsymbol{X}^{T}\boldsymbol{X})^{-1}\)</span> exits.
|
||
This is more likely to happen when the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is high-dimensional. In this case it is likely to encounter a situation where
|
||
the regression parameters <span class="math notranslate nohighlight">\(\beta_i\)</span> cannot be estimated.</p>
|
||
<p>A cheap <em>ad hoc</em> approach is simply to add a small diagonal component to the matrix to invert, that is we change</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{I}\)</span> is the identity matrix. When we discuss <strong>Ridge</strong> regression this is actually what we end up evaluating. The parameter <span class="math notranslate nohighlight">\(\lambda\)</span> is called a hyperparameter. More about this later.</p>
|
||
</div>
|
||
<div class="section" id="basic-math-of-the-svd">
|
||
<h2>Basic math of the SVD<a class="headerlink" href="#basic-math-of-the-svd" title="Permalink to this headline">¶</a></h2>
|
||
<p>From standard linear algebra we know that a square matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be diagonalized if and only it is
|
||
a so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Normal_matrix">normal matrix</a>, that is if <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times n}\)</span>
|
||
we have <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X}\)</span> or if <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{C}}^{n\times n}\)</span> we have <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X}\)</span>.
|
||
The matrix has then a set of eigenpairs</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n),
|
||
\]</div>
|
||
<p>and the eigenvalues are given by the diagonal matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n).
|
||
\]</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be written in terms of an orthogonal/unitary transformation <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I}\)</span> or <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I}\)</span>.</p>
|
||
<p>Not all square matrices are diagonalizable. A matrix like the one discussed above</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X} = \begin{bmatrix}
|
||
1& -1 \\
|
||
1& -1\\
|
||
\end{bmatrix}
|
||
\end{split}\]</div>
|
||
<p>is not diagonalizable, it is a so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Defective_matrix">defective matrix</a>. It is easy to see that the condition
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X}\)</span> is not fulfilled.</p>
|
||
</div>
|
||
<div class="section" id="the-svd-a-fantastic-algorithm">
|
||
<h2>The SVD, a Fantastic Algorithm<a class="headerlink" href="#the-svd-a-fantastic-algorithm" title="Permalink to this headline">¶</a></h2>
|
||
<p>However, and this is the strength of the SVD algorithm, any general
|
||
matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be decomposed in terms of a diagonal matrix and
|
||
two orthogonal/unitary matrices. The <a class="reference external" href="https://en.wikipedia.org/wiki/Singular_value_decomposition">Singular Value Decompostion
|
||
(SVD) theorem</a>
|
||
states that a general <span class="math notranslate nohighlight">\(m\times n\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> can be written in
|
||
terms of a diagonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> of dimensionality <span class="math notranslate nohighlight">\(m\times n\)</span>
|
||
and two orthognal matrices <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>, where the first has
|
||
dimensionality <span class="math notranslate nohighlight">\(m \times m\)</span> and the last dimensionality <span class="math notranslate nohighlight">\(n\times n\)</span>.
|
||
We have then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T
|
||
\]</div>
|
||
<p>As an example, the above defective matrix can be decomposed as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\end{split}\]</div>
|
||
<p>with eigenvalues <span class="math notranslate nohighlight">\(\sigma_1=2\)</span> and <span class="math notranslate nohighlight">\(\sigma_2=0\)</span>.
|
||
The SVD exits always!</p>
|
||
<p>The SVD
|
||
decomposition (singular values) gives eigenvalues
|
||
<span class="math notranslate nohighlight">\(\sigma_i\geq\sigma_{i+1}\)</span> for all <span class="math notranslate nohighlight">\(i\)</span> and for dimensions larger than <span class="math notranslate nohighlight">\(i=p\)</span>, the
|
||
eigenvalues (singular values) are zero.</p>
|
||
<p>In the general case, where our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has dimension
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span>, the matrix is thus decomposed into an <span class="math notranslate nohighlight">\(n\times n\)</span>
|
||
orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>, a <span class="math notranslate nohighlight">\(p\times p\)</span> orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>
|
||
and a diagonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> with <span class="math notranslate nohighlight">\(r=\mathrm{min}(n,p)\)</span>
|
||
singular values <span class="math notranslate nohighlight">\(\sigma_i\geq 0\)</span> on the main diagonal and zeros filling
|
||
the rest of the matrix. There are at most <span class="math notranslate nohighlight">\(p\)</span> singular values
|
||
assuming that <span class="math notranslate nohighlight">\(n > p\)</span>. In our regression examples for the nuclear
|
||
masses and the equation of state this is indeed the case, while for
|
||
the Ising model we have <span class="math notranslate nohighlight">\(p > n\)</span>. These are often cases that lead to
|
||
near singular or singular matrices.</p>
|
||
<p>The columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> are called the left singular vectors while the columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are the right singular vectors.</p>
|
||
</div>
|
||
<div class="section" id="economy-size-svd">
|
||
<h2>Economy-size SVD<a class="headerlink" href="#economy-size-svd" title="Permalink to this headline">¶</a></h2>
|
||
<p>If we assume that <span class="math notranslate nohighlight">\(n > p\)</span>, then our matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> has dimension <span class="math notranslate nohighlight">\(n
|
||
\times n\)</span>. The last <span class="math notranslate nohighlight">\(n-p\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> become however
|
||
irrelevant in our calculations since they are multiplied with the
|
||
zeros in <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span>.</p>
|
||
<p>The economy-size decomposition removes extra rows or columns of zeros
|
||
from the diagonal matrix of singular values, <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span>, along with the columns
|
||
in either <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> or <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> that multiply those zeros in the expression.
|
||
Removing these zeros and columns can improve execution time
|
||
and reduce storage requirements without compromising the accuracy of
|
||
the decomposition.</p>
|
||
<p>If <span class="math notranslate nohighlight">\(n > p\)</span>, we keep only the first <span class="math notranslate nohighlight">\(p\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> has dimension <span class="math notranslate nohighlight">\(p\times p\)</span>.
|
||
If <span class="math notranslate nohighlight">\(p > n\)</span>, then only the first <span class="math notranslate nohighlight">\(n\)</span> columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are computed and <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> has dimension <span class="math notranslate nohighlight">\(n\times n\)</span>.
|
||
The <span class="math notranslate nohighlight">\(n=p\)</span> case is obvious, we retain the full SVD.
|
||
In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy.</p>
|
||
</div>
|
||
<div class="section" id="codes-for-the-svd">
|
||
<h2>Codes for the SVD<a class="headerlink" href="#codes-for-the-svd" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="c1"># SVD inversion</span>
|
||
<span class="k">def</span> <span class="nf">SVD</span><span class="p">(</span><span class="n">A</span><span class="p">):</span>
|
||
<span class="sd">''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).</span>
|
||
<span class="sd"> SVD is numerically more stable than the inversion algorithms provided by</span>
|
||
<span class="sd"> numpy and scipy.linalg at the cost of being slower.</span>
|
||
<span class="sd"> '''</span>
|
||
<span class="n">U</span><span class="p">,</span> <span class="n">S</span><span class="p">,</span> <span class="n">VT</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linalg</span><span class="o">.</span><span class="n">svd</span><span class="p">(</span><span class="n">A</span><span class="p">,</span><span class="n">full_matrices</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'test U'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">U</span><span class="p">)</span> <span class="o">@</span> <span class="n">U</span> <span class="o">-</span> <span class="n">U</span> <span class="nd">@np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">U</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="s1">'test VT'</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">VT</span><span class="p">)</span> <span class="o">@</span> <span class="n">VT</span> <span class="o">-</span> <span class="n">VT</span> <span class="nd">@np</span><span class="o">.</span><span class="n">transpose</span><span class="p">(</span><span class="n">VT</span><span class="p">)))</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">U</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">S</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">VT</span><span class="p">)</span>
|
||
|
||
<span class="n">D</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="n">U</span><span class="p">),</span><span class="nb">len</span><span class="p">(</span><span class="n">VT</span><span class="p">)))</span>
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="nb">len</span><span class="p">(</span><span class="n">VT</span><span class="p">)):</span>
|
||
<span class="n">D</span><span class="p">[</span><span class="n">i</span><span class="p">,</span><span class="n">i</span><span class="p">]</span><span class="o">=</span><span class="n">S</span><span class="p">[</span><span class="n">i</span><span class="p">]</span>
|
||
<span class="k">return</span> <span class="n">U</span> <span class="o">@</span> <span class="n">D</span> <span class="o">@</span> <span class="n">VT</span>
|
||
|
||
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span><span class="o">-</span><span class="mf">1.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.0</span><span class="p">,</span><span class="o">-</span><span class="mf">1.0</span><span class="p">]])</span>
|
||
<span class="c1">#X = np.array([[1, 2], [3, 4], [5, 6]])</span>
|
||
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">SVD</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="c1"># Print the difference between the original matrix and the SVD one</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="o">-</span><span class="n">X</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[ 1. -1.]
|
||
[ 1. -1.]]
|
||
test U
|
||
[[0. 0.]
|
||
[0. 0.]]
|
||
test VT
|
||
[[0. 0.]
|
||
[0. 0.]]
|
||
[[-0.70710678 -0.70710678]
|
||
[-0.70710678 0.70710678]]
|
||
[2.00000000e+00 3.35470445e-17]
|
||
[[-0.70710678 0.70710678]
|
||
[ 0.70710678 0.70710678]]
|
||
[[-3.33066907e-16 4.44089210e-16]
|
||
[ 0.00000000e+00 2.22044605e-16]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>The matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> has columns that are linearly dependent. The first
|
||
column is the row-wise sum of the other two columns. The rank of a
|
||
matrix (the column rank) is the dimension of space spanned by the
|
||
column vectors. The rank of the matrix is the number of linearly
|
||
independent columns, in this case just <span class="math notranslate nohighlight">\(2\)</span>. We see this from the
|
||
singular values when running the above code. Running the standard
|
||
inversion algorithm for matrix inversion with <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> results
|
||
in the program terminating due to a singular matrix.</p>
|
||
</div>
|
||
<div class="section" id="note-about-svd-calculations">
|
||
<h2>Note about SVD Calculations<a class="headerlink" href="#note-about-svd-calculations" title="Permalink to this headline">¶</a></h2>
|
||
<p>The <span class="math notranslate nohighlight">\(U\)</span>, <span class="math notranslate nohighlight">\(S\)</span>, and <span class="math notranslate nohighlight">\(V\)</span> matrices returned from the <strong>svd()</strong> function
|
||
cannot be multiplied directly.</p>
|
||
<p>As you can see from the code, the <span class="math notranslate nohighlight">\(S\)</span> vector must be converted into a
|
||
diagonal matrix. This may cause a problem as the size of the matrices
|
||
do not fit the rules of matrix multiplication, where the number of
|
||
columns in a matrix must match the number of rows in the subsequent
|
||
matrix.</p>
|
||
<p>If you wish to include the zero singular values, you will need to
|
||
resize the matrices and set up a diagonal matrix as done in the above
|
||
example</p>
|
||
</div>
|
||
<div class="section" id="mathematics-of-the-svd-and-implications">
|
||
<h2>Mathematics of the SVD and implications<a class="headerlink" href="#mathematics-of-the-svd-and-implications" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let us take a closer look at the mathematics of the SVD and the various implications for machine learning studies.</p>
|
||
<p>Our starting point is our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> of dimension <span class="math notranslate nohighlight">\(n\times p\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\
|
||
x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\
|
||
x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\
|
||
\dots & \dots & \dots & \dots \dots & \dots \\
|
||
x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\
|
||
x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>We can SVD decompose our matrix as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> is an orthogonal matrix of dimension <span class="math notranslate nohighlight">\(n\times n\)</span>, meaning that <span class="math notranslate nohighlight">\(\boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{U}^T\boldsymbol{U}=\boldsymbol{I}_n\)</span>. Here <span class="math notranslate nohighlight">\(\boldsymbol{I}_n\)</span> is the unit matrix of dimension <span class="math notranslate nohighlight">\(n \times n\)</span>.</p>
|
||
<p>Similarly, <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> is an orthogonal matrix of dimension <span class="math notranslate nohighlight">\(p\times p\)</span>, meaning that <span class="math notranslate nohighlight">\(\boldsymbol{V}\boldsymbol{V}^T=\boldsymbol{V}^T\boldsymbol{V}=\boldsymbol{I}_p\)</span>. Here <span class="math notranslate nohighlight">\(\boldsymbol{I}_n\)</span> is the unit matrix of dimension <span class="math notranslate nohighlight">\(p \times p\)</span>.</p>
|
||
<p>Finally <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> contains the singular values <span class="math notranslate nohighlight">\(\sigma_i\)</span>. This matrix has dimension <span class="math notranslate nohighlight">\(n\times p\)</span> and the singular values <span class="math notranslate nohighlight">\(\sigma_i\)</span> are all positive. The non-zero values are ordered in descending order, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sigma_0 > \sigma_1 > \sigma_2 > \dots > \sigma_{p-1} > 0.
|
||
\]</div>
|
||
<p>All values beyond <span class="math notranslate nohighlight">\(p-1\)</span> are all zero.</p>
|
||
</div>
|
||
<div class="section" id="example-matrix">
|
||
<h2>Example Matrix<a class="headerlink" href="#example-matrix" title="Permalink to this headline">¶</a></h2>
|
||
<p>As an example, consider the following <span class="math notranslate nohighlight">\(3\times 2\)</span> example for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}=
|
||
\begin{bmatrix}
|
||
2& 0 \\
|
||
0 & 1 \\
|
||
0 & 0 \\
|
||
\end{bmatrix}
|
||
\end{split}\]</div>
|
||
<p>The singular values are <span class="math notranslate nohighlight">\(\sigma_0=2\)</span> and <span class="math notranslate nohighlight">\(\sigma_1=1\)</span>. It is common to rewrite the matrix <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}=
|
||
\begin{bmatrix}
|
||
\boldsymbol{\tilde{\Sigma}}\\
|
||
\boldsymbol{0}\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\tilde{\Sigma}}=
|
||
\begin{bmatrix}
|
||
2& 0 \\
|
||
0 & 1 \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>contains only the singular values. Note also (and we will use this below) that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}=
|
||
\begin{bmatrix}
|
||
4& 0 \\
|
||
0 & 1 \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which is a <span class="math notranslate nohighlight">\(2\times 2 \)</span> matrix while</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T=
|
||
\begin{bmatrix}
|
||
4& 0 & 0\\
|
||
0 & 1 & 0\\
|
||
0 & 0 & 0\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>is a <span class="math notranslate nohighlight">\(3\times 3 \)</span> matrix. The last row and column of this last matrix
|
||
contain only zeros. This will have important consequences for our SVD
|
||
decomposition of the design matrix.</p>
|
||
</div>
|
||
<div class="section" id="setting-up-the-matrix-to-be-inverted">
|
||
<h2>Setting up the Matrix to be inverted<a class="headerlink" href="#setting-up-the-matrix-to-be-inverted" title="Permalink to this headline">¶</a></h2>
|
||
<p>The matrix that may cause problems for us is <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. Using the SVD we can rewrite this matrix as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T,
|
||
\]</div>
|
||
<p>and using the orthogonality of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>We define <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}=\tilde{\boldsymbol{\Sigma}}^2\)</span> which is a diagonal matrix containing only the singular values squared. It has dimensionality <span class="math notranslate nohighlight">\(p \times p\)</span>.</p>
|
||
<p>We can now insert the result for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> into our equation for ordinary least squares where</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_{\mathrm{OLS}}=\boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>and using our SVD decomposition of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\left(\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^{2}(\boldsymbol{V}^T\right)^{-1}\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>which gives us, using the orthogonality of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{y}_{\mathrm{OLS}}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_i\boldsymbol{u}^T_i\boldsymbol{y},
|
||
\]</div>
|
||
<p>It means that the ordinary least square model (with the optimal
|
||
parameters) <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}}\)</span>, corresponds to an orthogonal
|
||
transformation of the output (or target) vector <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> by the
|
||
vectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>. <strong>Note that the summation ends at</strong>
|
||
<span class="math notranslate nohighlight">\(p-1\)</span>, that is <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}}\ne \boldsymbol{y}\)</span>. We can thus not use the
|
||
orthogonality relation for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>. This can already be
|
||
when we multiply the matrices <span class="math notranslate nohighlight">\(\boldsymbol{\Sigma}^T\boldsymbol{U}^T\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="further-properties-important-for-our-analyses-later">
|
||
<h2>Further properties (important for our analyses later)<a class="headerlink" href="#further-properties-important-for-our-analyses-later" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let us study again <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> in terms of our SVD,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>If we now multiply from the right with <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> (using the orthogonality of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>) we get</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}.
|
||
\]</div>
|
||
<p>This means the vectors <span class="math notranslate nohighlight">\(\boldsymbol{v}_i\)</span> of the orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are the eigenvectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>
|
||
with eigenvalues given by the singular values squared, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2.
|
||
\]</div>
|
||
<p>Similarly, if we use the SVD decomposition for the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T\boldsymbol{U}^T.
|
||
\]</div>
|
||
<p>If we now multiply from the right with <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> (using the orthogonality of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>) we get</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{U}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T.
|
||
\]</div>
|
||
<p>This means the vectors <span class="math notranslate nohighlight">\(\boldsymbol{u}_i\)</span> of the orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> are the eigenvectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span>
|
||
with eigenvalues given by the singular values squared, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}\boldsymbol{X}^T\right)\boldsymbol{u}_i=\boldsymbol{u}_i\sigma_i^2.
|
||
\]</div>
|
||
<p><strong>Important note</strong>: we have defined our design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> to be an
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span> matrix. In most supervised learning cases we have that <span class="math notranslate nohighlight">\(n
|
||
\ge p\)</span>, and quite often we have <span class="math notranslate nohighlight">\(n >> p\)</span>. For linear algebra based methods like ordinary least squares or Ridge regression, this leads to a matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> which is small and thereby easier to handle from a computational point of view (in terms of number of floating point operations).</p>
|
||
<p>In our lectures, the number of columns will
|
||
always refer to the number of features in our data set, while the
|
||
number of rows represents the number of data inputs. Note that in
|
||
other texts you may find the opposite notation. This has consequences
|
||
for the definition of for example the covariance matrix and its relation to the SVD.</p>
|
||
</div>
|
||
<div class="section" id="meet-the-covariance-matrix">
|
||
<h2>Meet the Covariance Matrix<a class="headerlink" href="#meet-the-covariance-matrix" title="Permalink to this headline">¶</a></h2>
|
||
<p>Before we move on to a discussion of Ridge and Lasso regression, we want to show an important example of the above.</p>
|
||
<p>We have already noted that the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> in ordinary
|
||
least squares is proportional to the second derivative of the cost
|
||
function, that is we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial^2 C(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}\partial \boldsymbol{\beta}^T} =\frac{2}{n}\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>This quantity defines was what is called the Hessian matrix (the second derivative of a function we want to optimize).</p>
|
||
<p>The Hessian matrix plays an important role and is defined in this course as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{H}=\boldsymbol{X}^T\boldsymbol{X}.
|
||
\]</div>
|
||
<p>The Hessian matrix for ordinary least squares is also proportional to
|
||
the covariance matrix. This means also that we can use the SVD to find
|
||
the eigenvalues of the covariance matrix and the Hessian matrix in
|
||
terms of the singular values. Let us develop these arguments, as they will play an important role in our machine learning studies.</p>
|
||
</div>
|
||
<div class="section" id="introducing-the-covariance-and-correlation-functions">
|
||
<h2>Introducing the Covariance and Correlation functions<a class="headerlink" href="#introducing-the-covariance-and-correlation-functions" title="Permalink to this headline">¶</a></h2>
|
||
<p>Before we discuss the link between for example Ridge regression and the singular value decomposition, we need to remind ourselves about
|
||
the definition of the covariance and the correlation function. These are quantities that play a central role in machine learning methods.</p>
|
||
<p>Suppose we have defined two vectors
|
||
<span class="math notranslate nohighlight">\(\hat{x}\)</span> and <span class="math notranslate nohighlight">\(\hat{y}\)</span> with <span class="math notranslate nohighlight">\(n\)</span> elements each. The covariance matrix <span class="math notranslate nohighlight">\(\boldsymbol{C}\)</span> is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{cov}[\boldsymbol{x},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{cov}[\boldsymbol{y},\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{y},\boldsymbol{y}] \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where for example</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}).
|
||
\]</div>
|
||
<p>With this definition and recalling that the variance is defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{var}[\boldsymbol{x}]=\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})^2,
|
||
\]</div>
|
||
<p>we can rewrite the covariance matrix as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} \mathrm{var}[\boldsymbol{x}] & \mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}] & \mathrm{var}[\boldsymbol{y}] \\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p><strong>Note:</strong> we have used <span class="math notranslate nohighlight">\(1/n\)</span> in the above definitions of the <em>sample</em> variance and covariance. We assume then that we can calculate the exact mean value.
|
||
What you will find in essentially all statistics texts are equations
|
||
with a factor <span class="math notranslate nohighlight">\(1/(n-1)\)</span>. This is called <a class="reference external" href="https://mathworld.wolfram.com/BesselsCorrection.html">Bessel’s correction</a>. This
|
||
method corrects the bias in the estimation of the population variance
|
||
and covariance. It also partially corrects the bias in the estimation
|
||
of the population standard deviation. If you use a library like
|
||
<strong>Scikit-Learn</strong> or <strong>nunmpy’s</strong> function to calculate the covariance, this
|
||
quantity will be computed with a factor <span class="math notranslate nohighlight">\(1/(n-1)\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="covariance-and-correlation-matrix">
|
||
<h2>Covariance and Correlation Matrix<a class="headerlink" href="#covariance-and-correlation-matrix" title="Permalink to this headline">¶</a></h2>
|
||
<p>The covariance takes values between zero and infinity and may thus
|
||
lead to problems with loss of numerical precision for particularly
|
||
large values. It is common to scale the covariance matrix by
|
||
introducing instead the correlation matrix defined via the so-called
|
||
correlation function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]=\frac{\mathrm{cov}[\boldsymbol{x},\boldsymbol{y}]}{\sqrt{\mathrm{var}[\boldsymbol{x}] \mathrm{var}[\boldsymbol{y}]}}.
|
||
\]</div>
|
||
<p>The correlation function is then given by values <span class="math notranslate nohighlight">\(\mathrm{corr}[\boldsymbol{x},\boldsymbol{y}]
|
||
\in [-1,1]\)</span>. This avoids eventual problems with too large values. We
|
||
can then define the correlation matrix for the two vectors <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span>
|
||
and <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{K}[\boldsymbol{x},\boldsymbol{y}] = \begin{bmatrix} 1 & \mathrm{corr}[\boldsymbol{x},\boldsymbol{y}] \\
|
||
\mathrm{corr}[\boldsymbol{y},\boldsymbol{x}] & 1 \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>In the above example this is the function we constructed using <strong>pandas</strong>.</p>
|
||
</div>
|
||
<div class="section" id="correlation-function-and-design-feature-matrix">
|
||
<h2>Correlation Function and Design/Feature Matrix<a class="headerlink" href="#correlation-function-and-design-feature-matrix" title="Permalink to this headline">¶</a></h2>
|
||
<p>In our derivation of the various regression algorithms like <strong>Ordinary Least Squares</strong> or <strong>Ridge regression</strong>
|
||
we defined the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{0,0} & x_{0,1} & x_{0,2}& \dots & \dots x_{0,p-1}\\
|
||
x_{1,0} & x_{1,1} & x_{1,2}& \dots & \dots x_{1,p-1}\\
|
||
x_{2,0} & x_{2,1} & x_{2,2}& \dots & \dots x_{2,p-1}\\
|
||
\dots & \dots & \dots & \dots \dots & \dots \\
|
||
x_{n-2,0} & x_{n-2,1} & x_{n-2,2}& \dots & \dots x_{n-2,p-1}\\
|
||
x_{n-1,0} & x_{n-1,1} & x_{n-1,2}& \dots & \dots x_{n-1,p-1}\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>, with the predictors/features <span class="math notranslate nohighlight">\(p\)</span> refering to the column numbers and the
|
||
entries <span class="math notranslate nohighlight">\(n\)</span> being the row elements.
|
||
We can rewrite the design/feature matrix in terms of its column vectors as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}=\begin{bmatrix} \boldsymbol{x}_0 & \boldsymbol{x}_1 & \boldsymbol{x}_2 & \dots & \dots & \boldsymbol{x}_{p-1}\end{bmatrix},
|
||
\]</div>
|
||
<p>with a given vector</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{x}_i^T = \begin{bmatrix}x_{0,i} & x_{1,i} & x_{2,i}& \dots & \dots x_{n-1,i}\end{bmatrix}.
|
||
\]</div>
|
||
<p>With these definitions, we can now rewrite our <span class="math notranslate nohighlight">\(2\times 2\)</span>
|
||
correlation/covariance matrix in terms of a moe general design/feature
|
||
matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>. This leads to a <span class="math notranslate nohighlight">\(p\times p\)</span>
|
||
covariance matrix for the vectors <span class="math notranslate nohighlight">\(\boldsymbol{x}_i\)</span> with <span class="math notranslate nohighlight">\(i=0,1,\dots,p-1\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x}] = \begin{bmatrix}
|
||
\mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_1] & \mathrm{var}[\boldsymbol{x}_2] & \dots & \dots & \mathrm{cov}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{cov}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & \mathrm{var}[\boldsymbol{x}_{p-1}]\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>and the correlation matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{K}[\boldsymbol{x}] = \begin{bmatrix}
|
||
1 & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_0,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_0] & 1 & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_2] & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_1,\boldsymbol{x}_{p-1}]\\
|
||
\mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_1] & 1 & \dots & \dots & \mathrm{corr}[\boldsymbol{x}_2,\boldsymbol{x}_{p-1}]\\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\dots & \dots & \dots & \dots & \dots & \dots \\
|
||
\mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_0] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_1] & \mathrm{corr}[\boldsymbol{x}_{p-1},\boldsymbol{x}_{2}] & \dots & \dots & 1\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
</div>
|
||
<div class="section" id="covariance-matrix-examples">
|
||
<h2>Covariance Matrix Examples<a class="headerlink" href="#covariance-matrix-examples" title="Permalink to this headline">¶</a></h2>
|
||
<p>The Numpy function <strong>np.cov</strong> calculates the covariance elements using
|
||
the factor <span class="math notranslate nohighlight">\(1/(n-1)\)</span> instead of <span class="math notranslate nohighlight">\(1/n\)</span> since it assumes we do not have
|
||
the exact mean values. The following simple function uses the
|
||
<strong>np.vstack</strong> function which takes each vector of dimension <span class="math notranslate nohighlight">\(1\times n\)</span>
|
||
and produces a <span class="math notranslate nohighlight">\(2\times n\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{W}\)</span></p>
|
||
<p>Note that this assumes you have the features as the rows, and the inputs as columns, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{W} = \begin{bmatrix} x_0 & x_1 & x_2 & \dots & x_{n-2} & x_{n-1} \\
|
||
y_0 & y_1 & y_2 & \dots & y_{n-2} & y_{n-1} \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which in turn is converted into into the <span class="math notranslate nohighlight">\(2\times 2\)</span> covariance matrix
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{C}\)</span> via the Numpy function <strong>np.cov()</strong>. We note that we can also calculate
|
||
the mean value of each set of samples <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> etc using the Numpy
|
||
function <strong>np.mean(x)</strong>. We can also extract the eigenvalues of the
|
||
covariance matrix through the <strong>np.linalg.eig()</strong> function.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Importing various packages</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">))</span>
|
||
<span class="n">W</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">((</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">))</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">cov</span><span class="p">(</span><span class="n">W</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.10790125813226321
|
||
4.340071371496255
|
||
[[ 1.04193203 3.08165104]
|
||
[ 3.08165104 10.18383522]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="section" id="correlation-matrix">
|
||
<h2>Correlation Matrix<a class="headerlink" href="#correlation-matrix" title="Permalink to this headline">¶</a></h2>
|
||
<p>The previous example can be converted into the correlation matrix by
|
||
simply scaling the matrix elements with the variances. We should also
|
||
subtract the mean values for each column. This leads to the following
|
||
code which sets up the correlations matrix for the previous example in
|
||
a more brute force way. Here we scale the mean values for each column of the design matrix, calculate the relevant mean values and variances and then finally set up the <span class="math notranslate nohighlight">\(2\times 2\)</span> correlation matrix (since we have only two vectors).</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="c1"># define two vectors </span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">random</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="c1">#scaling the x and y vectors </span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">y</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="n">variance_x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@x</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">variance_y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">y</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">variance_x</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">variance_y</span><span class="p">)</span>
|
||
<span class="n">cov_xy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">cov_xx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">x</span><span class="nd">@x</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">cov_yy</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">y</span><span class="nd">@y</span><span class="p">)</span><span class="o">/</span><span class="n">n</span>
|
||
<span class="n">C</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_xx</span><span class="o">/</span><span class="n">variance_x</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_yy</span><span class="o">/</span><span class="n">variance_y</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span><span class="o">=</span> <span class="n">cov_xy</span><span class="o">/</span><span class="n">np</span><span class="o">.</span><span class="n">sqrt</span><span class="p">(</span><span class="n">variance_y</span><span class="o">*</span><span class="n">variance_x</span><span class="p">)</span>
|
||
<span class="n">C</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">0</span><span class="p">]</span><span class="o">=</span> <span class="n">C</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">]</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">C</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>0.08881497884574564
|
||
1.7086067479626619
|
||
[[1. 0.66080313]
|
||
[0.66080313 1. ]]
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We see that the matrix elements along the diagonal are one as they
|
||
should be and that the matrix is symmetric. Furthermore, diagonalizing
|
||
this matrix we easily see that it is a positive definite matrix.</p>
|
||
<p>The above procedure with <strong>numpy</strong> can be made more compact if we use <strong>pandas</strong>.</p>
|
||
</div>
|
||
<div class="section" id="correlation-matrix-with-pandas">
|
||
<h2>Correlation Matrix with Pandas<a class="headerlink" href="#correlation-matrix-with-pandas" title="Permalink to this headline">¶</a></h2>
|
||
<p>We whow here how we can set up the correlation matrix using <strong>pandas</strong>, as done in this simple code</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">10</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">x</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="mi">4</span><span class="o">+</span><span class="mi">3</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">y</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
<span class="c1"># Note that we transpose the matrix in order to stay with our ordering n x p</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">((</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)))</span><span class="o">.</span><span class="n">T</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="n">Xpd</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">Xpd</span><span class="p">)</span>
|
||
<span class="n">correlation_matrix</span> <span class="o">=</span> <span class="n">Xpd</span><span class="o">.</span><span class="n">corr</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">correlation_matrix</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>[[-0.40620066 -2.01265755]
|
||
[ 0.01458611 0.37737221]
|
||
[-1.0895387 -3.65442354]
|
||
[ 0.2338675 1.12044974]
|
||
[ 0.4676059 1.54393936]
|
||
[-0.65891389 -3.16304863]
|
||
[-0.1715252 0.39197698]
|
||
[ 0.71142161 2.95511792]
|
||
[ 0.39214397 0.13069442]
|
||
[ 0.50655336 2.3105791 ]]
|
||
0 1
|
||
0 -0.406201 -2.012658
|
||
1 0.014586 0.377372
|
||
2 -1.089539 -3.654424
|
||
3 0.233868 1.120450
|
||
4 0.467606 1.543939
|
||
5 -0.658914 -3.163049
|
||
6 -0.171525 0.391977
|
||
7 0.711422 2.955118
|
||
8 0.392144 0.130694
|
||
9 0.506553 2.310579
|
||
0 1
|
||
0 1.000000 0.952387
|
||
1 0.952387 1.000000
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We expand this model to the Franke function discussed above.</p>
|
||
</div>
|
||
<div class="section" id="correlation-matrix-with-pandas-and-the-franke-function">
|
||
<h2>Correlation Matrix with Pandas and the Franke function<a class="headerlink" href="#correlation-matrix-with-pandas-and-the-franke-function" title="Permalink to this headline">¶</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># Common imports</span>
|
||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">FrankeFunction</span><span class="p">(</span><span class="n">x</span><span class="p">,</span><span class="n">y</span><span class="p">):</span>
|
||
<span class="n">term1</span> <span class="o">=</span> <span class="mf">0.75</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mf">0.25</span><span class="o">*</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">-</span> <span class="mf">0.25</span><span class="o">*</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">term2</span> <span class="o">=</span> <span class="mf">0.75</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="mf">49.0</span> <span class="o">-</span> <span class="mf">0.1</span><span class="o">*</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">+</span><span class="mi">1</span><span class="p">))</span>
|
||
<span class="n">term3</span> <span class="o">=</span> <span class="mf">0.5</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">7</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="o">/</span><span class="mf">4.0</span> <span class="o">-</span> <span class="mf">0.25</span><span class="o">*</span><span class="p">((</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">3</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">))</span>
|
||
<span class="n">term4</span> <span class="o">=</span> <span class="o">-</span><span class="mf">0.2</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">x</span><span class="o">-</span><span class="mi">4</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span> <span class="o">-</span> <span class="p">(</span><span class="mi">9</span><span class="o">*</span><span class="n">y</span><span class="o">-</span><span class="mi">7</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">return</span> <span class="n">term1</span> <span class="o">+</span> <span class="n">term2</span> <span class="o">+</span> <span class="n">term3</span> <span class="o">+</span> <span class="n">term4</span>
|
||
|
||
|
||
<span class="k">def</span> <span class="nf">create_X</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n</span> <span class="p">):</span>
|
||
<span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span> <span class="o">></span> <span class="mi">1</span><span class="p">:</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ravel</span><span class="p">(</span><span class="n">y</span><span class="p">)</span>
|
||
|
||
<span class="n">N</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||
<span class="n">l</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">n</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">n</span><span class="o">+</span><span class="mi">2</span><span class="p">)</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span> <span class="c1"># Number of elements in beta</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="n">N</span><span class="p">,</span><span class="n">l</span><span class="p">))</span>
|
||
|
||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span><span class="n">n</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">q</span> <span class="o">=</span> <span class="nb">int</span><span class="p">((</span><span class="n">i</span><span class="p">)</span><span class="o">*</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">)</span><span class="o">/</span><span class="mi">2</span><span class="p">)</span>
|
||
<span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
|
||
<span class="n">X</span><span class="p">[:,</span><span class="n">q</span><span class="o">+</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">x</span><span class="o">**</span><span class="p">(</span><span class="n">i</span><span class="o">-</span><span class="n">k</span><span class="p">))</span><span class="o">*</span><span class="p">(</span><span class="n">y</span><span class="o">**</span><span class="n">k</span><span class="p">)</span>
|
||
|
||
<span class="k">return</span> <span class="n">X</span>
|
||
|
||
|
||
<span class="c1"># Making meshgrid of datapoints and compute Franke's function</span>
|
||
<span class="n">n</span> <span class="o">=</span> <span class="mi">4</span>
|
||
<span class="n">N</span> <span class="o">=</span> <span class="mi">100</span>
|
||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">N</span><span class="p">))</span>
|
||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sort</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="n">N</span><span class="p">))</span>
|
||
<span class="n">z</span> <span class="o">=</span> <span class="n">FrankeFunction</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
|
||
<span class="n">X</span> <span class="o">=</span> <span class="n">create_X</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">n</span><span class="o">=</span><span class="n">n</span><span class="p">)</span>
|
||
|
||
<span class="n">Xpd</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
|
||
<span class="c1"># subtract the mean values and set up the covariance matrix</span>
|
||
<span class="n">Xpd</span> <span class="o">=</span> <span class="n">Xpd</span> <span class="o">-</span> <span class="n">Xpd</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||
<span class="n">covariance_matrix</span> <span class="o">=</span> <span class="n">Xpd</span><span class="o">.</span><span class="n">cov</span><span class="p">()</span>
|
||
<span class="nb">print</span><span class="p">(</span><span class="n">covariance_matrix</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="cell_output docutils container">
|
||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 0 1 2 3 4 5 6 7 \
|
||
0 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
|
||
1 0.0 0.078974 0.081276 0.075889 0.077168 0.078540 0.065410 0.066474
|
||
2 0.0 0.081276 0.084076 0.078336 0.079946 0.081655 0.067707 0.069028
|
||
3 0.0 0.075889 0.078336 0.077460 0.078986 0.080616 0.069452 0.070737
|
||
4 0.0 0.077168 0.079946 0.078986 0.080764 0.082653 0.070986 0.072476
|
||
5 0.0 0.078540 0.081655 0.080616 0.082653 0.084809 0.072621 0.074323
|
||
6 0.0 0.065410 0.067707 0.069452 0.070986 0.072621 0.064074 0.065378
|
||
7 0.0 0.066474 0.069028 0.070737 0.072476 0.074323 0.065378 0.066854
|
||
8 0.0 0.067637 0.070457 0.072132 0.074084 0.076150 0.066787 0.068441
|
||
9 0.0 0.068906 0.072000 0.073644 0.075816 0.078110 0.068307 0.070146
|
||
10 0.0 0.055734 0.057835 0.060872 0.062337 0.063894 0.057393 0.058645
|
||
11 0.0 0.056683 0.058996 0.062016 0.063653 0.065390 0.058552 0.059951
|
||
12 0.0 0.057722 0.060254 0.063260 0.065077 0.066999 0.059807 0.061359
|
||
13 0.0 0.058854 0.061614 0.064609 0.066612 0.068727 0.061163 0.062874
|
||
14 0.0 0.060083 0.063080 0.066066 0.068264 0.070582 0.062624 0.064501
|
||
|
||
8 9 10 11 12 13 14
|
||
0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
|
||
1 0.067637 0.068906 0.055734 0.056683 0.057722 0.058854 0.060083
|
||
2 0.070457 0.072000 0.057835 0.058996 0.060254 0.061614 0.063080
|
||
3 0.072132 0.073644 0.060872 0.062016 0.063260 0.064609 0.066066
|
||
4 0.074084 0.075816 0.062337 0.063653 0.065077 0.066612 0.068264
|
||
5 0.076150 0.078110 0.063894 0.065390 0.066999 0.068727 0.070582
|
||
6 0.066787 0.068307 0.057393 0.058552 0.059807 0.061163 0.062624
|
||
7 0.068441 0.070146 0.058645 0.059951 0.061359 0.062874 0.064501
|
||
8 0.070213 0.072111 0.059993 0.061452 0.063019 0.064699 0.066500
|
||
9 0.072111 0.074210 0.061443 0.063061 0.064793 0.066647 0.068629
|
||
10 0.059993 0.061443 0.052305 0.053417 0.054617 0.055910 0.057300
|
||
11 0.061452 0.063061 0.053417 0.054655 0.055987 0.057418 0.058952
|
||
12 0.063019 0.064793 0.054617 0.055987 0.057457 0.059031 0.060716
|
||
13 0.064699 0.066647 0.055910 0.057418 0.059031 0.060756 0.062599
|
||
14 0.066500 0.068629 0.057300 0.058952 0.060716 0.062599 0.064606
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We note here that the covariance is zero for the first rows and
|
||
columns since all matrix elements in the design matrix were set to one
|
||
(we are fitting the function in terms of a polynomial of degree <span class="math notranslate nohighlight">\(n\)</span>).</p>
|
||
<p>This means that the variance for these elements will be zero and will
|
||
cause problems when we set up the correlation matrix. We can simply
|
||
drop these elements and construct a correlation
|
||
matrix without these elements.</p>
|
||
</div>
|
||
<div class="section" id="rewriting-the-covariance-and-or-correlation-matrix">
|
||
<h2>Rewriting the Covariance and/or Correlation Matrix<a class="headerlink" href="#rewriting-the-covariance-and-or-correlation-matrix" title="Permalink to this headline">¶</a></h2>
|
||
<p>We can rewrite the covariance matrix in a more compact form in terms of the design/feature matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{C}[\boldsymbol{x}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}= \mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}].
|
||
\]</div>
|
||
<p>To see this let us simply look at a design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{2\times 2}\)</span></p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}=\begin{bmatrix}
|
||
x_{00} & x_{01}\\
|
||
x_{10} & x_{11}\\
|
||
\end{bmatrix}=\begin{bmatrix}
|
||
\boldsymbol{x}_{0} & \boldsymbol{x}_{1}\\
|
||
\end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>If we then compute the expectation value (note the <span class="math notranslate nohighlight">\(1/n\)</span> factor instead of <span class="math notranslate nohighlight">\(1/(n-1)\)</span>)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\mathbb{E}[\boldsymbol{X}^T\boldsymbol{X}] = \frac{1}{n}\boldsymbol{X}^T\boldsymbol{X}=\frac{1}{n}\begin{bmatrix}
|
||
x_{00}^2+x_{10}^2 & x_{00}x_{01}+x_{10}x_{11}\\
|
||
x_{01}x_{00}+x_{11}x_{10} & x_{01}^2+x_{11}^2\\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>which is just</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]=\begin{bmatrix} \mathrm{var}[\boldsymbol{x}_0] & \mathrm{cov}[\boldsymbol{x}_0,\boldsymbol{x}_1] \\
|
||
\mathrm{cov}[\boldsymbol{x}_1,\boldsymbol{x}_0] & \mathrm{var}[\boldsymbol{x}_1] \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where we wrote $<span class="math notranslate nohighlight">\(\boldsymbol{C}[\boldsymbol{x}_0,\boldsymbol{x}_1] = \boldsymbol{C}[\boldsymbol{x}]\)</span><span class="math notranslate nohighlight">\( to indicate that this is the covariance of the vectors \)</span>\boldsymbol{x}<span class="math notranslate nohighlight">\( of the design/feature matrix \)</span>\boldsymbol{X}$.</p>
|
||
<p>It is easy to generalize this to a matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in {\mathbb{R}}^{n\times p}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="linking-with-the-svd">
|
||
<h2>Linking with the SVD<a class="headerlink" href="#linking-with-the-svd" title="Permalink to this headline">¶</a></h2>
|
||
<p>We saw earlier that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T=\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>Since the matrices here have dimension <span class="math notranslate nohighlight">\(p\times p\)</span>, with <span class="math notranslate nohighlight">\(p\)</span> corresponding to the singular values, we defined earlier the matrix</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}^T\boldsymbol{\Sigma} = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>where the tilde-matrix <span class="math notranslate nohighlight">\(\tilde{\boldsymbol{\Sigma}}\)</span> is a matrix of dimension <span class="math notranslate nohighlight">\(p\times p\)</span> containing only the singular values <span class="math notranslate nohighlight">\(\sigma_i\)</span>, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\tilde{\boldsymbol{\Sigma}}=\begin{bmatrix} \sigma_0 & 0 & 0 & \dots & 0 & 0 \\
|
||
0 & \sigma_1 & 0 & \dots & 0 & 0 \\
|
||
0 & 0 & \sigma_2 & \dots & 0 & 0 \\
|
||
0 & 0 & 0 & \dots & \sigma_{p-2} & 0 \\
|
||
0 & 0 & 0 & \dots & 0 & \sigma_{p-1} \\
|
||
\end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>meaning we can write</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2\boldsymbol{V}^T.
|
||
\]</div>
|
||
<p>Multiplying from the right with <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> (using the orthogonality of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>) we get</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{V}=\boldsymbol{V}\tilde{\boldsymbol{\Sigma}}^2.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="what-does-it-mean">
|
||
<h2>What does it mean?<a class="headerlink" href="#what-does-it-mean" title="Permalink to this headline">¶</a></h2>
|
||
<p>This means the vectors <span class="math notranslate nohighlight">\(\boldsymbol{v}_i\)</span> of the orthogonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>
|
||
are the eigenvectors of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span> with eigenvalues
|
||
given by the singular values squared, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\left(\boldsymbol{X}^T\boldsymbol{X}\right)\boldsymbol{v}_i=\boldsymbol{v}_i\sigma_i^2.
|
||
\]</div>
|
||
<p>In other words, each non-zero singular value of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is a positive
|
||
square root of an eigenvalue of <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. It means also that
|
||
the columns of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span> are the eigenvectors of
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. Since we have ordered the singular values of
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> in a descending order, it means that the column vectors
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{v}_i\)</span> are hierarchically ordered by how much correlation they
|
||
encode from the columns of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
<p>Note that these are also the eigenvectors and eigenvalues of the
|
||
Hessian matrix. Note also that the Hessian matrix we are discussing here is from a cost function defined by the mean squared error only.</p>
|
||
<p>If we now recall the definition of the covariance matrix (not using
|
||
Bessel’s correction) we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{C}[\boldsymbol{X}]=\frac{1}{n}\boldsymbol{X}^T\boldsymbol{X},
|
||
\]</div>
|
||
<p>meaning that every squared non-singular value of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> divided by <span class="math notranslate nohighlight">\(n\)</span> (
|
||
the number of samples) are the eigenvalues of the covariance
|
||
matrix. Every singular value of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is thus a positive square
|
||
root of an eigenvalue of <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. If the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is
|
||
self-adjoint, the singular values of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> are equal to the
|
||
absolute value of the eigenvalues of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="and-finally-boldsymbol-x-boldsymbol-x-t">
|
||
<h2>And finally <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span><a class="headerlink" href="#and-finally-boldsymbol-x-boldsymbol-x-t" title="Permalink to this headline">¶</a></h2>
|
||
<p>For <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span> we found</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T\boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T=\boldsymbol{U}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{U}^T.
|
||
\]</div>
|
||
<p>Since the matrices here have dimension <span class="math notranslate nohighlight">\(n\times n\)</span>, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{\Sigma}\boldsymbol{\Sigma}^T = \begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \\ \boldsymbol{0}\\ \end{bmatrix}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} \boldsymbol{0}\\ \end{bmatrix}=\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix},
|
||
\end{split}\]</div>
|
||
<p>leading to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}\boldsymbol{U}^T.
|
||
\end{split}\]</div>
|
||
<p>Multiplying with <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> from the right gives us the eigenvalue problem</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U}=\boldsymbol{U}\begin{bmatrix} \tilde{\boldsymbol{\Sigma}} & \boldsymbol{0} \\ \boldsymbol{0} & \boldsymbol{0}\\ \end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>It means that the eigenvalues of <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span> are again given by
|
||
the non-zero singular values plus now a series of zeros. The column
|
||
vectors of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> are the eigenvectors of <span class="math notranslate nohighlight">\(\boldsymbol{X}\boldsymbol{X}^T\)</span> and
|
||
measure how much correlations are contained in the rows of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
<p>Since we will mainly be interested in the correlations among the features
|
||
of our data (the columns of <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>, the quantity of interest for us are the non-zero singular
|
||
values and the column vectors of <span class="math notranslate nohighlight">\(\boldsymbol{V}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="ridge-and-lasso-regression">
|
||
<h2>Ridge and LASSO Regression<a class="headerlink" href="#ridge-and-lasso-regression" title="Permalink to this headline">¶</a></h2>
|
||
<p>Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
|
||
our optimization problem is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
|
||
\]</div>
|
||
<p>or we can state it as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2,
|
||
\]</div>
|
||
<p>where we have used the definition of a norm-2 vector, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
|
||
\]</div>
|
||
<p>By minimizing the above equation with respect to the parameters
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> we could then obtain an analytical expression for the
|
||
parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>. We can add a regularization parameter <span class="math notranslate nohighlight">\(\lambda\)</span> by
|
||
defining a new cost function to be optimized, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2
|
||
\]</div>
|
||
<p>which leads to the Ridge regression minimization problem where we
|
||
require that <span class="math notranslate nohighlight">\(\vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t\)</span>, where <span class="math notranslate nohighlight">\(t\)</span> is
|
||
a finite number larger than zero. By defining</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1,
|
||
\]</div>
|
||
<p>we have a new optimization equation</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
{\displaystyle \min_{\boldsymbol{\beta}\in
|
||
{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1
|
||
\]</div>
|
||
<p>which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator.</p>
|
||
<p>Here we have defined the norm-1 as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
|
||
\]</div>
|
||
</div>
|
||
<div class="section" id="deriving-the-ridge-regression-equations">
|
||
<h2>Deriving the Ridge Regression Equations<a class="headerlink" href="#deriving-the-ridge-regression-equations" title="Permalink to this headline">¶</a></h2>
|
||
<p>Using the matrix-vector expression for Ridge regression and dropping the parameter <span class="math notranslate nohighlight">\(1/n\)</span> in front of the standard means squared error equation, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta},
|
||
\]</div>
|
||
<p>and
|
||
taking the derivatives with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> we obtain then
|
||
a slightly modified matrix inversion problem which for finite values
|
||
of <span class="math notranslate nohighlight">\(\lambda\)</span> does not suffer from singularity problems. We obtain
|
||
the optimal parameters</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(\boldsymbol{I}\)</span> being a <span class="math notranslate nohighlight">\(p\times p\)</span> identity matrix with the constraint that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sum_{i=0}^{p-1} \beta_i^2 \leq t,
|
||
\]</div>
|
||
<p>with <span class="math notranslate nohighlight">\(t\)</span> a finite positive number.</p>
|
||
<p>If we keep the <span class="math notranslate nohighlight">\(1/n\)</span> factor, the equation for the optimal <span class="math notranslate nohighlight">\(\beta\)</span> changes to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+n\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>In many textbooks the <span class="math notranslate nohighlight">\(1/n\)</span> term is often omitted. Note that a library like <strong>Scikit-Learn</strong> does not include the <span class="math notranslate nohighlight">\(1/n\)</span> factor in the setup of the cost function.</p>
|
||
<p>When we compare this with the ordinary least squares result we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\hat{\boldsymbol{\beta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>.</p>
|
||
<p>We see that Ridge regression is nothing but the standard OLS with a
|
||
modified diagonal term added to <span class="math notranslate nohighlight">\(\boldsymbol{X}^T\boldsymbol{X}\)</span>. The consequences, in
|
||
particular for our discussion of the bias-variance tradeoff are rather
|
||
interesting. We will see that for specific values of <span class="math notranslate nohighlight">\(\lambda\)</span>, we may
|
||
even reduce the variance of the optimal parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>. These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.</p>
|
||
<p>Using our insights about the SVD of the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>
|
||
We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}_{\mathrm{OLS}}=\boldsymbol{X}\boldsymbol{\beta} =\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>For Ridge regression this becomes</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\tilde{\boldsymbol{y}}_{\mathrm{Ridge}}=\boldsymbol{X}\boldsymbol{\beta}_{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{\Sigma}^2\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y},
|
||
\]</div>
|
||
<p>with the vectors <span class="math notranslate nohighlight">\(\boldsymbol{u}_j\)</span> being the columns of <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span> from the SVD of the matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>.</p>
|
||
</div>
|
||
<div class="section" id="interpreting-the-ridge-results">
|
||
<h2>Interpreting the Ridge results<a class="headerlink" href="#interpreting-the-ridge-results" title="Permalink to this headline">¶</a></h2>
|
||
<p>Since <span class="math notranslate nohighlight">\(\lambda \geq 0\)</span>, it means that compared to OLS, we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1.
|
||
\]</div>
|
||
<p>Ridge regression finds the coordinates of <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> with respect to the
|
||
orthonormal basis <span class="math notranslate nohighlight">\(\boldsymbol{U}\)</span>, it then shrinks the coordinates by
|
||
<span class="math notranslate nohighlight">\(\frac{\sigma_j^2}{\sigma_j^2+\lambda}\)</span>. Recall that the SVD has
|
||
eigenvalues ordered in a descending way, that is <span class="math notranslate nohighlight">\(\sigma_i \geq
|
||
\sigma_{i+1}\)</span>.</p>
|
||
<p>For small eigenvalues <span class="math notranslate nohighlight">\(\sigma_i\)</span> it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.</p>
|
||
</div>
|
||
<div class="section" id="more-interpretations">
|
||
<h2>More interpretations<a class="headerlink" href="#more-interpretations" title="Permalink to this headline">¶</a></h2>
|
||
<p>For the sake of simplicity, let us assume that the design matrix is orthonormal, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}.
|
||
\]</div>
|
||
<p>In this case the standard OLS results in</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{n-1}\boldsymbol{u}_i\boldsymbol{u}_i^T\boldsymbol{y},
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}},
|
||
\]</div>
|
||
<p>that is the Ridge estimator scales the OLS estimator by the inverse of a factor <span class="math notranslate nohighlight">\(1+\lambda\)</span>, and
|
||
the Ridge estimator converges to zero when the hyperparameter goes to
|
||
infinity.</p>
|
||
<p>We will come back to more interpreations after we have gone through some of the statistical analysis part.</p>
|
||
<p>For more discussions of Ridge and Lasso regression, <a class="reference external" href="https://arxiv.org/abs/1509.09169">Wessel van Wieringen’s</a> article is highly recommended.
|
||
Similarly, <a class="reference external" href="https://arxiv.org/abs/1803.08823">Mehta et al’s article</a> is also recommended.</p>
|
||
</div>
|
||
<div class="section" id="deriving-the-lasso-regression-equations">
|
||
<h2>Deriving the Lasso Regression Equations<a class="headerlink" href="#deriving-the-lasso-regression-equations" title="Permalink to this headline">¶</a></h2>
|
||
<p>Using the matrix-vector expression for Lasso regression, we have the following <strong>cost</strong> function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_1,
|
||
\]</div>
|
||
<p>Taking the derivative with respect to <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span> and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\frac{d \vert \beta\vert}{d \beta}=\mathrm{sgn}(\beta)=\left\{\begin{array}{cc} 1 & \beta > 0 \\-1 & \beta < 0, \end{array}\right.
|
||
\end{split}\]</div>
|
||
<p>we have that the derivative of the cost function is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial C(\boldsymbol{X},\boldsymbol{\beta})}{\partial \boldsymbol{\beta}}=-\frac{2}{n}\boldsymbol{X}^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})+\lambda sgn(\boldsymbol{\beta})=0,
|
||
\]</div>
|
||
<p>and reordering we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta}+\frac{n}{2}\lambda sgn(\boldsymbol{\beta})=2\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>We can redefine <span class="math notranslate nohighlight">\(\lambda\)</span> to absorb the constant <span class="math notranslate nohighlight">\(n/2\)</span> and we rewrite the last equation as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta}+\lambda sgn(\boldsymbol{\beta})=2\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package <a class="reference external" href="https://cvxopt.org/">CVXOPT</a>. We will discuss this later.</p>
|
||
</div>
|
||
</div>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
kernelName: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Previous / next buttons -->
|
||
<div class='prev-next-area'>
|
||
<a class='left-prev' id="prev-link" href="exercisesweek35.html" title="previous page">
|
||
<i class="fas fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title">Exercises week 35</p>
|
||
</div>
|
||
</a>
|
||
<a class='right-next' id="next-link" href="exercisesweek36.html" title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title">Exercises week 36</p>
|
||
</div>
|
||
<i class="fas fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
<footer class="footer">
|
||
<p>
|
||
|
||
By Morten Hjorth-Jensen<br/>
|
||
|
||
© Copyright 2021.<br/>
|
||
</p>
|
||
</footer>
|
||
</main>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
|
||
<script src="_static/js/index.be7d3bbb2ef33a8344ce.js"></script>
|
||
|
||
</body>
|
||
</html> |