Morten Hjorth-Jensen
1087 lines
52 KiB
HTML
1087 lines
52 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>6. Logistic Regression — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'chapter4';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="7. Optimization, the central part of any Machine Learning algortithm" href="chapteroptimization.html" />
|
||
<link rel="prev" title="5. Resampling Methods" href="chapter3.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Applied Data Analysis and Machine Learning - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Course setting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter7.html">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Weekly Exercises 3</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/chapter4.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Logistic Regression</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id1">6.1. Logistic Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#basics">6.2. Basics</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-logistic-function">6.3. The logistic function</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#examples-of-likelihood-functions-used-in-logistic-regression-and-neural-networks">6.4. Examples of likelihood functions used in logistic regression and neural networks</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#wisconsin-cancer-data">6.5. Wisconsin Cancer Data</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<!-- HTML file automatically generated from DocOnce source (https://github.com/doconce/doconce/)
|
||
doconce format html chapter4.do.txt --><section class="tex2jax_ignore mathjax_ignore" id="logistic-regression">
|
||
<h1><span class="section-number">6. </span>Logistic Regression<a class="headerlink" href="#logistic-regression" title="Link to this heading">#</a></h1>
|
||
<section id="id1">
|
||
<h2><span class="section-number">6.1. </span>Logistic Regression<a class="headerlink" href="#id1" title="Link to this heading">#</a></h2>
|
||
<p>In linear regression our main interest was centered on learning the
|
||
coefficients of a functional fit (say a polynomial) in order to be
|
||
able to predict the response of a continuous variable on some unseen
|
||
data. The fit to the continuous variable <span class="math notranslate nohighlight">\(y_i\)</span> is based on some
|
||
independent variables <span class="math notranslate nohighlight">\(x_i\)</span>. Linear regression resulted in
|
||
analytical expressions for standard ordinary Least Squares or Ridge
|
||
regression (in terms of matrices to invert) for several quantities,
|
||
ranging from the variance and thereby the confidence intervals of the
|
||
optimal parameters <span class="math notranslate nohighlight">\(\hat{\theta}\)</span> to the mean squared error. If we can invert
|
||
the product of the design matrices, linear regression gives then a
|
||
simple recipe for fitting our data.</p>
|
||
<p>Classification problems, however, are concerned with outcomes taking
|
||
the form of discrete variables (i.e. categories). We may for example,
|
||
on the basis of DNA sequencing for a number of patients, like to find
|
||
out which mutations are important for a certain disease; or based on
|
||
scans of various patients’ brains, figure out if there is a tumor or
|
||
not; or given a specific physical system, we’d like to identify its
|
||
state, say whether it is an ordered or disordered system (typical
|
||
situation in solid state physics); or classify the status of a
|
||
patient, whether she/he has a stroke or not and many other similar
|
||
situations.</p>
|
||
<p>The most common situation we encounter when we apply logistic
|
||
regression is that of two possible outcomes, normally denoted as a
|
||
binary outcome, true or false, positive or negative, success or
|
||
failure etc.</p>
|
||
<p>Logistic regression will also serve as our stepping stone towards
|
||
neural network algorithms and supervised deep learning. For logistic
|
||
learning, the minimization of the cost function leads to a non-linear
|
||
equation in the parameters <span class="math notranslate nohighlight">\(\hat{\theta}\)</span>. The optimization of the
|
||
problem calls therefore for minimization algorithms. This forms the
|
||
bottle neck of all machine learning algorithms, namely how to find
|
||
reliable minima of a multi-variable function. This leads us to the
|
||
family of gradient descent methods. The latter are the working horses
|
||
of basically all modern machine learning algorithms.</p>
|
||
<p>We note also that many of the topics discussed here on logistic
|
||
regression are also commonly used in modern supervised Deep Learning
|
||
models, as we will see later.</p>
|
||
</section>
|
||
<section id="basics">
|
||
<h2><span class="section-number">6.2. </span>Basics<a class="headerlink" href="#basics" title="Link to this heading">#</a></h2>
|
||
<p>We consider the case where the dependent variables, also called the
|
||
responses or the outcomes, <span class="math notranslate nohighlight">\(y_i\)</span> are discrete and only take values
|
||
from <span class="math notranslate nohighlight">\(k=0,\dots,K-1\)</span> (i.e. <span class="math notranslate nohighlight">\(K\)</span> classes).</p>
|
||
<p>The goal is to predict the
|
||
output classes from the design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\in\mathbb{R}^{n\times p}\)</span>
|
||
made of <span class="math notranslate nohighlight">\(n\)</span> samples, each of which carries <span class="math notranslate nohighlight">\(p\)</span> features or predictors. The
|
||
primary goal is to identify the classes to which new unseen samples
|
||
belong.</p>
|
||
<p>Let us specialize to the case of two classes only, with outputs
|
||
<span class="math notranslate nohighlight">\(y_i=0\)</span> and <span class="math notranslate nohighlight">\(y_i=1\)</span>. Our outcomes could represent the status of a
|
||
credit card user that could default or not on her/his credit card
|
||
debt. That is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
y_i = \begin{bmatrix} 0 & \mathrm{no}\\ 1 & \mathrm{yes} \end{bmatrix}.
|
||
\end{split}\]</div>
|
||
<p>Before moving to the logistic model, let us try to use our linear
|
||
regression model to classify these two outcomes. We could for example
|
||
fit a linear model to the default case if <span class="math notranslate nohighlight">\(y_i > 0.5\)</span> and the no
|
||
default case <span class="math notranslate nohighlight">\(y_i \leq 0.5\)</span>.</p>
|
||
<p>We would then have our
|
||
weighted linear combination, namely</p>
|
||
<!-- Equation labels as ordinary links -->
|
||
<div id="_auto1"></div>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\begin{equation}
|
||
\boldsymbol{y} = \boldsymbol{X}^T\boldsymbol{\theta} + \boldsymbol{\epsilon},
|
||
\label{_auto1} \tag{1}
|
||
\end{equation}
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> is a vector representing the possible outcomes, <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> is our
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span> design matrix and <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> represents our estimators/predictors.</p>
|
||
<p>The main problem with our function is that it takes values on the
|
||
entire real axis. In the case of logistic regression, however, the
|
||
labels <span class="math notranslate nohighlight">\(y_i\)</span> are discrete variables. A typical example is the credit
|
||
card data discussed below here, where we can set the state of
|
||
defaulting the debt to <span class="math notranslate nohighlight">\(y_i=1\)</span> and not to <span class="math notranslate nohighlight">\(y_i=0\)</span> for one the persons
|
||
in the data set (see the full example below).</p>
|
||
<p>One simple way to get a discrete output is to have sign
|
||
functions that map the output of a linear regressor to values <span class="math notranslate nohighlight">\(\{0,1\}\)</span>,
|
||
<span class="math notranslate nohighlight">\(f(s_i)=sign(s_i)=1\)</span> if <span class="math notranslate nohighlight">\(s_i\ge 0\)</span> and 0 if otherwise.
|
||
We will encounter this model in our first demonstration of neural networks. Historically it is called the <code class="docutils literal notranslate"><span class="pre">perceptron"</span> <span class="pre">model</span> <span class="pre">in</span> <span class="pre">the</span> <span class="pre">machine</span> <span class="pre">learning</span> <span class="pre">literature.</span> <span class="pre">This</span> <span class="pre">model</span> <span class="pre">is</span> <span class="pre">extremely</span> <span class="pre">simple.</span> <span class="pre">However,</span> <span class="pre">in</span> <span class="pre">many</span> <span class="pre">cases</span> <span class="pre">it</span> <span class="pre">is</span> <span class="pre">more</span> <span class="pre">favorable</span> <span class="pre">to</span> <span class="pre">use</span> <span class="pre">a</span> </code>soft” classifier that outputs
|
||
the probability of a given category. This leads us to the logistic function.</p>
|
||
<p>The following example on data for coronary heart disease (CHD) as function of age may serve as an illustration. In the code here we read and plot whether a person has had CHD (output = 1) or not (output = 0). This ouput is plotted the person’s against age. Clearly, the figure shows that attempting to make a standard linear regression fit may not be very meaningful.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>%matplotlib inline
|
||
|
||
# Common imports
|
||
import os
|
||
import numpy as np
|
||
import pandas as pd
|
||
import matplotlib.pyplot as plt
|
||
from sklearn.linear_model import LinearRegression, Ridge, Lasso
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.utils import resample
|
||
from sklearn.metrics import mean_squared_error
|
||
from IPython.display import display
|
||
from pylab import plt, mpl
|
||
plt.style.use('seaborn')
|
||
mpl.rcParams['font.family'] = 'serif'
|
||
|
||
# Where to save the figures and data files
|
||
PROJECT_ROOT_DIR = "Results"
|
||
FIGURE_ID = "Results/FigureFiles"
|
||
DATA_ID = "DataFiles/"
|
||
|
||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||
os.mkdir(PROJECT_ROOT_DIR)
|
||
|
||
if not os.path.exists(FIGURE_ID):
|
||
os.makedirs(FIGURE_ID)
|
||
|
||
if not os.path.exists(DATA_ID):
|
||
os.makedirs(DATA_ID)
|
||
|
||
def image_path(fig_id):
|
||
return os.path.join(FIGURE_ID, fig_id)
|
||
|
||
def data_path(dat_id):
|
||
return os.path.join(DATA_ID, dat_id)
|
||
|
||
def save_fig(fig_id):
|
||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||
|
||
infile = open(data_path("chddata.csv"),'r')
|
||
|
||
# Read the chd data as csv file and organize the data into arrays with age group, age, and chd
|
||
chd = pd.read_csv(infile, names=('ID', 'Age', 'Agegroup', 'CHD'))
|
||
chd.columns = ['ID', 'Age', 'Agegroup', 'CHD']
|
||
output = chd['CHD']
|
||
age = chd['Age']
|
||
agegroup = chd['Agegroup']
|
||
numberID = chd['ID']
|
||
display(chd)
|
||
|
||
plt.scatter(age, output, marker='o')
|
||
plt.axis([18,70.0,-0.1, 1.2])
|
||
plt.xlabel(r'Age')
|
||
plt.ylabel(r'CHD')
|
||
plt.title(r'Age distribution and Coronary heart disease')
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>What we could attempt however is to plot the mean value for each group.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>agegroupmean = np.array([0.1, 0.133, 0.250, 0.333, 0.462, 0.625, 0.765, 0.800])
|
||
group = np.array([1, 2, 3, 4, 5, 6, 7, 8])
|
||
plt.plot(group, agegroupmean, "r-")
|
||
plt.axis([0,9,0, 1.0])
|
||
plt.xlabel(r'Age group')
|
||
plt.ylabel(r'CHD mean values')
|
||
plt.title(r'Mean values for each age group')
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We are now trying to find a function <span class="math notranslate nohighlight">\(f(y\vert x)\)</span>, that is a function which gives us an expected value for the output <span class="math notranslate nohighlight">\(y\)</span> with a given input <span class="math notranslate nohighlight">\(x\)</span>.
|
||
In standard linear regression with a linear dependence on <span class="math notranslate nohighlight">\(x\)</span>, we would write this in terms of our model</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f(y_i\vert x_i)=\theta_0+\theta_1 x_i.
|
||
\]</div>
|
||
<p>This expression implies however that <span class="math notranslate nohighlight">\(f(y_i\vert x_i)\)</span> could take any
|
||
value from minus infinity to plus infinity. If we however let
|
||
<span class="math notranslate nohighlight">\(f(y\vert y)\)</span> be represented by the mean value, the above example
|
||
shows us that we can constrain the function to take values between
|
||
zero and one, that is we have <span class="math notranslate nohighlight">\(0 \le f(y_i\vert x_i) \le 1\)</span>. Looking
|
||
at our last curve we see also that it has an S-shaped form. This leads
|
||
us to a very popular model for the function <span class="math notranslate nohighlight">\(f\)</span>, namely the so-called
|
||
Sigmoid function or logistic model. We will consider this function as
|
||
representing the probability for finding a value of <span class="math notranslate nohighlight">\(y_i\)</span> with a given
|
||
<span class="math notranslate nohighlight">\(x_i\)</span>.</p>
|
||
</section>
|
||
<section id="the-logistic-function">
|
||
<h2><span class="section-number">6.3. </span>The logistic function<a class="headerlink" href="#the-logistic-function" title="Link to this heading">#</a></h2>
|
||
<p>Another widely studied model, is the so-called
|
||
perceptron model, which is an example of a “hard classification” model. We
|
||
will encounter this model when we discuss neural networks as
|
||
well. Each datapoint is deterministically assigned to a category (i.e
|
||
<span class="math notranslate nohighlight">\(y_i=0\)</span> or <span class="math notranslate nohighlight">\(y_i=1\)</span>). In many cases, and the coronary heart disease data forms one of many such examples, it is favorable to have a “soft”
|
||
classifier that outputs the probability of a given category rather
|
||
than a single value. For example, given <span class="math notranslate nohighlight">\(x_i\)</span>, the classifier
|
||
outputs the probability of being in a category <span class="math notranslate nohighlight">\(k\)</span>. Logistic regression
|
||
is the most common example of a so-called soft classifier. In logistic
|
||
regression, the probability that a data point <span class="math notranslate nohighlight">\(x_i\)</span>
|
||
belongs to a category <span class="math notranslate nohighlight">\(y_i=\{0,1\}\)</span> is given by the so-called logit function (or Sigmoid) which is meant to represent the likelihood for a given event,</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
p(t) = \frac{1}{1+\mathrm \exp{-t}}=\frac{\exp{t}}{1+\mathrm \exp{t}}.
|
||
\]</div>
|
||
<p>Note that <span class="math notranslate nohighlight">\(1-p(t)= p(-t)\)</span>.</p>
|
||
</section>
|
||
<section id="examples-of-likelihood-functions-used-in-logistic-regression-and-neural-networks">
|
||
<h2><span class="section-number">6.4. </span>Examples of likelihood functions used in logistic regression and neural networks<a class="headerlink" href="#examples-of-likelihood-functions-used-in-logistic-regression-and-neural-networks" title="Link to this heading">#</a></h2>
|
||
<p>The following code plots the logistic function, the step function and other functions we will encounter from here and on.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>"""The sigmoid function (or the logistic curve) is a
|
||
function that takes any real number, z, and outputs a number (0,1).
|
||
It is useful in neural networks for assigning weights on a relative scale.
|
||
The value z is the weighted sum of parameters involved in the learning algorithm."""
|
||
|
||
import numpy
|
||
import matplotlib.pyplot as plt
|
||
import math as mt
|
||
|
||
z = numpy.arange(-5, 5, .1)
|
||
sigma_fn = numpy.vectorize(lambda z: 1/(1+numpy.exp(-z)))
|
||
sigma = sigma_fn(z)
|
||
|
||
fig = plt.figure()
|
||
ax = fig.add_subplot(111)
|
||
ax.plot(z, sigma)
|
||
ax.set_ylim([-0.1, 1.1])
|
||
ax.set_xlim([-5,5])
|
||
ax.grid(True)
|
||
ax.set_xlabel('z')
|
||
ax.set_title('sigmoid function')
|
||
|
||
plt.show()
|
||
|
||
"""Step Function"""
|
||
z = numpy.arange(-5, 5, .02)
|
||
step_fn = numpy.vectorize(lambda z: 1.0 if z >= 0.0 else 0.0)
|
||
step = step_fn(z)
|
||
|
||
fig = plt.figure()
|
||
ax = fig.add_subplot(111)
|
||
ax.plot(z, step)
|
||
ax.set_ylim([-0.5, 1.5])
|
||
ax.set_xlim([-5,5])
|
||
ax.grid(True)
|
||
ax.set_xlabel('z')
|
||
ax.set_title('step function')
|
||
|
||
plt.show()
|
||
|
||
"""tanh Function"""
|
||
z = numpy.arange(-2*mt.pi, 2*mt.pi, 0.1)
|
||
t = numpy.tanh(z)
|
||
|
||
fig = plt.figure()
|
||
ax = fig.add_subplot(111)
|
||
ax.plot(z, t)
|
||
ax.set_ylim([-1.0, 1.0])
|
||
ax.set_xlim([-2*mt.pi,2*mt.pi])
|
||
ax.grid(True)
|
||
ax.set_xlabel('z')
|
||
ax.set_title('tanh function')
|
||
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>We assume now that we have two classes with <span class="math notranslate nohighlight">\(y_i\)</span> either <span class="math notranslate nohighlight">\(0\)</span> or <span class="math notranslate nohighlight">\(1\)</span>. Furthermore we assume also that we have only two parameters <span class="math notranslate nohighlight">\(\theta\)</span> in our fitting of the Sigmoid function, that is we define probabilities</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
p(y_i=1|x_i,\boldsymbol{\theta}) &= \frac{\exp{(\theta_0+\theta_1x_i)}}{1+\exp{(\theta_0+\theta_1x_i)}},\nonumber\\
|
||
p(y_i=0|x_i,\boldsymbol{\theta}) &= 1 - p(y_i=1|x_i,\boldsymbol{\theta}),
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> are the weights we wish to extract from data, in our case <span class="math notranslate nohighlight">\(\theta_0\)</span> and <span class="math notranslate nohighlight">\(\theta_1\)</span>.</p>
|
||
<p>Note that we used</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
p(y_i=0\vert x_i, \boldsymbol{\theta}) = 1-p(y_i=1\vert x_i, \boldsymbol{\theta}).
|
||
\]</div>
|
||
<p>In order to define the total likelihood for all possible outcomes from a<br />
|
||
dataset <span class="math notranslate nohighlight">\(\mathcal{D}=\{(y_i,x_i)\}\)</span>, with the binary labels
|
||
<span class="math notranslate nohighlight">\(y_i\in\{0,1\}\)</span> and where the data points are drawn independently, we use the so-called <a class="reference external" href="https://en.wikipedia.org/wiki/Maximum_likelihood_estimation">Maximum Likelihood Estimation</a> (MLE) principle.
|
||
We aim thus at maximizing
|
||
the probability of seeing the observed data. We can then approximate the
|
||
likelihood in terms of the product of the individual probabilities of a specific outcome <span class="math notranslate nohighlight">\(y_i\)</span>, that is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[\begin{split}
|
||
\begin{align*}
|
||
P(\mathcal{D}|\boldsymbol{\theta})& = \prod_{i=1}^n \left[p(y_i=1|x_i,\boldsymbol{\theta})\right]^{y_i}\left[1-p(y_i=1|x_i,\boldsymbol{\theta}))\right]^{1-y_i}\nonumber \\
|
||
\end{align*}
|
||
\end{split}\]</div>
|
||
<p>from which we obtain the log-likelihood and our <strong>cost/loss</strong> function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathcal{C}(\boldsymbol{\theta}) = \sum_{i=1}^n \left( y_i\log{p(y_i=1|x_i,\boldsymbol{\theta})} + (1-y_i)\log\left[1-p(y_i=1|x_i,\boldsymbol{\theta}))\right]\right).
|
||
\]</div>
|
||
<p>Reordering the logarithms, we can rewrite the <strong>cost/loss</strong> function as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathcal{C}(\boldsymbol{\theta}) = \sum_{i=1}^n \left(y_i(\theta_0+\theta_1x_i) -\log{(1+\exp{(\theta_0+\theta_1x_i)})}\right).
|
||
\]</div>
|
||
<p>The maximum likelihood estimator is defined as the set of parameters that maximize the log-likelihood where we maximize with respect to <span class="math notranslate nohighlight">\(\theta\)</span>.
|
||
Since the cost (error) function is just the negative log-likelihood, for logistic regression we have that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathcal{C}(\boldsymbol{\theta})=-\sum_{i=1}^n \left(y_i(\theta_0+\theta_1x_i) -\log{(1+\exp{(\theta_0+\theta_1x_i)})}\right).
|
||
\]</div>
|
||
<p>This equation is known in statistics as the <strong>cross entropy</strong>. Finally, we note that just as in linear regression,
|
||
in practice we often supplement the cross-entropy with additional regularization terms, usually <span class="math notranslate nohighlight">\(L_1\)</span> and <span class="math notranslate nohighlight">\(L_2\)</span> regularization as we did for Ridge and Lasso regression.</p>
|
||
<p>The cross entropy is a convex function of the weights <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> and,
|
||
therefore, any local minimizer is a global minimizer.</p>
|
||
<p>Minimizing this
|
||
cost function with respect to the two parameters <span class="math notranslate nohighlight">\(\theta_0\)</span> and <span class="math notranslate nohighlight">\(\theta_1\)</span> we obtain</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \theta_0} = -\sum_{i=1}^n \left(y_i -\frac{\exp{(\theta_0+\theta_1x_i)}}{1+\exp{(\theta_0+\theta_1x_i)}}\right),
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \theta_1} = -\sum_{i=1}^n \left(y_ix_i -x_i\frac{\exp{(\theta_0+\theta_1x_i)}}{1+\exp{(\theta_0+\theta_1x_i)}}\right).
|
||
\]</div>
|
||
<p>Let us now define a vector <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span> with <span class="math notranslate nohighlight">\(n\)</span> elements <span class="math notranslate nohighlight">\(y_i\)</span>, an
|
||
<span class="math notranslate nohighlight">\(n\times p\)</span> matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> which contains the <span class="math notranslate nohighlight">\(x_i\)</span> values and a
|
||
vector <span class="math notranslate nohighlight">\(\boldsymbol{p}\)</span> of fitted probabilities <span class="math notranslate nohighlight">\(p(y_i\vert x_i,\boldsymbol{\theta})\)</span>. We can rewrite in a more compact form the first
|
||
derivative of cost function as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}} = -\boldsymbol{X}^T\left(\boldsymbol{y}-\boldsymbol{p}\right).
|
||
\]</div>
|
||
<p>If we in addition define a diagonal matrix <span class="math notranslate nohighlight">\(\boldsymbol{W}\)</span> with elements
|
||
<span class="math notranslate nohighlight">\(p(y_i\vert x_i,\boldsymbol{\theta})(1-p(y_i\vert x_i,\boldsymbol{\theta})\)</span>, we can obtain a compact expression of the second derivative as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial^2 \mathcal{C}(\boldsymbol{\theta})}{\partial \boldsymbol{\theta}\partial \boldsymbol{\theta}^T} = \boldsymbol{X}^T\boldsymbol{W}\boldsymbol{X}.
|
||
\]</div>
|
||
<p>Within a binary classification problem, we can easily expand our model to include multiple predictors. Our ratio between likelihoods is then with <span class="math notranslate nohighlight">\(p\)</span> predictors</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\log{ \frac{p(\boldsymbol{\theta}\boldsymbol{x})}{1-p(\boldsymbol{\theta}\boldsymbol{x})}} = \theta_0+\theta_1x_1+\theta_2x_2+\dots+\theta_px_p.
|
||
\]</div>
|
||
<p>Here we defined <span class="math notranslate nohighlight">\(\boldsymbol{x}=[1,x_1,x_2,\dots,x_p]\)</span> and <span class="math notranslate nohighlight">\(\boldsymbol{\theta}=[\theta_0, \theta_1, \dots, \theta_p]\)</span> leading to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
p(\boldsymbol{\theta}\boldsymbol{x})=\frac{ \exp{(\theta_0+\theta_1x_1+\theta_2x_2+\dots+\theta_px_p)}}{1+\exp{(\theta_0+\theta_1x_1+\theta_2x_2+\dots+\theta_px_p)}}.
|
||
\]</div>
|
||
<p>Till now we have mainly focused on two classes, the so-called binary
|
||
system. Suppose we wish to extend to <span class="math notranslate nohighlight">\(K\)</span> classes. Let us for the sake
|
||
of simplicity assume we have only two predictors. We have then following model</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\log{\frac{p(C=1\vert x)}{p(K\vert x)}} = \theta_{10}+\theta_{11}x_1,
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\log{\frac{p(C=2\vert x)}{p(K\vert x)}} = \theta_{20}+\theta_{21}x_1,
|
||
\]</div>
|
||
<p>and so on till the class <span class="math notranslate nohighlight">\(C=K-1\)</span> class</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\log{\frac{p(C=K-1\vert x)}{p(K\vert x)}} = \theta_{(K-1)0}+\theta_{(K-1)1}x_1,
|
||
\]</div>
|
||
<p>and the model is specified in term of <span class="math notranslate nohighlight">\(K-1\)</span> so-called log-odds or
|
||
<strong>logit</strong> transformations.</p>
|
||
<p>In our discussion of neural networks we will encounter the above again
|
||
in terms of a slightly modified function, the so-called <strong>Softmax</strong> function.</p>
|
||
<p>The softmax function is used in various multiclass classification
|
||
methods, such as multinomial logistic regression (also known as
|
||
softmax regression), multiclass linear discriminant analysis, naive
|
||
Bayes classifiers, and artificial neural networks. Specifically, in
|
||
multinomial logistic regression and linear discriminant analysis, the
|
||
input to the function is the result of <span class="math notranslate nohighlight">\(K\)</span> distinct linear functions,
|
||
and the predicted probability for the <span class="math notranslate nohighlight">\(k\)</span>-th class given a sample
|
||
vector <span class="math notranslate nohighlight">\(\boldsymbol{x}\)</span> and a weighting vector <span class="math notranslate nohighlight">\(\boldsymbol{\theta}\)</span> is (with two
|
||
predictors):</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
p(C=k\vert \mathbf {x} )=\frac{\exp{(\theta_{k0}+\theta_{k1}x_1)}}{1+\sum_{l=1}^{K-1}\exp{(\theta_{l0}+\theta_{l1}x_1)}}.
|
||
\]</div>
|
||
<p>It is easy to extend to more predictors. The final class is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
p(C=K\vert \mathbf {x} )=\frac{1}{1+\sum_{l=1}^{K-1}\exp{(\theta_{l0}+\theta_{l1}x_1)}},
|
||
\]</div>
|
||
<p>and they sum to one. Our earlier discussions were all specialized to
|
||
the case with two classes only. It is easy to see from the above that
|
||
what we derived earlier is compatible with these equations.</p>
|
||
<p>To find the optimal parameters we would typically use a gradient
|
||
descent method. Newton’s method and gradient descent methods are
|
||
discussed in the material on <a class="reference external" href="https://compphysics.github.io/MachineLearning/doc/pub/Splines/html/Splines-bs.html">optimization
|
||
methods</a>.</p>
|
||
</section>
|
||
<section id="wisconsin-cancer-data">
|
||
<h2><span class="section-number">6.5. </span>Wisconsin Cancer Data<a class="headerlink" href="#wisconsin-cancer-data" title="Link to this heading">#</a></h2>
|
||
<p>We show here how we can use a simple regression case on the breast
|
||
cancer data using Logistic regression as our algorithm for
|
||
classification.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import load_breast_cancer
|
||
from sklearn.linear_model import LogisticRegression
|
||
|
||
# Load the data
|
||
cancer = load_breast_cancer()
|
||
|
||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||
print(X_train.shape)
|
||
print(X_test.shape)
|
||
# Logistic Regression
|
||
logreg = LogisticRegression(solver='lbfgs')
|
||
logreg.fit(X_train, y_train)
|
||
print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
|
||
#now scale the data
|
||
from sklearn.preprocessing import StandardScaler
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
# Logistic Regression
|
||
logreg.fit(X_train_scaled, y_train)
|
||
print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>In addition to the above scores, we could also study the covariance (and the correlation matrix).
|
||
We use <strong>Pandas</strong> to compute the correlation matrix.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import load_breast_cancer
|
||
from sklearn.linear_model import LogisticRegression
|
||
cancer = load_breast_cancer()
|
||
import pandas as pd
|
||
# Making a data frame
|
||
cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||
|
||
fig, axes = plt.subplots(15,2,figsize=(10,20))
|
||
malignant = cancer.data[cancer.target == 0]
|
||
benign = cancer.data[cancer.target == 1]
|
||
ax = axes.ravel()
|
||
|
||
for i in range(30):
|
||
_, bins = np.histogram(cancer.data[:,i], bins =50)
|
||
ax[i].hist(malignant[:,i], bins = bins, alpha = 0.5)
|
||
ax[i].hist(benign[:,i], bins = bins, alpha = 0.5)
|
||
ax[i].set_title(cancer.feature_names[i])
|
||
ax[i].set_yticks(())
|
||
ax[0].set_xlabel("Feature magnitude")
|
||
ax[0].set_ylabel("Frequency")
|
||
ax[0].legend(["Malignant", "Benign"], loc ="best")
|
||
fig.tight_layout()
|
||
plt.show()
|
||
|
||
import seaborn as sns
|
||
correlation_matrix = cancerpd.corr().round(1)
|
||
# use the heatmap function from seaborn to plot the correlation matrix
|
||
# annot = True to print the values inside the square
|
||
plt.figure(figsize=(15,8))
|
||
sns.heatmap(data=correlation_matrix, annot=True)
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>In the above example we note two things. In the first plot we display
|
||
the overlap of benign and malignant tumors as functions of the various
|
||
features in the Wisconsing breast cancer data set. We see that for
|
||
some of the features we can distinguish clearly the benign and
|
||
malignant cases while for other features we cannot. This can point to
|
||
us which features may be of greater interest when we wish to classify
|
||
a benign or not benign tumour.</p>
|
||
<p>In the second figure we have computed the so-called correlation
|
||
matrix, which in our case with thirty features becomes a <span class="math notranslate nohighlight">\(30\times 30\)</span>
|
||
matrix.</p>
|
||
<p>We constructed this matrix using <strong>pandas</strong> via the statements</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>cancerpd = pd.DataFrame(cancer.data, columns=cancer.feature_names)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>and then</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>correlation_matrix = cancerpd.corr().round(1)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Diagonalizing this matrix we can in turn say something about which
|
||
features are of relevance and which are not. This leads us to
|
||
the classical Principal Component Analysis (PCA) theorem with
|
||
applications. This will be discussed later this semester (<a class="reference external" href="https://compphysics.github.io/MachineLearning/doc/pub/week43/html/week43-bs.html">week 43</a>).</p>
|
||
<p>Here we present a further way to present our results in terms of a so-called <strong>confusion matrix</strong>, the cumulative gain and the <strong>ROC</strong> curve.
|
||
This way of displaying our data are based upon different ways to classify our possible outcomes. Before we proceed we need some definitions.</p>
|
||
<ol class="arabic simple">
|
||
<li><p><strong>TP</strong>: true positive or in other words, something equivalent with a proper classification</p></li>
|
||
<li><p><strong>TN</strong>: true negative, which is equivalent with a correct rejection</p></li>
|
||
<li><p><strong>FP</strong>: false positive, or in simpler words something that is equivalent with a false alarm</p></li>
|
||
<li><p><strong>FN</strong>: false negative, which is mean to be equivalent with a miss.</p></li>
|
||
</ol>
|
||
<p>The total data set is then the sum of the true positive and true negative targets or outputs, labeled by <span class="math notranslate nohighlight">\(n\)</span>.
|
||
Based on this we can then define the accuracy score as the sum of correctly predicted <strong>TP</strong> and <strong>TN</strong> cases divided by the sum of true positive and treue negative events in our data set, or as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{Accuracy} = \frac{\sum_{i=0}^{n-1}I(y_i=\tilde{y}_i)}{n}.
|
||
\]</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import load_breast_cancer
|
||
from sklearn.linear_model import LogisticRegression
|
||
|
||
# Load the data
|
||
cancer = load_breast_cancer()
|
||
|
||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||
print(X_train.shape)
|
||
print(X_test.shape)
|
||
# Logistic Regression
|
||
logreg = LogisticRegression(solver='lbfgs')
|
||
logreg.fit(X_train, y_train)
|
||
print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
|
||
#now scale the data
|
||
from sklearn.preprocessing import StandardScaler
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
# Logistic Regression
|
||
logreg.fit(X_train_scaled, y_train)
|
||
print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||
|
||
|
||
from sklearn.preprocessing import LabelEncoder
|
||
from sklearn.model_selection import cross_validate
|
||
#Cross validation
|
||
accuracy = cross_validate(logreg,X_test_scaled,y_test,cv=10)['test_score']
|
||
print(accuracy)
|
||
print("Test set accuracy with Logistic Regression and scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||
|
||
|
||
import scikitplot as skplt
|
||
y_pred = logreg.predict(X_test_scaled)
|
||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||
plt.show()
|
||
y_probas = logreg.predict_proba(X_test_scaled)
|
||
skplt.metrics.plot_roc(y_test, y_probas)
|
||
plt.show()
|
||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="chapter3.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title"><span class="section-number">5. </span>Resampling Methods</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="chapteroptimization.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title"><span class="section-number">7. </span>Optimization, the central part of any Machine Learning algortithm</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#id1">6.1. Logistic Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#basics">6.2. Basics</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#the-logistic-function">6.3. The logistic function</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#examples-of-likelihood-functions-used-in-logistic-regression-and-neural-networks">6.4. Examples of likelihood functions used in logistic regression and neural networks</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#wisconsin-cancer-data">6.5. Wisconsin Cancer Data</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |