KarlHenrik
1571 lines
78 KiB
HTML
1571 lines
78 KiB
HTML
|
||
<!DOCTYPE html>
|
||
|
||
|
||
<html lang="en" data-content_root="./" >
|
||
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods — Applied Data Analysis and Machine Learning</title>
|
||
|
||
|
||
|
||
<script data-cfasync="false">
|
||
document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
|
||
document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
|
||
</script>
|
||
|
||
<!-- Loaded before other Sphinx assets -->
|
||
<link href="_static/styles/theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/bootstrap.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link href="_static/styles/pydata-sphinx-theme.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
|
||
|
||
<link href="_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=dfe6caa3a7d634c4db9b" rel="stylesheet" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
|
||
<link rel="preload" as="font" type="font/woff2" crossorigin href="_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />
|
||
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=03e43079" />
|
||
<link rel="stylesheet" type="text/css" href="_static/styles/sphinx-book-theme.css?v=eba8b062" />
|
||
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css?v=13237357" />
|
||
<link rel="stylesheet" type="text/css" href="_static/copybutton.css?v=76b2166b" />
|
||
<link rel="stylesheet" type="text/css" href="_static/mystnb.8ecb98da25f57f5357bf6f572d296f466b2cfe2517ffebfabe82451661e28f02.css?v=6644e6bb" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css?v=4fa983c6" />
|
||
<link rel="stylesheet" type="text/css" href="_static/sphinx-design.min.css?v=95c83b7e" />
|
||
|
||
<!-- Pre-loaded scripts that we'll load fully later -->
|
||
<link rel="preload" as="script" href="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b" />
|
||
<script src="_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<script src="_static/documentation_options.js?v=9eb32ce0"></script>
|
||
<script src="_static/doctools.js?v=9a2dae69"></script>
|
||
<script src="_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<script src="_static/clipboard.min.js?v=a7894cd8"></script>
|
||
<script src="_static/copybutton.js?v=f281be69"></script>
|
||
<script src="_static/scripts/sphinx-book-theme.js?v=887ef09a"></script>
|
||
<script>let toggleHintShow = 'Click to show';</script>
|
||
<script>let toggleHintHide = 'Click to hide';</script>
|
||
<script>let toggleOpenOnPrint = 'true';</script>
|
||
<script src="_static/togglebutton.js?v=4a39c7ea"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script src="_static/design-tabs.js?v=f930bc37"></script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script async="async" src="_static/sphinx-thebe.js?v=c100c467"></script>
|
||
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown';</script>
|
||
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"; const thebe_selector = ".thebe,.cell"; const thebe_selector_input = "pre"; const thebe_selector_output = ".output, .cell_output"</script>
|
||
<script>window.MathJax = {"options": {"processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
|
||
<script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
||
<script>DOCUMENTATION_OPTIONS.pagename = 'chapter7';</script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="11. Basic ideas of the Principal Component Analysis (PCA)" href="chapter8.html" />
|
||
<link rel="prev" title="9. Decision trees, overarching aims" href="chapter6.html" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||
<meta name="docsearch:language" content="en"/>
|
||
</head>
|
||
|
||
|
||
<body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">
|
||
|
||
|
||
|
||
<div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
|
||
|
||
<div id="pst-scroll-pixel-helper"></div>
|
||
|
||
<button type="button" class="btn rounded-pill" id="pst-back-to-top">
|
||
<i class="fa-solid fa-arrow-up"></i>Back to top</button>
|
||
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-primary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
|
||
|
||
<input type="checkbox"
|
||
class="sidebar-toggle"
|
||
id="pst-secondary-sidebar-checkbox"/>
|
||
<label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
|
||
|
||
<div class="search-button__wrapper">
|
||
<div class="search-button__overlay"></div>
|
||
<div class="search-button__search-container">
|
||
<form class="bd-search d-flex align-items-center"
|
||
action="search.html"
|
||
method="get">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<input type="search"
|
||
class="form-control"
|
||
name="q"
|
||
id="search-input"
|
||
placeholder="Search this book..."
|
||
aria-label="Search this book..."
|
||
autocomplete="off"
|
||
autocorrect="off"
|
||
autocapitalize="off"
|
||
spellcheck="false"/>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
|
||
</form></div>
|
||
</div>
|
||
|
||
<div class="pst-async-banner-revealer d-none">
|
||
<aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
|
||
</div>
|
||
|
||
|
||
<header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
|
||
</header>
|
||
|
||
|
||
<div class="bd-container">
|
||
<div class="bd-container__inner bd-page-width">
|
||
|
||
|
||
|
||
<div class="bd-sidebar-primary bd-sidebar">
|
||
|
||
|
||
|
||
<div class="sidebar-header-items sidebar-primary__section">
|
||
|
||
|
||
|
||
|
||
</div>
|
||
|
||
<div class="sidebar-primary-items__start sidebar-primary__section">
|
||
<div class="sidebar-primary-item">
|
||
|
||
|
||
|
||
|
||
|
||
<a class="navbar-brand logo" href="intro.html">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<img src="_static/logo.png" class="logo__image only-light" alt="Applied Data Analysis and Machine Learning - Home"/>
|
||
<script>document.write(`<img src="_static/logo.png" class="logo__image only-dark" alt="Applied Data Analysis and Machine Learning - Home"/>`);</script>
|
||
|
||
|
||
</a></div>
|
||
<div class="sidebar-primary-item">
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass"></i>
|
||
<span class="search-button__default-text">Search</span>
|
||
<span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
|
||
</button>
|
||
`);
|
||
</script></div>
|
||
<div class="sidebar-primary-item"><nav class="bd-links bd-docs-nav" aria-label="Main">
|
||
<div class="bd-toc-item navbar-nav active">
|
||
|
||
<ul class="nav bd-sidenav bd-sidenav__home-link">
|
||
<li class="toctree-l1">
|
||
<a class="reference internal" href="intro.html">
|
||
Applied Data Analysis and Machine Learning
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">About the course</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="schedule.html">Course setting</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="teachers.html">Teachers and Grading</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="textbooks.html">Textbooks</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Review of Statistics with Resampling Techniques and Linear Algebra</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="statistics.html">1. Elements of Probability Theory and Statistical Data Analysis</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="linalg.html">2. Linear Algebra, Handling of Arrays and more Python Features</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">From Regression to Support Vector Machines</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter1.html">3. Linear Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter2.html">4. Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter3.html">5. Resampling Methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter4.html">6. Logistic Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapteroptimization.html">7. Optimization, the central part of any Machine Learning algortithm</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter5.html">8. Support Vector Machines, overarching aims</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Decision Trees, Ensemble Methods and Boosting</span></p>
|
||
<ul class="current nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter6.html">9. Decision trees, overarching aims</a></li>
|
||
<li class="toctree-l1 current active"><a class="current reference internal" href="#">10. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Dimensionality Reduction</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter8.html">11. Basic ideas of the Principal Component Analysis (PCA)</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="clustering.html">12. Clustering and Unsupervised Learning</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Deep Learning Methods</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter9.html">13. Neural networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter10.html">14. Building a Feed Forward Neural Network</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter11.html">15. Solving Differential Equations with Deep Learning</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter12.html">16. Convolutional Neural Networks</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="chapter13.html">17. Recurrent neural networks: Overarching view</a></li>
|
||
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Weekly material, notes and exercises</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek34.html">Exercises week 34</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week34.html">Week 34: Introduction to the course, Logistics and Practicalities</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek35.html">Exercises week 35</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week35.html">Week 35: From Ordinary Linear Regression to Ridge and Lasso Regression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek36.html">Exercises week 36</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week36.html">Week 36: Linear Regression and Gradient descent</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek37.html">Exercises week 37</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="week37.html">Week 37: Gradient descent methods</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="exercisesweek38.html">Exercises week 38</a></li>
|
||
</ul>
|
||
<p aria-level="2" class="caption" role="heading"><span class="caption-text">Projects</span></p>
|
||
<ul class="nav bd-sidenav">
|
||
<li class="toctree-l1"><a class="reference internal" href="project1.html">Project 1 on Machine Learning, deadline October 6 (midnight), 2025</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</nav></div>
|
||
</div>
|
||
|
||
|
||
<div class="sidebar-primary-items__end sidebar-primary__section">
|
||
</div>
|
||
|
||
<div id="rtd-footer-container"></div>
|
||
|
||
|
||
</div>
|
||
|
||
<main id="main-content" class="bd-main" role="main">
|
||
|
||
|
||
|
||
<div class="sbt-scroll-pixel-helper"></div>
|
||
|
||
<div class="bd-content">
|
||
<div class="bd-article-container">
|
||
|
||
<div class="bd-header-article d-print-none">
|
||
<div class="header-article-items header-article__inner">
|
||
|
||
<div class="header-article-items__start">
|
||
|
||
<div class="header-article-item"><button class="sidebar-toggle primary-toggle btn btn-sm" title="Toggle primary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-bars"></span>
|
||
</button></div>
|
||
|
||
</div>
|
||
|
||
|
||
<div class="header-article-items__end">
|
||
|
||
<div class="header-article-item">
|
||
|
||
<div class="article-header-buttons">
|
||
|
||
|
||
|
||
|
||
|
||
<div class="dropdown dropdown-download-buttons">
|
||
<button class="btn dropdown-toggle" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-label="Download this page">
|
||
<i class="fas fa-download"></i>
|
||
</button>
|
||
<ul class="dropdown-menu">
|
||
|
||
|
||
|
||
<li><a href="_sources/chapter7.ipynb" target="_blank"
|
||
class="btn btn-sm btn-download-source-button dropdown-item"
|
||
title="Download source file"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file"></i>
|
||
</span>
|
||
<span class="btn__text-container">.ipynb</span>
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
<li>
|
||
<button onclick="window.print()"
|
||
class="btn btn-sm btn-download-pdf-button dropdown-item"
|
||
title="Print to PDF"
|
||
data-bs-placement="left" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-file-pdf"></i>
|
||
</span>
|
||
<span class="btn__text-container">.pdf</span>
|
||
</button>
|
||
</li>
|
||
|
||
</ul>
|
||
</div>
|
||
|
||
|
||
|
||
|
||
<button onclick="toggleFullScreen()"
|
||
class="btn btn-sm btn-fullscreen-button"
|
||
title="Fullscreen mode"
|
||
data-bs-placement="bottom" data-bs-toggle="tooltip"
|
||
>
|
||
|
||
|
||
<span class="btn__icon-container">
|
||
<i class="fas fa-expand"></i>
|
||
</span>
|
||
|
||
</button>
|
||
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm nav-link pst-navbar-icon theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="theme-switch fa-solid fa-sun fa-lg" data-mode="light"></i>
|
||
<i class="theme-switch fa-solid fa-moon fa-lg" data-mode="dark"></i>
|
||
<i class="theme-switch fa-solid fa-circle-half-stroke fa-lg" data-mode="auto"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
|
||
|
||
<script>
|
||
document.write(`
|
||
<button class="btn btn-sm pst-navbar-icon search-button search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<i class="fa-solid fa-magnifying-glass fa-lg"></i>
|
||
</button>
|
||
`);
|
||
</script>
|
||
<button class="sidebar-toggle secondary-toggle btn btn-sm" title="Toggle secondary sidebar" data-bs-placement="bottom" data-bs-toggle="tooltip">
|
||
<span class="fa-solid fa-list"></span>
|
||
</button>
|
||
</div></div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="jb-print-docs-body" class="onlyprint">
|
||
<h1>Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</h1>
|
||
<!-- Table of contents -->
|
||
<div id="print-main-content">
|
||
<div id="jb-print-toc">
|
||
|
||
<div>
|
||
<h2> Contents </h2>
|
||
</div>
|
||
<nav aria-label="Page">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#an-overview-of-ensemble-methods">10.1. An Overview of Ensemble Methods</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#bagging">10.2. Bagging</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#bagging-examples">10.3. Bagging Examples</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#making-your-own-bootstrap-changing-the-level-of-the-decision-tree">10.3.1. Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#random-forests">10.4. Random forests</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#compare-bagging-on-trees-with-random-forests">10.4.1. Compare Bagging on Trees with Random Forests</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#boosting-a-bird-s-eye-view">10.5. Boosting, a Bird’s Eye View</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#iterative-fitting-regression-and-squared-error-cost-function">10.5.1. Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#iterative-fitting-classification-and-adaboost">10.5.2. Iterative Fitting, Classification and AdaBoost</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#adaptive-boosting-adaboost-basic-algorithm">10.5.3. Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent">10.6. Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gradient-boosting-examples-of-regression">10.7. Gradient Boosting, Examples of Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gradient-boosting-classification-example">10.8. Gradient Boosting, Classification Example</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#xgboost-extreme-gradient-boosting">10.9. XGBoost: Extreme Gradient Boosting</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#regression-case">10.10. Regression Case</a></li>
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div id="searchbox"></div>
|
||
<article class="bd-article">
|
||
|
||
<section class="tex2jax_ignore mathjax_ignore" id="ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods">
|
||
<h1><span class="section-number">10. </span>Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods<a class="headerlink" href="#ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods" title="Link to this heading">#</a></h1>
|
||
<p>As stated previously and seen in many of the examples discussed in the previous chapter about
|
||
a single decision tree, we often end up overfitting our training
|
||
data. This normally means that we have a high variance. Can we reduce
|
||
the variance of a statistical learning method?</p>
|
||
<p>This leads us to a set of different methods that can combine different
|
||
machine learning algorithms or just use one of them to construct
|
||
forests and jungles of trees, homogeneous ones or heterogenous
|
||
ones. These methods are recognized by different names which we will
|
||
try to explain here. These are</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Voting classifiers</p></li>
|
||
<li><p>Bagging and Pasting</p></li>
|
||
<li><p>Random forests</p></li>
|
||
<li><p>Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost)</p></li>
|
||
</ol>
|
||
<p>We discuss these methods here.</p>
|
||
<section id="an-overview-of-ensemble-methods">
|
||
<h2><span class="section-number">10.1. </span>An Overview of Ensemble Methods<a class="headerlink" href="#an-overview-of-ensemble-methods" title="Link to this heading">#</a></h2>
|
||
<!-- FIGURE: [DataFiles/ensembleoverview.png, width=600 frac=0.8] -->
|
||
</section>
|
||
<section id="bagging">
|
||
<h2><span class="section-number">10.2. </span>Bagging<a class="headerlink" href="#bagging" title="Link to this heading">#</a></h2>
|
||
<p>The <strong>plain</strong> decision trees suffer from high
|
||
variance. This means that if we split the training data into two parts
|
||
at random, and fit a decision tree to both halves, the results that we
|
||
get could be quite different. In contrast, a procedure with low
|
||
variance will yield similar results if applied repeatedly to distinct
|
||
data sets; linear regression tends to have low variance, if the ratio
|
||
of <span class="math notranslate nohighlight">\(n\)</span> to <span class="math notranslate nohighlight">\(p\)</span> is moderately large.</p>
|
||
<p><strong>Bootstrap aggregation</strong>, or just <strong>bagging</strong>, is a
|
||
general-purpose procedure for reducing the variance of a statistical
|
||
learning method.</p>
|
||
<p>Bagging typically results in improved accuracy
|
||
over prediction using a single tree. Unfortunately, however, it can be
|
||
difficult to interpret the resulting model. Recall that one of the
|
||
advantages of decision trees is the attractive and easily interpreted
|
||
diagram that results.</p>
|
||
<p>However, when we bag a large number of trees, it is no longer
|
||
possible to represent the resulting statistical learning procedure
|
||
using a single tree, and it is no longer clear which variables are
|
||
most important to the procedure. Thus, bagging improves prediction
|
||
accuracy at the expense of interpretability. Although the collection
|
||
of bagged trees is much more difficult to interpret than a single
|
||
tree, one can obtain an overall summary of the importance of each
|
||
predictor using the MSE (for bagging regression trees) or the Gini
|
||
index (for bagging classification trees). In the case of bagging
|
||
regression trees, we can record the total amount that the MSE is
|
||
decreased due to splits over a given predictor, averaged over all <span class="math notranslate nohighlight">\(B\)</span> possible
|
||
trees. A large value indicates an important predictor. Similarly, in
|
||
the context of bagging classification trees, we can add up the total
|
||
amount that the Gini index is decreased by splits over a given
|
||
predictor, averaged over all <span class="math notranslate nohighlight">\(B\)</span> trees.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>heads_proba = 0.51
|
||
coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
|
||
cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
|
||
plt.figure(figsize=(8,3.5))
|
||
plt.plot(cumulative_heads_ratio)
|
||
plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
|
||
plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
|
||
plt.xlabel("Number of coin tosses")
|
||
plt.ylabel("Heads ratio")
|
||
plt.legend(loc="lower right")
|
||
plt.axis([0, 10000, 0.42, 0.58])
|
||
save_fig("votingsimple")
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import make_moons
|
||
|
||
X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
|
||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
|
||
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
from sklearn.ensemble import VotingClassifier
|
||
from sklearn.linear_model import LogisticRegression
|
||
from sklearn.svm import SVC
|
||
|
||
log_clf = LogisticRegression(solver="liblinear", random_state=42)
|
||
rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||
svm_clf = SVC(gamma="auto", random_state=42)
|
||
|
||
voting_clf = VotingClassifier(
|
||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||
voting='hard')
|
||
|
||
voting_clf.fit(X_train, y_train)
|
||
|
||
from sklearn.metrics import accuracy_score
|
||
|
||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||
clf.fit(X_train, y_train)
|
||
y_pred = clf.predict(X_test)
|
||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||
|
||
log_clf = LogisticRegression(solver="liblinear", random_state=42)
|
||
rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
|
||
svm_clf = SVC(gamma="auto", probability=True, random_state=42)
|
||
|
||
voting_clf = VotingClassifier(
|
||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||
voting='soft')
|
||
voting_clf.fit(X_train, y_train)
|
||
|
||
from sklearn.metrics import accuracy_score
|
||
|
||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||
clf.fit(X_train, y_train)
|
||
y_pred = clf.predict(X_test)
|
||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import make_moons
|
||
|
||
X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
|
||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
from sklearn.ensemble import VotingClassifier
|
||
from sklearn.linear_model import LogisticRegression
|
||
from sklearn.svm import SVC
|
||
|
||
log_clf = LogisticRegression(random_state=42)
|
||
rnd_clf = RandomForestClassifier(random_state=42)
|
||
svm_clf = SVC(random_state=42)
|
||
|
||
voting_clf = VotingClassifier(
|
||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||
voting='hard')
|
||
voting_clf.fit(X_train, y_train)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from sklearn.metrics import accuracy_score
|
||
|
||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||
clf.fit(X_train, y_train)
|
||
y_pred = clf.predict(X_test)
|
||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>log_clf = LogisticRegression(random_state=42)
|
||
rnd_clf = RandomForestClassifier(random_state=42)
|
||
svm_clf = SVC(probability=True, random_state=42)
|
||
|
||
voting_clf = VotingClassifier(
|
||
estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
|
||
voting='soft')
|
||
voting_clf.fit(X_train, y_train)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from sklearn.metrics import accuracy_score
|
||
|
||
for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
|
||
clf.fit(X_train, y_train)
|
||
y_pred = clf.predict(X_test)
|
||
print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="bagging-examples">
|
||
<h2><span class="section-number">10.3. </span>Bagging Examples<a class="headerlink" href="#bagging-examples" title="Link to this heading">#</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from sklearn.ensemble import BaggingClassifier
|
||
from sklearn.tree import DecisionTreeClassifier
|
||
|
||
bag_clf = BaggingClassifier(
|
||
DecisionTreeClassifier(random_state=42), n_estimators=500,
|
||
max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
|
||
bag_clf.fit(X_train, y_train)
|
||
y_pred = bag_clf.predict(X_test)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from sklearn.metrics import accuracy_score
|
||
print(accuracy_score(y_test, y_pred))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>tree_clf = DecisionTreeClassifier(random_state=42)
|
||
tree_clf.fit(X_train, y_train)
|
||
y_pred_tree = tree_clf.predict(X_test)
|
||
print(accuracy_score(y_test, y_pred_tree))
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>%matplotlib inline
|
||
|
||
from matplotlib.colors import ListedColormap
|
||
|
||
def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
|
||
x1s = np.linspace(axes[0], axes[1], 100)
|
||
x2s = np.linspace(axes[2], axes[3], 100)
|
||
x1, x2 = np.meshgrid(x1s, x2s)
|
||
X_new = np.c_[x1.ravel(), x2.ravel()]
|
||
y_pred = clf.predict(X_new).reshape(x1.shape)
|
||
custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
|
||
plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
|
||
if contour:
|
||
custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
|
||
plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
|
||
plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
|
||
plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
|
||
plt.axis(axes)
|
||
plt.xlabel(r"$x_1$", fontsize=18)
|
||
plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
|
||
plt.figure(figsize=(11,4))
|
||
plt.subplot(121)
|
||
plot_decision_boundary(tree_clf, X, y)
|
||
plt.title("Decision Tree", fontsize=14)
|
||
plt.subplot(122)
|
||
plot_decision_boundary(bag_clf, X, y)
|
||
plt.title("Decision Trees with Bagging", fontsize=14)
|
||
save_fig("baggingtree")
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<section id="making-your-own-bootstrap-changing-the-level-of-the-decision-tree">
|
||
<h3><span class="section-number">10.3.1. </span>Making your own Bootstrap: Changing the Level of the Decision Tree<a class="headerlink" href="#making-your-own-bootstrap-changing-the-level-of-the-decision-tree" title="Link to this heading">#</a></h3>
|
||
<p>Let us bring up our good old boostrap example from the linear regression lectures. We change the linerar regression algorithm with
|
||
a decision tree wth different depths and perform a bootstrap aggregate (in this case we perform as many bootstraps as data points <span class="math notranslate nohighlight">\(n\)</span>).</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.pipeline import make_pipeline
|
||
from sklearn.utils import resample
|
||
from sklearn.tree import DecisionTreeRegressor
|
||
|
||
n = 100
|
||
n_boostraps = 100
|
||
maxdepth = 8
|
||
|
||
# Make data set.
|
||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||
error = np.zeros(maxdepth)
|
||
bias = np.zeros(maxdepth)
|
||
variance = np.zeros(maxdepth)
|
||
polydegree = np.zeros(maxdepth)
|
||
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
|
||
|
||
from sklearn.preprocessing import StandardScaler
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
|
||
# we produce a simple tree first as benchmark
|
||
simpletree = DecisionTreeRegressor(max_depth=3)
|
||
simpletree.fit(X_train_scaled, y_train)
|
||
simpleprediction = simpletree.predict(X_test_scaled)
|
||
for degree in range(1,maxdepth):
|
||
model = DecisionTreeRegressor(max_depth=degree)
|
||
y_pred = np.empty((y_test.shape[0], n_boostraps))
|
||
for i in range(n_boostraps):
|
||
x_, y_ = resample(X_train_scaled, y_train)
|
||
model.fit(x_, y_)
|
||
y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
|
||
|
||
polydegree[degree] = degree
|
||
error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
|
||
bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
|
||
variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
|
||
print('Polynomial degree:', degree)
|
||
print('Error:', error[degree])
|
||
print('Bias^2:', bias[degree])
|
||
print('Var:', variance[degree])
|
||
print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
|
||
|
||
mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**2)
|
||
print(mse_simpletree)
|
||
plt.xlim(1,maxdepth)
|
||
plt.plot(polydegree, error, label='MSE')
|
||
plt.plot(polydegree, bias, label='bias')
|
||
plt.plot(polydegree, variance, label='Variance')
|
||
plt.legend()
|
||
save_fig("baggingboot")
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="random-forests">
|
||
<h2><span class="section-number">10.4. </span>Random forests<a class="headerlink" href="#random-forests" title="Link to this heading">#</a></h2>
|
||
<p>Random forests provide an improvement over bagged trees by way of a
|
||
small tweak that decorrelates the trees.</p>
|
||
<p>As in bagging, we build a
|
||
number of decision trees on bootstrapped training samples. But when
|
||
building these decision trees, each time a split in a tree is
|
||
considered, a random sample of <span class="math notranslate nohighlight">\(m\)</span> predictors is chosen as split
|
||
candidates from the full set of <span class="math notranslate nohighlight">\(p\)</span> predictors. The split is allowed to
|
||
use only one of those <span class="math notranslate nohighlight">\(m\)</span> predictors.</p>
|
||
<p>A fresh sample of <span class="math notranslate nohighlight">\(m\)</span> predictors is
|
||
taken at each split, and typically we choose</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
m\approx \sqrt{p}.
|
||
\]</div>
|
||
<p>In building a random forest, at
|
||
each split in the tree, the algorithm is not even allowed to consider
|
||
a majority of the available predictors.</p>
|
||
<p>The reason for this is rather clever. Suppose that there is one very
|
||
strong predictor in the data set, along with a number of other
|
||
moderately strong predictors. Then in the collection of bagged
|
||
variable importance random forest trees, most or all of the trees will
|
||
use this strong predictor in the top split. Consequently, all of the
|
||
bagged trees will look quite similar to each other. Hence the
|
||
predictions from the bagged trees will be highly correlated.
|
||
Unfortunately, averaging many highly correlated quantities does not
|
||
lead to as large of a reduction in variance as averaging many
|
||
uncorrelated quantities. In particular, this means that bagging will
|
||
not lead to a substantial reduction in variance over a single tree in
|
||
this setting.</p>
|
||
<p>The algorithm described here can be applied to both classification and regression problems.</p>
|
||
<p>We will grow of forest of say <span class="math notranslate nohighlight">\(B\)</span> trees.</p>
|
||
<ol class="arabic simple">
|
||
<li><p>For <span class="math notranslate nohighlight">\(b=1:B\)</span></p></li>
|
||
</ol>
|
||
<ul class="simple">
|
||
<li><p>Draw a bootstrap sample from the training data organized in our <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> matrix.</p></li>
|
||
<li><p>We grow then a random forest tree <span class="math notranslate nohighlight">\(T_b\)</span> based on the bootstrapped data by repeating the steps outlined till we reach the maximum node size is reached</p></li>
|
||
</ul>
|
||
<ol class="arabic simple">
|
||
<li><p>we select <span class="math notranslate nohighlight">\(m \le p\)</span> variables at random from the <span class="math notranslate nohighlight">\(p\)</span> predictors/features</p></li>
|
||
<li><p>pick the best split point among the <span class="math notranslate nohighlight">\(m\)</span> features using for example the CART algorithm and create a new node</p></li>
|
||
<li><p>split the node into daughter nodes</p></li>
|
||
<li><p>Output then the ensemble of trees <span class="math notranslate nohighlight">\(\{T_b\}_1^{B}\)</span> and make predictions for either a regression type of problem or a classification type of problem.</p></li>
|
||
</ol>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import load_breast_cancer
|
||
from sklearn.svm import SVC
|
||
from sklearn.linear_model import LogisticRegression
|
||
from sklearn.tree import DecisionTreeClassifier
|
||
from sklearn.ensemble import BaggingClassifier
|
||
|
||
# Load the data
|
||
cancer = load_breast_cancer()
|
||
|
||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||
print(X_train.shape)
|
||
print(X_test.shape)
|
||
# Logistic Regression
|
||
logreg = LogisticRegression(solver='lbfgs')
|
||
logreg.fit(X_train, y_train)
|
||
print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
|
||
# Support vector machine
|
||
svm = SVC(gamma='auto', C=100)
|
||
svm.fit(X_train, y_train)
|
||
print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
|
||
# Decision Trees
|
||
deep_tree_clf = DecisionTreeClassifier(max_depth=None)
|
||
deep_tree_clf.fit(X_train, y_train)
|
||
print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
|
||
#now scale the data
|
||
from sklearn.preprocessing import StandardScaler
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
# Logistic Regression
|
||
logreg.fit(X_train_scaled, y_train)
|
||
print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||
# Support Vector Machine
|
||
svm.fit(X_train_scaled, y_train)
|
||
print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
|
||
# Decision Trees
|
||
deep_tree_clf.fit(X_train_scaled, y_train)
|
||
print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
|
||
|
||
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
from sklearn.preprocessing import LabelEncoder
|
||
from sklearn.model_selection import cross_validate
|
||
# Data set not specificied
|
||
#Instantiate the model with 500 trees and entropy as splitting criteria
|
||
Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
|
||
Random_Forest_model.fit(X_train_scaled, y_train)
|
||
#Cross validation
|
||
accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
|
||
print(accuracy)
|
||
print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
|
||
|
||
|
||
import scikitplot as skplt
|
||
y_pred = Random_Forest_model.predict(X_test_scaled)
|
||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||
plt.show()
|
||
y_probas = Random_Forest_model.predict_proba(X_test_scaled)
|
||
skplt.metrics.plot_roc(y_test, y_probas)
|
||
plt.show()
|
||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>Recall that the cumulative gains curve shows the percentage of the
|
||
overall number of cases in a given category <em>gained</em> by targeting a
|
||
percentage of the total number of cases.</p>
|
||
<p>Similarly, the receiver operating characteristic curve, or ROC curve,
|
||
displays the diagnostic ability of a binary classifier system as its
|
||
discrimination threshold is varied. It plots the true positive rate against the false positive rate.</p>
|
||
<section id="compare-bagging-on-trees-with-random-forests">
|
||
<h3><span class="section-number">10.4.1. </span>Compare Bagging on Trees with Random Forests<a class="headerlink" href="#compare-bagging-on-trees-with-random-forests" title="Link to this heading">#</a></h3>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>bag_clf = BaggingClassifier(
|
||
DecisionTreeClassifier(splitter="random", max_leaf_nodes=16, random_state=42),
|
||
n_estimators=500, max_samples=1.0, bootstrap=True, n_jobs=-1, random_state=42)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>bag_clf.fit(X_train, y_train)
|
||
y_pred = bag_clf.predict(X_test)
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
rnd_clf = RandomForestClassifier(n_estimators=500, max_leaf_nodes=16, n_jobs=-1, random_state=42)
|
||
rnd_clf.fit(X_train, y_train)
|
||
y_pred_rf = rnd_clf.predict(X_test)
|
||
np.sum(y_pred == y_pred_rf) / len(y_pred)
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="boosting-a-bird-s-eye-view">
|
||
<h2><span class="section-number">10.5. </span>Boosting, a Bird’s Eye View<a class="headerlink" href="#boosting-a-bird-s-eye-view" title="Link to this heading">#</a></h2>
|
||
<p>The basic idea is to combine weak classifiers in order to create a good
|
||
classifier. With a weak classifier we often intend a classifier which
|
||
produces results which are only slightly better than we would get by
|
||
random guesses.</p>
|
||
<p>This is done by applying in an iterative way a weak (or a standard
|
||
classifier like decision trees) to modify the data. In each iteration
|
||
we emphasize those observations which are misclassified by weighting
|
||
them with a factor.</p>
|
||
<p>Boosting is a way of fitting an additive expansion in a set of
|
||
elementary basis functions like for example some simple polynomials.
|
||
Assume for example that we have a function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(\beta_m\)</span> are the expansion parameters to be determined in a
|
||
minimization process and <span class="math notranslate nohighlight">\(b(x;\gamma_m)\)</span> are some simple functions of
|
||
the multivariable parameter <span class="math notranslate nohighlight">\(x\)</span> which is characterized by the
|
||
parameters <span class="math notranslate nohighlight">\(\gamma_m\)</span>.</p>
|
||
<p>As an example, consider the Sigmoid function we used in logistic
|
||
regression. In that case, we can translate the function
|
||
<span class="math notranslate nohighlight">\(b(x;\gamma_m)\)</span> into the Sigmoid function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\sigma(t) = \frac{1}{1+\exp{(-t)}},
|
||
\]</div>
|
||
<p>where <span class="math notranslate nohighlight">\(t=\gamma_0+\gamma_1 x\)</span> and the parameters <span class="math notranslate nohighlight">\(\gamma_0\)</span> and
|
||
<span class="math notranslate nohighlight">\(\gamma_1\)</span> were determined by the Logistic Regression fitting
|
||
algorithm.</p>
|
||
<p>As another example, consider the cost function we defined for linear regression</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||
\]</div>
|
||
<p>In this case the function <span class="math notranslate nohighlight">\(f(x)\)</span> was replaced by the design matrix
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span> and the unknown linear regression parameters <span class="math notranslate nohighlight">\(\boldsymbol{\beta}\)</span>,
|
||
that is <span class="math notranslate nohighlight">\(\boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta}\)</span>. In linear regression we can
|
||
simply invert a matrix and obtain the parameters <span class="math notranslate nohighlight">\(\beta\)</span> by</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
||
\]</div>
|
||
<p>In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters <span class="math notranslate nohighlight">\(\beta_m\)</span> and <span class="math notranslate nohighlight">\(\gamma_m\)</span>.</p>
|
||
<section id="iterative-fitting-regression-and-squared-error-cost-function">
|
||
<h3><span class="section-number">10.5.1. </span>Iterative Fitting, Regression and Squared-error Cost Function<a class="headerlink" href="#iterative-fitting-regression-and-squared-error-cost-function" title="Link to this heading">#</a></h3>
|
||
<p>The way we proceed is as follows (here we specialize to the squared-error cost function)</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Establish a cost function, here <span class="math notranslate nohighlight">\(\cal{C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2\)</span> with <span class="math notranslate nohighlight">\(f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m)\)</span>.</p></li>
|
||
<li><p>Initialize with a guess <span class="math notranslate nohighlight">\(f_0(x)\)</span>. It could be one or even zero or some random numbers.</p></li>
|
||
<li><p>For <span class="math notranslate nohighlight">\(m=1:M\)</span></p></li>
|
||
</ol>
|
||
<p>a. minimize <span class="math notranslate nohighlight">\(\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2\)</span> wrt <span class="math notranslate nohighlight">\(\gamma\)</span> and <span class="math notranslate nohighlight">\(\beta\)</span></p>
|
||
<p>b. This gives the optimal values <span class="math notranslate nohighlight">\(\beta_m\)</span> and <span class="math notranslate nohighlight">\(\gamma_m\)</span></p>
|
||
<p>c. Determine then the new values <span class="math notranslate nohighlight">\(f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m)\)</span></p>
|
||
<p>We could use any of the algorithms we have discussed till now. If we
|
||
use trees, <span class="math notranslate nohighlight">\(\gamma\)</span> parameterizes the split variables and split points
|
||
at the internal nodes, and the predictions at the terminal nodes.</p>
|
||
<p>To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.</p>
|
||
<p>For simplicity we assume also that our functions <span class="math notranslate nohighlight">\(b(x;\gamma)=1+\gamma x\)</span>.</p>
|
||
<p>This means that for every iteration <span class="math notranslate nohighlight">\(m\)</span>, we need to optimize</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2.
|
||
\]</div>
|
||
<p>We start our iteration by simply setting <span class="math notranslate nohighlight">\(f_0(x)=0\)</span>.
|
||
Taking the derivatives with respect to <span class="math notranslate nohighlight">\(\beta\)</span> and <span class="math notranslate nohighlight">\(\gamma\)</span> we obtain</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \cal{C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0,
|
||
\]</div>
|
||
<p>and</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\frac{\partial \cal{C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0.
|
||
\]</div>
|
||
<p>We can then rewrite these equations as (defining <span class="math notranslate nohighlight">\(\boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x})\)</span> with <span class="math notranslate nohighlight">\(\boldsymbol{e}\)</span> being the unit vector)</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0,
|
||
\]</div>
|
||
<p>which gives us <span class="math notranslate nohighlight">\(\beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w})\)</span>. Similarly we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0,
|
||
\]</div>
|
||
<p>which leads to <span class="math notranslate nohighlight">\(\gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x})\)</span>. Inserting
|
||
for <span class="math notranslate nohighlight">\(\beta\)</span> gives us an equation for <span class="math notranslate nohighlight">\(\gamma\)</span>. This is a non-linear equation in the unknown <span class="math notranslate nohighlight">\(\gamma\)</span> and has to be solved numerically.</p>
|
||
<p>The solution to these two equations gives us in turn <span class="math notranslate nohighlight">\(\beta_1\)</span> and <span class="math notranslate nohighlight">\(\gamma_1\)</span> leading to the new expression for <span class="math notranslate nohighlight">\(f_1(x)\)</span> as
|
||
<span class="math notranslate nohighlight">\(f_1(x) = \beta_1(1+\gamma_1x)\)</span>. Doing this <span class="math notranslate nohighlight">\(M\)</span> times results in our final estimate for the function <span class="math notranslate nohighlight">\(f\)</span>.</p>
|
||
</section>
|
||
<section id="iterative-fitting-classification-and-adaboost">
|
||
<h3><span class="section-number">10.5.2. </span>Iterative Fitting, Classification and AdaBoost<a class="headerlink" href="#iterative-fitting-classification-and-adaboost" title="Link to this heading">#</a></h3>
|
||
<p>Let us consider a binary classification problem with two outcomes <span class="math notranslate nohighlight">\(y_i \in \{-1,1\}\)</span> and <span class="math notranslate nohighlight">\(i=0,1,2,\dots,n-1\)</span> as our set of
|
||
observations. We define a classification function <span class="math notranslate nohighlight">\(G(x)\)</span> which produces a prediction taking one or the other of the two values
|
||
<span class="math notranslate nohighlight">\(\{-1,1\}\)</span>.</p>
|
||
<p>The error rate of the training sample is then</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)).
|
||
\]</div>
|
||
<p>The iterative procedure starts with defining a weak classifier whose
|
||
error rate is barely better than random guessing. The iterative
|
||
procedure in boosting is to sequentially apply a weak
|
||
classification algorithm to repeatedly modified versions of the data
|
||
producing a sequence of weak classifiers <span class="math notranslate nohighlight">\(G_m(x)\)</span>.</p>
|
||
<p>Here we will express our function <span class="math notranslate nohighlight">\(f(x)\)</span> in terms of <span class="math notranslate nohighlight">\(G(x)\)</span>. That is</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
||
\]</div>
|
||
<p>will be a function of</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x).
|
||
\]</div>
|
||
<p>In our iterative procedure we define thus</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f_m(x) = f_{m-1}(x)+\beta_mG_m(x).
|
||
\]</div>
|
||
<p>The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the
|
||
exponential cost/loss function defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}.
|
||
\]</div>
|
||
<p>We optimize <span class="math notranslate nohighlight">\(\beta\)</span> and <span class="math notranslate nohighlight">\(G\)</span> for each value of <span class="math notranslate nohighlight">\(m=1:M\)</span> as we did in the regression case.
|
||
This is normally done in two steps. Let us however first rewrite the cost function as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))},
|
||
\]</div>
|
||
<p>where we have defined <span class="math notranslate nohighlight">\(w_i^m= \exp{(-y_if_{m-1}(x_i))}\)</span>.</p>
|
||
<p>First, for any <span class="math notranslate nohighlight">\(\beta > 0\)</span>, we optimize <span class="math notranslate nohighlight">\(G\)</span> by setting</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)),
|
||
\]</div>
|
||
<p>which is the classifier that minimizes the weighted error rate in predicting <span class="math notranslate nohighlight">\(y\)</span>.</p>
|
||
<p>We can do this by rewriting</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m,
|
||
\]</div>
|
||
<p>which can be rewritten as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0,
|
||
\]</div>
|
||
<p>which leads to</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}},
|
||
\]</div>
|
||
<p>where we have redefined the error as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
|
||
\]</div>
|
||
<p>which leads to an update of</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f_m(x) = f_{m-1}(x) +\beta_m G_m(x).
|
||
\]</div>
|
||
<p>This leads to the new weights</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))}
|
||
\]</div>
|
||
</section>
|
||
<section id="adaptive-boosting-adaboost-basic-algorithm">
|
||
<h3><span class="section-number">10.5.3. </span>Adaptive boosting: AdaBoost, Basic Algorithm<a class="headerlink" href="#adaptive-boosting-adaboost-basic-algorithm" title="Link to this heading">#</a></h3>
|
||
<p>The algorithm here is rather straightforward. Assume that our weak
|
||
classifier is a decision tree and we consider a binary set of outputs
|
||
with <span class="math notranslate nohighlight">\(y_i \in \{-1,1\}\)</span> and <span class="math notranslate nohighlight">\(i=0,1,2,\dots,n-1\)</span> as our set of
|
||
observations. Our design matrix is given in terms of the
|
||
feature/predictor vectors
|
||
<span class="math notranslate nohighlight">\(\boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}]\)</span>. Finally, we define also a
|
||
classifier determined by our data via a function <span class="math notranslate nohighlight">\(G(x)\)</span>. This function tells us how well we are able to classify our outputs/targets <span class="math notranslate nohighlight">\(\boldsymbol{y}\)</span>.</p>
|
||
<p>We have already defined the misclassification error <span class="math notranslate nohighlight">\(\mathrm{err}\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)),
|
||
\]</div>
|
||
<p>where the function <span class="math notranslate nohighlight">\(I()\)</span> is one if we misclassify and zero if we classify correctly.</p>
|
||
<p>With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||
The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.</p>
|
||
<ol class="arabic simple">
|
||
<li><p>We start by initializing all weights to <span class="math notranslate nohighlight">\(w_i = 1/n\)</span>, with <span class="math notranslate nohighlight">\(i=0,1,2,\dots n-1\)</span>. It is easy to see that we must have <span class="math notranslate nohighlight">\(\sum_{i=0}^{n-1}w_i = 1\)</span>.</p></li>
|
||
<li><p>We rewrite the misclassification error as</p></li>
|
||
</ol>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
|
||
\]</div>
|
||
<ol class="arabic simple">
|
||
<li><p>Then we start looping over all attempts at classifying, namely we start an iterative process for <span class="math notranslate nohighlight">\(m=1:M\)</span>, where <span class="math notranslate nohighlight">\(M\)</span> is the final number of classifications. Our given classifier could for example be a plain decision tree.</p></li>
|
||
</ol>
|
||
<p>a. Fit then a given classifier to the training set using the weights <span class="math notranslate nohighlight">\(w_i\)</span>.</p>
|
||
<p>b. Compute then <span class="math notranslate nohighlight">\(\mathrm{err}\)</span> and figure out which events are classified properly and which are classified wrongly.</p>
|
||
<p>c. Define a quantity <span class="math notranslate nohighlight">\(\alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m}\)</span></p>
|
||
<p>d. Set the new weights to <span class="math notranslate nohighlight">\(w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)}\)</span>.</p>
|
||
<ol class="arabic simple" start="5">
|
||
<li><p>Compute the new classifier <span class="math notranslate nohighlight">\(G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i)\)</span>.</p></li>
|
||
</ol>
|
||
<p>For the iterations with <span class="math notranslate nohighlight">\(m \le 2\)</span> the weights are modified
|
||
individually at each steps. The observations which were misclassified
|
||
at iteration <span class="math notranslate nohighlight">\(m-1\)</span> have a weight which is larger than those which were
|
||
classified properly. As this proceeds, the observations which were
|
||
difficult to classifiy correctly are given a larger influence. Each
|
||
new classification step <span class="math notranslate nohighlight">\(m\)</span> is then forced to concentrate on those
|
||
observations that are missed in the previous iterations.</p>
|
||
<p>Using <strong>Scikit-Learn</strong> it is easy to apply the adaptive boosting algorithm, as done here.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>from sklearn.ensemble import AdaBoostClassifier
|
||
|
||
ada_clf = AdaBoostClassifier(
|
||
DecisionTreeClassifier(max_depth=1), n_estimators=200,
|
||
algorithm="SAMME.R", learning_rate=0.5, random_state=42)
|
||
ada_clf.fit(X_train, y_train)
|
||
|
||
from sklearn.ensemble import AdaBoostClassifier
|
||
|
||
ada_clf = AdaBoostClassifier(
|
||
DecisionTreeClassifier(max_depth=1), n_estimators=200,
|
||
algorithm="SAMME.R", learning_rate=0.5, random_state=42)
|
||
ada_clf.fit(X_train_scaled, y_train)
|
||
y_pred = ada_clf.predict(X_test_scaled)
|
||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||
plt.show()
|
||
y_probas = ada_clf.predict_proba(X_test_scaled)
|
||
skplt.metrics.plot_roc(y_test, y_probas)
|
||
plt.show()
|
||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="gradient-boosting-basics-with-steepest-descent-functional-gradient-descent">
|
||
<h2><span class="section-number">10.6. </span>Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent<a class="headerlink" href="#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent" title="Link to this heading">#</a></h2>
|
||
<p>Gradient boosting is again a similar technique to Adaptive boosting,
|
||
it combines so-called weak classifiers or regressors into a strong
|
||
method via a series of iterations.</p>
|
||
<p>In order to understand the method, let us illustrate its basics by
|
||
bringing back the essential steps in linear regression, where our cost
|
||
function was the least squares function.</p>
|
||
<p>We start again with our cost function <span class="math notranslate nohighlight">\(\cal{C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}\cal{L}(y_i, f(x_i))\)</span> where we want to minimize
|
||
This means that for every iteration, we need to optimize</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||
\]</div>
|
||
<p>We define a real function <span class="math notranslate nohighlight">\(h_m(x)\)</span> that defines our final function <span class="math notranslate nohighlight">\(f_M(x)\)</span> as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f_M(x) = \sum_{m=0}^M h_m(x).
|
||
\]</div>
|
||
<p>In the steepest decent approach we approximate <span class="math notranslate nohighlight">\(h_m(x) = -\rho_m g_m(x)\)</span>, where <span class="math notranslate nohighlight">\(\rho_m\)</span> is a scalar and <span class="math notranslate nohighlight">\(g_m(x)\)</span> the gradient defined as</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
g_m(x_i) = \left[ \frac{\partial \cal{L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}.
|
||
\]</div>
|
||
<p>With the new gradient we can update <span class="math notranslate nohighlight">\(f_m(x) = f_{m-1}(x) -\rho_m g_m(x)\)</span>. Using the above squared-error function we see that
|
||
the gradient is <span class="math notranslate nohighlight">\(g_m(x_i) = -2(y_i-f(x_i))\)</span>.</p>
|
||
<p>Choosing <span class="math notranslate nohighlight">\(f_0(x)=0\)</span> we obtain <span class="math notranslate nohighlight">\(g_m(x) = -2y_i\)</span> and inserting this into the minimization problem for the cost function we have</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2.
|
||
\]</div>
|
||
<p>Optimizing with respect to <span class="math notranslate nohighlight">\(\rho\)</span> we obtain (taking the derivative) that <span class="math notranslate nohighlight">\(\rho_1 = -1/2\)</span>. We have then that</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i.
|
||
\]</div>
|
||
<p>We can then proceed and compute</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
g_2(x_i) = \left[ \frac{\partial \cal{L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i,
|
||
\]</div>
|
||
<p>and find a new value for <span class="math notranslate nohighlight">\(\rho_2=-1/2\)</span> and continue till we have reached <span class="math notranslate nohighlight">\(m=M\)</span>. We can modify the steepest descent method, or steepest boosting, by introducing what is called <strong>gradient boosting</strong>.</p>
|
||
<p>Steepest descent is however not much used, since it only optimizes <span class="math notranslate nohighlight">\(f\)</span> at a fixed set of <span class="math notranslate nohighlight">\(n\)</span> points,
|
||
so we do not learn a function that can generalize. However, we can modify the algorithm by
|
||
fitting a weak learner to approximate the negative gradient signal.</p>
|
||
<p>Suppose we have a cost function <span class="math notranslate nohighlight">\(C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i))\)</span> where <span class="math notranslate nohighlight">\(y_i\)</span> is our target and <span class="math notranslate nohighlight">\(f(x_i)\)</span> the function which is meant to model <span class="math notranslate nohighlight">\(y_i\)</span>. The above cost function could be our standard squared-error function</p>
|
||
<div class="math notranslate nohighlight">
|
||
\[
|
||
C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
||
\]</div>
|
||
<p>The way we proceed in an iterative fashion is to</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Initialize our estimate <span class="math notranslate nohighlight">\(f_0(x)\)</span>.</p></li>
|
||
<li><p>For <span class="math notranslate nohighlight">\(m=1:M\)</span>, we</p></li>
|
||
</ol>
|
||
<p>a. compute the negative gradient vector <span class="math notranslate nohighlight">\(\boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x)\)</span> at <span class="math notranslate nohighlight">\(f(x) = f_{m-1}(x)\)</span>;</p>
|
||
<p>b. fit the so-called base-learner to the negative gradient <span class="math notranslate nohighlight">\(h_m(u_m,x)\)</span>;</p>
|
||
<p>c. update the estimate <span class="math notranslate nohighlight">\(f_m(x) = f_{m-1}(x)+h_m(u_m,x)\)</span>;</p>
|
||
<ol class="arabic simple" start="4">
|
||
<li><p>The final estimate is then <span class="math notranslate nohighlight">\(f_M(x) = \sum_{m=1}^M h_m(u_m,x)\)</span>.</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="gradient-boosting-examples-of-regression">
|
||
<h2><span class="section-number">10.7. </span>Gradient Boosting, Examples of Regression<a class="headerlink" href="#gradient-boosting-examples-of-regression" title="Link to this heading">#</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.ensemble import GradientBoostingRegressor
|
||
from sklearn.preprocessing import StandardScaler
|
||
import scikitplot as skplt
|
||
from sklearn.metrics import mean_squared_error
|
||
|
||
n = 100
|
||
maxdegree = 6
|
||
|
||
# Make data set.
|
||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||
|
||
error = np.zeros(maxdegree)
|
||
bias = np.zeros(maxdegree)
|
||
variance = np.zeros(maxdegree)
|
||
polydegree = np.zeros(maxdegree)
|
||
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
|
||
for degree in range(1,maxdegree):
|
||
model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)
|
||
model.fit(X_train_scaled,y_train)
|
||
y_pred = model.predict(X_test_scaled)
|
||
polydegree[degree] = degree
|
||
error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
|
||
bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
|
||
variance[degree] = np.mean( np.var(y_pred) )
|
||
print('Max depth:', degree)
|
||
print('Error:', error[degree])
|
||
print('Bias^2:', bias[degree])
|
||
print('Var:', variance[degree])
|
||
print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
|
||
|
||
plt.xlim(1,maxdegree-1)
|
||
plt.plot(polydegree, error, label='Error')
|
||
plt.plot(polydegree, bias, label='bias')
|
||
plt.plot(polydegree, variance, label='Variance')
|
||
plt.legend()
|
||
save_fig("gdregression")
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="gradient-boosting-classification-example">
|
||
<h2><span class="section-number">10.8. </span>Gradient Boosting, Classification Example<a class="headerlink" href="#gradient-boosting-classification-example" title="Link to this heading">#</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import load_breast_cancer
|
||
import scikitplot as skplt
|
||
from sklearn.ensemble import GradientBoostingClassifier
|
||
from sklearn.model_selection import cross_validate
|
||
|
||
# Load the data
|
||
cancer = load_breast_cancer()
|
||
|
||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||
print(X_train.shape)
|
||
print(X_test.shape)
|
||
#now scale the data
|
||
from sklearn.preprocessing import StandardScaler
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
|
||
gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)
|
||
gd_clf.fit(X_train_scaled, y_train)
|
||
#Cross validation
|
||
accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
|
||
print(accuracy)
|
||
print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
|
||
|
||
import scikitplot as skplt
|
||
y_pred = gd_clf.predict(X_test_scaled)
|
||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||
save_fig("gdclassiffierconfusion")
|
||
plt.show()
|
||
y_probas = gd_clf.predict_proba(X_test_scaled)
|
||
skplt.metrics.plot_roc(y_test, y_probas)
|
||
save_fig("gdclassiffierroc")
|
||
plt.show()
|
||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||
save_fig("gdclassiffiercgain")
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<section id="xgboost-extreme-gradient-boosting">
|
||
<h2><span class="section-number">10.9. </span>XGBoost: Extreme Gradient Boosting<a class="headerlink" href="#xgboost-extreme-gradient-boosting" title="Link to this heading">#</a></h2>
|
||
<p><a class="reference external" href="https://github.com/dmlc/xgboost">XGBoost</a> or Extreme Gradient
|
||
Boosting, is an optimized distributed gradient boosting library
|
||
designed to be highly efficient, flexible and portable. It implements
|
||
machine learning algorithms under the Gradient Boosting
|
||
framework. XGBoost provides a parallel tree boosting that solve many
|
||
data science problems in a fast and accurate way. See the <a class="reference external" href="https://arxiv.org/abs/1603.02754">article by Chen and Guestrin</a>.</p>
|
||
<p>The authors design and build a highly scalable end-to-end tree
|
||
boosting system. It has a theoretically justified weighted quantile
|
||
sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning.</p>
|
||
<p>It is now the algorithm which wins essentially all ML competitions!!!</p>
|
||
</section>
|
||
<section id="regression-case">
|
||
<h2><span class="section-number">10.10. </span>Regression Case<a class="headerlink" href="#regression-case" title="Link to this heading">#</a></h2>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
import xgboost as xgb
|
||
from sklearn.preprocessing import StandardScaler
|
||
import scikitplot as skplt
|
||
from sklearn.metrics import mean_squared_error
|
||
|
||
n = 100
|
||
maxdegree = 6
|
||
|
||
# Make data set.
|
||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||
|
||
error = np.zeros(maxdegree)
|
||
bias = np.zeros(maxdegree)
|
||
variance = np.zeros(maxdegree)
|
||
polydegree = np.zeros(maxdegree)
|
||
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
|
||
for degree in range(maxdegree):
|
||
model = xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)
|
||
|
||
model.fit(X_train_scaled,y_train)
|
||
y_pred = model.predict(X_test_scaled)
|
||
polydegree[degree] = degree
|
||
error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
|
||
bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
|
||
variance[degree] = np.mean( np.var(y_pred) )
|
||
print('Max depth:', degree)
|
||
print('Error:', error[degree])
|
||
print('Bias^2:', bias[degree])
|
||
print('Var:', variance[degree])
|
||
print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
|
||
|
||
plt.xlim(1,maxdegree-1)
|
||
plt.plot(polydegree, error, label='Error')
|
||
plt.plot(polydegree, bias, label='bias')
|
||
plt.plot(polydegree, variance, label='Variance')
|
||
plt.legend()
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<p>As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.</p>
|
||
<div class="cell docutils container">
|
||
<div class="cell_input docutils container">
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>
|
||
import matplotlib.pyplot as plt
|
||
import numpy as np
|
||
from sklearn.model_selection import train_test_split
|
||
from sklearn.datasets import load_breast_cancer
|
||
from sklearn.preprocessing import LabelEncoder
|
||
from sklearn.model_selection import cross_validate
|
||
import scikitplot as skplt
|
||
import xgboost as xgb
|
||
# Load the data
|
||
cancer = load_breast_cancer()
|
||
|
||
X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
|
||
print(X_train.shape)
|
||
print(X_test.shape)
|
||
#now scale the data
|
||
from sklearn.preprocessing import StandardScaler
|
||
scaler = StandardScaler()
|
||
scaler.fit(X_train)
|
||
X_train_scaled = scaler.transform(X_train)
|
||
X_test_scaled = scaler.transform(X_test)
|
||
|
||
xg_clf = xgb.XGBClassifier()
|
||
xg_clf.fit(X_train_scaled,y_train)
|
||
|
||
y_test = xg_clf.predict(X_test_scaled)
|
||
|
||
print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
|
||
|
||
import scikitplot as skplt
|
||
y_pred = xg_clf.predict(X_test_scaled)
|
||
skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
|
||
save_fig("xdclassiffierconfusion")
|
||
plt.show()
|
||
y_probas = xg_clf.predict_proba(X_test_scaled)
|
||
skplt.metrics.plot_roc(y_test, y_probas)
|
||
save_fig("xdclassiffierroc")
|
||
plt.show()
|
||
skplt.metrics.plot_cumulative_gain(y_test, y_probas)
|
||
save_fig("gdclassiffiercgain")
|
||
plt.show()
|
||
|
||
|
||
xgb.plot_tree(xg_clf,num_trees=0)
|
||
plt.rcParams['figure.figsize'] = [50, 10]
|
||
save_fig("xgtree")
|
||
plt.show()
|
||
|
||
xgb.plot_importance(xg_clf)
|
||
plt.rcParams['figure.figsize'] = [5, 5]
|
||
save_fig("xgparams")
|
||
plt.show()
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
<script type="text/x-thebe-config">
|
||
{
|
||
requestKernel: true,
|
||
binderOptions: {
|
||
repo: "binder-examples/jupyter-stacks-datascience",
|
||
ref: "master",
|
||
},
|
||
codeMirrorConfig: {
|
||
theme: "abcdef",
|
||
mode: "python"
|
||
},
|
||
kernelOptions: {
|
||
name: "python3",
|
||
path: "./."
|
||
},
|
||
predefinedOutput: true
|
||
}
|
||
</script>
|
||
<script>kernelName = 'python3'</script>
|
||
|
||
</article>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<footer class="prev-next-footer d-print-none">
|
||
|
||
<div class="prev-next-area">
|
||
<a class="left-prev"
|
||
href="chapter6.html"
|
||
title="previous page">
|
||
<i class="fa-solid fa-angle-left"></i>
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">previous</p>
|
||
<p class="prev-next-title"><span class="section-number">9. </span>Decision trees, overarching aims</p>
|
||
</div>
|
||
</a>
|
||
<a class="right-next"
|
||
href="chapter8.html"
|
||
title="next page">
|
||
<div class="prev-next-info">
|
||
<p class="prev-next-subtitle">next</p>
|
||
<p class="prev-next-title"><span class="section-number">11. </span>Basic ideas of the Principal Component Analysis (PCA)</p>
|
||
</div>
|
||
<i class="fa-solid fa-angle-right"></i>
|
||
</a>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
|
||
|
||
|
||
<div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">
|
||
|
||
|
||
<div class="sidebar-secondary-item">
|
||
<div class="page-toc tocsection onthispage">
|
||
<i class="fa-solid fa-list"></i> Contents
|
||
</div>
|
||
<nav class="bd-toc-nav page-toc">
|
||
<ul class="visible nav section-nav flex-column">
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#an-overview-of-ensemble-methods">10.1. An Overview of Ensemble Methods</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#bagging">10.2. Bagging</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#bagging-examples">10.3. Bagging Examples</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#making-your-own-bootstrap-changing-the-level-of-the-decision-tree">10.3.1. Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#random-forests">10.4. Random forests</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#compare-bagging-on-trees-with-random-forests">10.4.1. Compare Bagging on Trees with Random Forests</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#boosting-a-bird-s-eye-view">10.5. Boosting, a Bird’s Eye View</a><ul class="nav section-nav flex-column">
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#iterative-fitting-regression-and-squared-error-cost-function">10.5.1. Iterative Fitting, Regression and Squared-error Cost Function</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#iterative-fitting-classification-and-adaboost">10.5.2. Iterative Fitting, Classification and AdaBoost</a></li>
|
||
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#adaptive-boosting-adaboost-basic-algorithm">10.5.3. Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gradient-boosting-basics-with-steepest-descent-functional-gradient-descent">10.6. Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gradient-boosting-examples-of-regression">10.7. Gradient Boosting, Examples of Regression</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#gradient-boosting-classification-example">10.8. Gradient Boosting, Classification Example</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#xgboost-extreme-gradient-boosting">10.9. XGBoost: Extreme Gradient Boosting</a></li>
|
||
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#regression-case">10.10. Regression Case</a></li>
|
||
</ul>
|
||
</nav></div>
|
||
|
||
</div></div>
|
||
|
||
|
||
</div>
|
||
<footer class="bd-footer-content">
|
||
|
||
<div class="bd-footer-content__inner container">
|
||
|
||
<div class="footer-item">
|
||
|
||
<p class="component-author">
|
||
By Morten Hjorth-Jensen
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
|
||
<p class="copyright">
|
||
|
||
© Copyright 2023.
|
||
<br/>
|
||
|
||
</p>
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
<div class="footer-item">
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</footer>
|
||
|
||
|
||
</main>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Scripts loaded after <body> so the DOM is not blocked -->
|
||
<script src="_static/scripts/bootstrap.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
<script src="_static/scripts/pydata-sphinx-theme.js?digest=dfe6caa3a7d634c4db9b"></script>
|
||
|
||
<footer class="bd-footer">
|
||
</footer>
|
||
</body>
|
||
</html> |