309 lines
19 KiB
HTML
309 lines
19 KiB
HTML
<!--
|
|
Automatically generated HTML file from DocOnce source
|
|
(https://github.com/hplgit/doconce/)
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/hplgit/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Data Analysis and Machine Learning: From Decision Trees to Forests and all that">
|
|
|
|
<title>Data Analysis and Machine Learning: From Decision Trees to Forests and all that</title>
|
|
|
|
<!-- Bootstrap style: bootstrap -->
|
|
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
|
|
<!-- not necessary
|
|
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
|
|
-->
|
|
|
|
<style type="text/css">
|
|
|
|
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
|
|
.dropdown-menu {
|
|
height: auto;
|
|
max-height: 400px;
|
|
overflow-x: hidden;
|
|
}
|
|
|
|
/* Adds an invisible element before each target to offset for the navigation
|
|
bar */
|
|
.anchor::before {
|
|
content:"";
|
|
display:block;
|
|
height:50px; /* fixed header height for style bootstrap */
|
|
margin:-50px 0 0; /* negative fixed header height */
|
|
}
|
|
</style>
|
|
|
|
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Decision trees, overarching aims', 2, None, '___sec0'),
|
|
('A typical Decision Tree with its pertinent Jargon, '
|
|
'Classification Problem',
|
|
2,
|
|
None,
|
|
'___sec1'),
|
|
('General Features', 2, None, '___sec2'),
|
|
('How do we set it up?', 2, None, '___sec3'),
|
|
('Decision trees and Regression', 2, None, '___sec4'),
|
|
('Building a tree, regression', 2, None, '___sec5'),
|
|
('A top-down approach, recursive binary splitting',
|
|
2,
|
|
None,
|
|
'___sec6'),
|
|
('Making a tree', 2, None, '___sec7'),
|
|
('Pruning the tree', 2, None, '___sec8'),
|
|
('Cost complexity pruning', 2, None, '___sec9'),
|
|
('Schematic Regression Procedure', 2, None, '___sec10'),
|
|
('A Classification Tree', 2, None, '___sec11'),
|
|
('Growing a classification tree', 2, None, '___sec12'),
|
|
('Classification tree, how to split nodes', 2, None, '___sec13'),
|
|
('Visualizing the Tree, Classification', 2, None, '___sec14'),
|
|
('Visualizing the Tree, The Moons', 2, None, '___sec15'),
|
|
('Algorithms for Setting up Decision Trees', 2, None, '___sec16'),
|
|
('The CART algorithm for Classification', 2, None, '___sec17'),
|
|
('The CART algorithm for Regression', 2, None, '___sec18'),
|
|
('Computing the Gini index', 2, None, '___sec19'),
|
|
('Simple Python Code to read in Data and perform Classification',
|
|
2,
|
|
None,
|
|
'___sec20'),
|
|
('Computing the Gini Factor', 2, None, '___sec21'),
|
|
('Entropy and the ID3 algorithm', 2, None, '___sec22'),
|
|
('Implementing the ID3 Algorithm', 2, None, '___sec23'),
|
|
('Cancer Data again now with Decision Trees and other Methods',
|
|
2,
|
|
None,
|
|
'___sec24'),
|
|
('Another example, the moons again', 2, None, '___sec25'),
|
|
('Playing around with regions', 2, None, '___sec26'),
|
|
('Regression trees', 2, None, '___sec27'),
|
|
('Final regressor code', 2, None, '___sec28'),
|
|
('Pros and cons of trees, pros', 2, None, '___sec29'),
|
|
('Disadvantages', 2, None, '___sec30'),
|
|
('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
|
|
'Boosting, Meet the Jungle of Methods',
|
|
2,
|
|
None,
|
|
'___sec31'),
|
|
('An Overview of Ensemble Methods', 2, None, '___sec32'),
|
|
('Bagging', 2, None, '___sec33'),
|
|
('More bagging', 2, None, '___sec34'),
|
|
('Simple Voting Example, head or tail', 2, None, '___sec35'),
|
|
('Using the Voting Classifier', 2, None, '___sec36'),
|
|
('Please, not the moons again! Voting and Bagging',
|
|
2,
|
|
None,
|
|
'___sec37'),
|
|
('Now Bagging', 2, None, '___sec38'),
|
|
('Making our own Bagging with Bootstrap', 2, None, '___sec39'),
|
|
('Changing the Level of the Decision Tree', 2, None, '___sec40'),
|
|
('Random forests', 2, None, '___sec41'),
|
|
('Random Forest Algorithm', 2, None, '___sec42'),
|
|
('Bootstrap with Random Forests Instead of a Single Tree, own '
|
|
'Bagging',
|
|
2,
|
|
None,
|
|
'___sec43'),
|
|
('Random Forests Compared with other Methods on the Cancer Data',
|
|
2,
|
|
None,
|
|
'___sec44'),
|
|
('Compare Bagging on Trees with Random Forests',
|
|
2,
|
|
None,
|
|
'___sec45'),
|
|
("Boosting, a Bird'e Eye", 2, None, '___sec46'),
|
|
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
|
2,
|
|
None,
|
|
'___sec47'),
|
|
('Basic Steps of AdaBoost', 2, None, '___sec48'),
|
|
('AdaBoost Examples', 2, None, '___sec49'),
|
|
('Gradient boosting: Basics', 2, None, '___sec50'),
|
|
('Gradient Boosting, algorithm', 2, None, '___sec51'),
|
|
('Gradient Boosting, Examples', 2, None, '___sec52'),
|
|
('Gradient Boots with Early Stopping', 2, None, '___sec53'),
|
|
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec54'),
|
|
('Regression Case', 2, None, '___sec55'),
|
|
('Xgboost on the Cancer Data', 2, None, '___sec56')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "none" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
|
|
|
|
<!-- Bootstrap navigation bar -->
|
|
<div class="navbar navbar-default navbar-fixed-top">
|
|
<div class="navbar-header">
|
|
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
<span class="icon-bar"></span>
|
|
</button>
|
|
<a class="navbar-brand" href="DecisionTrees-bs.html">Data Analysis and Machine Learning: From Decision Trees to Forests and all that</a>
|
|
</div>
|
|
|
|
<div class="navbar-collapse collapse navbar-responsive-collapse">
|
|
<ul class="nav navbar-nav navbar-right">
|
|
<li class="dropdown">
|
|
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
|
|
<ul class="dropdown-menu">
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs001.html#___sec0" style="font-size: 80%;">Decision trees, overarching aims</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs002.html#___sec1" style="font-size: 80%;">A typical Decision Tree with its pertinent Jargon, Classification Problem</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs003.html#___sec2" style="font-size: 80%;">General Features</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs004.html#___sec3" style="font-size: 80%;">How do we set it up?</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs005.html#___sec4" style="font-size: 80%;">Decision trees and Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs006.html#___sec5" style="font-size: 80%;">Building a tree, regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs007.html#___sec6" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs008.html#___sec7" style="font-size: 80%;">Making a tree</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs009.html#___sec8" style="font-size: 80%;">Pruning the tree</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs010.html#___sec9" style="font-size: 80%;">Cost complexity pruning</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs011.html#___sec10" style="font-size: 80%;">Schematic Regression Procedure</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs012.html#___sec11" style="font-size: 80%;">A Classification Tree</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs013.html#___sec12" style="font-size: 80%;">Growing a classification tree</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs014.html#___sec13" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs015.html#___sec14" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs016.html#___sec15" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs017.html#___sec16" style="font-size: 80%;">Algorithms for Setting up Decision Trees</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs018.html#___sec17" style="font-size: 80%;">The CART algorithm for Classification</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs019.html#___sec18" style="font-size: 80%;">The CART algorithm for Regression</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs020.html#___sec19" style="font-size: 80%;">Computing the Gini index</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs021.html#___sec20" style="font-size: 80%;">Simple Python Code to read in Data and perform Classification</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs022.html#___sec21" style="font-size: 80%;">Computing the Gini Factor</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs023.html#___sec22" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs024.html#___sec23" style="font-size: 80%;">Implementing the ID3 Algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs025.html#___sec24" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs026.html#___sec25" style="font-size: 80%;">Another example, the moons again</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs027.html#___sec26" style="font-size: 80%;">Playing around with regions</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs028.html#___sec27" style="font-size: 80%;">Regression trees</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs029.html#___sec28" style="font-size: 80%;">Final regressor code</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs030.html#___sec29" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
|
|
<!-- navigation toc: --> <li><a href="#___sec30" style="font-size: 80%;">Disadvantages</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs032.html#___sec31" style="font-size: 80%;">Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs033.html#___sec32" style="font-size: 80%;">An Overview of Ensemble Methods</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs034.html#___sec33" style="font-size: 80%;">Bagging</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs035.html#___sec34" style="font-size: 80%;">More bagging</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs036.html#___sec35" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs037.html#___sec36" style="font-size: 80%;">Using the Voting Classifier</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs038.html#___sec37" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs039.html#___sec38" style="font-size: 80%;">Now Bagging</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs040.html#___sec39" style="font-size: 80%;">Making our own Bagging with Bootstrap</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs041.html#___sec40" style="font-size: 80%;">Changing the Level of the Decision Tree</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs042.html#___sec41" style="font-size: 80%;">Random forests</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs043.html#___sec42" style="font-size: 80%;">Random Forest Algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs044.html#___sec43" style="font-size: 80%;">Bootstrap with Random Forests Instead of a Single Tree, own Bagging</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs045.html#___sec44" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs046.html#___sec45" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs047.html#___sec46" style="font-size: 80%;">Boosting, a Bird'e Eye</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs048.html#___sec47" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs049.html#___sec48" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs050.html#___sec49" style="font-size: 80%;">AdaBoost Examples</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs051.html#___sec50" style="font-size: 80%;">Gradient boosting: Basics</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs052.html#___sec51" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs053.html#___sec52" style="font-size: 80%;">Gradient Boosting, Examples</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs054.html#___sec53" style="font-size: 80%;">Gradient Boots with Early Stopping</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs055.html#___sec54" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs056.html#___sec55" style="font-size: 80%;">Regression Case</a></li>
|
|
<!-- navigation toc: --> <li><a href="._DecisionTrees-bs057.html#___sec56" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
|
|
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</div> <!-- end of navigation bar -->
|
|
|
|
<div class="container">
|
|
|
|
<p> </p><p> </p><p> </p> <!-- add vertical space -->
|
|
|
|
<a name="part0031"></a>
|
|
<!-- !split -->
|
|
|
|
<h2 id="___sec30" class="anchor">Disadvantages </h2>
|
|
|
|
<ul>
|
|
<li> Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches</li>
|
|
<li> If continuous features are used the tree may become quite large and hence less interpretable</li>
|
|
<li> Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented</li>
|
|
<li> Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests</li>
|
|
<li> Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones.</li>
|
|
<li> If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data</li>
|
|
<li> Features with many levels may be preferred over features with less levels since for them it is <em>more easy</em> to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain</li>
|
|
</ul>
|
|
|
|
However, by aggregating many decision trees, using methods like bagging, random forests, and boosting, the predictive performance of trees can be substantially improved.
|
|
|
|
<p>
|
|
<p>
|
|
<!-- navigation buttons at the bottom of the page -->
|
|
<ul class="pagination">
|
|
<li><a href="._DecisionTrees-bs030.html">«</a></li>
|
|
<li><a href="._DecisionTrees-bs000.html">1</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._DecisionTrees-bs023.html">24</a></li>
|
|
<li><a href="._DecisionTrees-bs024.html">25</a></li>
|
|
<li><a href="._DecisionTrees-bs025.html">26</a></li>
|
|
<li><a href="._DecisionTrees-bs026.html">27</a></li>
|
|
<li><a href="._DecisionTrees-bs027.html">28</a></li>
|
|
<li><a href="._DecisionTrees-bs028.html">29</a></li>
|
|
<li><a href="._DecisionTrees-bs029.html">30</a></li>
|
|
<li><a href="._DecisionTrees-bs030.html">31</a></li>
|
|
<li class="active"><a href="._DecisionTrees-bs031.html">32</a></li>
|
|
<li><a href="._DecisionTrees-bs032.html">33</a></li>
|
|
<li><a href="._DecisionTrees-bs033.html">34</a></li>
|
|
<li><a href="._DecisionTrees-bs034.html">35</a></li>
|
|
<li><a href="._DecisionTrees-bs035.html">36</a></li>
|
|
<li><a href="._DecisionTrees-bs036.html">37</a></li>
|
|
<li><a href="._DecisionTrees-bs037.html">38</a></li>
|
|
<li><a href="._DecisionTrees-bs038.html">39</a></li>
|
|
<li><a href="._DecisionTrees-bs039.html">40</a></li>
|
|
<li><a href="._DecisionTrees-bs040.html">41</a></li>
|
|
<li><a href="">...</a></li>
|
|
<li><a href="._DecisionTrees-bs057.html">58</a></li>
|
|
<li><a href="._DecisionTrees-bs032.html">»</a></li>
|
|
</ul>
|
|
<!-- ------------------- end of main content --------------- -->
|
|
|
|
</div> <!-- end container -->
|
|
<!-- include javascript, jQuery *first* -->
|
|
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
|
|
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
|
|
|
|
<!-- Bootstrap footer
|
|
<footer>
|
|
<a href="http://..."><img width="250" align=right src="http://..."></a>
|
|
</footer>
|
|
-->
|
|
|
|
|
|
<center style="font-size:80%">
|
|
<!-- copyright only on the titlepage -->
|
|
</center>
|
|
|
|
|
|
</body>
|
|
</html>
|
|
|
|
|