Files
FYS-STK4155/doc/pub/week44/html/._week44-bs059.html
T
Morten Hjorth-Jensen 4565722caf small addition
2021-11-05 06:31:22 +01:00

396 lines
23 KiB
HTML

<!--
HTML file automatically generated from DocOnce source
(https://github.com/doconce/doconce/)
doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week44-bs --no_mako
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="description" content="Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees">
<title>Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</title>
<!-- Bootstrap style: bootstrap -->
<!-- doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=week44-bs --no_mako -->
<link href="https://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css" rel="stylesheet">
<!-- not necessary
<link href="https://netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
-->
<style type="text/css">
/* Add scrollbar to dropdown menus in bootstrap navigation bar */
.dropdown-menu {
height: auto;
max-height: 400px;
overflow-x: hidden;
}
/* Adds an invisible element before each target to offset for the navigation
bar */
.anchor::before {
content:"";
display:block;
height:50px; /* fixed header height for style bootstrap */
margin:-50px 0 0; /* negative fixed header height */
}
</style>
</head>
<!-- tocinfo
{'highest level': 2,
'sections': [('Overview of week 44', 2, None, 'overview-of-week-44'),
('Digression First', 2, None, 'digression-first'),
('A short Discussion of Project 2',
2,
None,
'a-short-discussion-of-project-2'),
('Learning Rate and more', 2, None, 'learning-rate-and-more'),
('Thursday, Principal Component Analysis',
2,
None,
'thursday-principal-component-analysis'),
("A kind of Bird's view on PCA",
2,
None,
'a-kind-of-bird-s-view-on-pca'),
('Thursday: Clustering and Unsupervised Learning',
2,
None,
'thursday-clustering-and-unsupervised-learning'),
('Basic Idea of the $k$-means Clustering Algorithm',
2,
None,
'basic-idea-of-the-k-means-clustering-algorithm'),
('The $k$-means Algorithm', 2, None, 'the-k-means-algorithm'),
('Basic Math of the $k$-means Algorithm',
2,
None,
'basic-math-of-the-k-means-algorithm'),
('Within Cluster Point Scatter',
2,
None,
'within-cluster-point-scatter'),
('More Details', 2, None, 'more-details'),
('Total Cluster Variance', 2, None, 'total-cluster-variance'),
('The $k$-means Clustering Algorithm',
2,
None,
'the-k-means-clustering-algorithm'),
('Summarizing', 2, None, 'summarizing'),
('Writing our own Code, the Data Set',
2,
None,
'writing-our-own-code-the-data-set'),
('Implementing the $k$-means Algorithm',
2,
None,
'implementing-the-k-means-algorithm'),
('Plotting', 2, None, 'plotting'),
('Continuing', 2, None, 'continuing'),
('Wrapping it up', 2, None, 'wrapping-it-up'),
('Decision trees, overarching aims',
2,
None,
'decision-trees-overarching-aims'),
('Basics of a tree', 2, None, 'basics-of-a-tree'),
('A Sketch of a Tree, Regression problem',
2,
None,
'a-sketch-of-a-tree-regression-problem'),
('A Sketch of a Tree, Classification problem',
2,
None,
'a-sketch-of-a-tree-classification-problem'),
('A typical Decision Tree with its pertinent Jargon, '
'Classification Problem',
2,
None,
'a-typical-decision-tree-with-its-pertinent-jargon-classification-problem'),
('General Features', 2, None, 'general-features'),
('How do we set it up?', 2, None, 'how-do-we-set-it-up'),
('Decision trees and Regression',
2,
None,
'decision-trees-and-regression'),
('Building a tree, regression',
2,
None,
'building-a-tree-regression'),
('A top-down approach, recursive binary splitting',
2,
None,
'a-top-down-approach-recursive-binary-splitting'),
('Making a tree', 2, None, 'making-a-tree'),
('Pruning the tree', 2, None, 'pruning-the-tree'),
('Cost complexity pruning', 2, None, 'cost-complexity-pruning'),
('Schematic Regression Procedure',
2,
None,
'schematic-regression-procedure'),
('A Classification Tree', 2, None, 'a-classification-tree'),
('Growing a classification tree',
2,
None,
'growing-a-classification-tree'),
('Classification tree, how to split nodes',
2,
None,
'classification-tree-how-to-split-nodes'),
('Visualizing the Tree, Classification',
2,
None,
'visualizing-the-tree-classification'),
('Visualizing the Tree, The Moons',
2,
None,
'visualizing-the-tree-the-moons'),
('Other ways of visualizing the trees',
2,
None,
'other-ways-of-visualizing-the-trees'),
('Printing out as text', 2, None, 'printing-out-as-text'),
('Algorithms for Setting up Decision Trees',
2,
None,
'algorithms-for-setting-up-decision-trees'),
('The CART algorithm for Classification',
2,
None,
'the-cart-algorithm-for-classification'),
('The CART algorithm for Regression',
2,
None,
'the-cart-algorithm-for-regression'),
('Computing the Gini index', 2, None, 'computing-the-gini-index'),
('Simple Python Code to read in Data and perform Classification',
2,
None,
'simple-python-code-to-read-in-data-and-perform-classification'),
('Computing the Gini Factor',
2,
None,
'computing-the-gini-factor'),
('Entropy and the ID3 algorithm',
2,
None,
'entropy-and-the-id3-algorithm'),
('Cancer Data again now with Decision Trees and other Methods',
2,
None,
'cancer-data-again-now-with-decision-trees-and-other-methods'),
('Another example, the moons again',
2,
None,
'another-example-the-moons-again'),
('Playing around with regions',
2,
None,
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
('Pros and cons of trees, pros',
2,
None,
'pros-and-cons-of-trees-pros'),
('Disadvantages', 2, None, 'disadvantages'),
('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
'Boosting, Meet the Jungle of Methods',
2,
None,
'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
('An Overview of Ensemble Methods',
2,
None,
'an-overview-of-ensemble-methods'),
('Bagging', 2, None, 'bagging'),
('More bagging', 2, None, 'more-bagging'),
('Simple Voting Example, head or tail',
2,
None,
'simple-voting-example-head-or-tail'),
('Using the Voting Classifier',
2,
None,
'using-the-voting-classifier'),
('Please, not the moons again! Voting and Bagging',
2,
None,
'please-not-the-moons-again-voting-and-bagging'),
('Bagging Examples', 2, None, 'bagging-examples'),
('Making your own Bootstrap: Changing the Level of the Decision '
'Tree',
2,
None,
'making-your-own-bootstrap-changing-the-level-of-the-decision-tree')]}
end of tocinfo -->
<body>
<script type="text/x-mathjax-config">
MathJax.Hub.Config({
TeX: {
equationNumbers: { autoNumber: "none" },
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
}
});
</script>
<script type="text/javascript" async
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- Bootstrap navigation bar -->
<div class="navbar navbar-default navbar-fixed-top">
<div class="navbar-header">
<button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-responsive-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a class="navbar-brand" href="week44-bs.html">Week 44: Dimensionality Reduction, PCA and Clustering. Decision Trees</a>
</div>
<div class="navbar-collapse collapse navbar-responsive-collapse">
<ul class="nav navbar-nav navbar-right">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Contents <b class="caret"></b></a>
<ul class="dropdown-menu">
<!-- navigation toc: --> <li><a href="._week44-bs001.html#overview-of-week-44" style="font-size: 80%;">Overview of week 44</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs002.html#digression-first" style="font-size: 80%;">Digression First</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs003.html#a-short-discussion-of-project-2" style="font-size: 80%;">A short Discussion of Project 2</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs004.html#learning-rate-and-more" style="font-size: 80%;">Learning Rate and more</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs005.html#thursday-principal-component-analysis" style="font-size: 80%;">Thursday, Principal Component Analysis</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs006.html#a-kind-of-bird-s-view-on-pca" style="font-size: 80%;">A kind of Bird's view on PCA</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs007.html#thursday-clustering-and-unsupervised-learning" style="font-size: 80%;">Thursday: Clustering and Unsupervised Learning</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs008.html#basic-idea-of-the-k-means-clustering-algorithm" style="font-size: 80%;">Basic Idea of the \( k \)-means Clustering Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs009.html#the-k-means-algorithm" style="font-size: 80%;">The \( k \)-means Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs010.html#basic-math-of-the-k-means-algorithm" style="font-size: 80%;">Basic Math of the \( k \)-means Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs011.html#within-cluster-point-scatter" style="font-size: 80%;">Within Cluster Point Scatter</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs012.html#more-details" style="font-size: 80%;">More Details</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs013.html#total-cluster-variance" style="font-size: 80%;">Total Cluster Variance</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs014.html#the-k-means-clustering-algorithm" style="font-size: 80%;">The \( k \)-means Clustering Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs015.html#summarizing" style="font-size: 80%;">Summarizing</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs016.html#writing-our-own-code-the-data-set" style="font-size: 80%;">Writing our own Code, the Data Set</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs017.html#implementing-the-k-means-algorithm" style="font-size: 80%;">Implementing the \( k \)-means Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs018.html#plotting" style="font-size: 80%;">Plotting</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs019.html#continuing" style="font-size: 80%;">Continuing</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs020.html#wrapping-it-up" style="font-size: 80%;">Wrapping it up</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs021.html#decision-trees-overarching-aims" style="font-size: 80%;">Decision trees, overarching aims</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs022.html#basics-of-a-tree" style="font-size: 80%;">Basics of a tree</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs023.html#a-sketch-of-a-tree-regression-problem" style="font-size: 80%;">A Sketch of a Tree, Regression problem</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs024.html#a-sketch-of-a-tree-classification-problem" style="font-size: 80%;">A Sketch of a Tree, Classification problem</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs025.html#a-typical-decision-tree-with-its-pertinent-jargon-classification-problem" style="font-size: 80%;">A typical Decision Tree with its pertinent Jargon, Classification Problem</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs026.html#general-features" style="font-size: 80%;">General Features</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs027.html#how-do-we-set-it-up" style="font-size: 80%;">How do we set it up?</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs028.html#decision-trees-and-regression" style="font-size: 80%;">Decision trees and Regression</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs029.html#building-a-tree-regression" style="font-size: 80%;">Building a tree, regression</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs030.html#a-top-down-approach-recursive-binary-splitting" style="font-size: 80%;">A top-down approach, recursive binary splitting</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs031.html#making-a-tree" style="font-size: 80%;">Making a tree</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs032.html#pruning-the-tree" style="font-size: 80%;">Pruning the tree</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs033.html#cost-complexity-pruning" style="font-size: 80%;">Cost complexity pruning</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs034.html#schematic-regression-procedure" style="font-size: 80%;">Schematic Regression Procedure</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs035.html#a-classification-tree" style="font-size: 80%;">A Classification Tree</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs036.html#growing-a-classification-tree" style="font-size: 80%;">Growing a classification tree</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs037.html#classification-tree-how-to-split-nodes" style="font-size: 80%;">Classification tree, how to split nodes</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs038.html#visualizing-the-tree-classification" style="font-size: 80%;">Visualizing the Tree, Classification</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs039.html#visualizing-the-tree-the-moons" style="font-size: 80%;">Visualizing the Tree, The Moons</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs040.html#other-ways-of-visualizing-the-trees" style="font-size: 80%;">Other ways of visualizing the trees</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs041.html#printing-out-as-text" style="font-size: 80%;">Printing out as text</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs042.html#algorithms-for-setting-up-decision-trees" style="font-size: 80%;">Algorithms for Setting up Decision Trees</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs043.html#the-cart-algorithm-for-classification" style="font-size: 80%;">The CART algorithm for Classification</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs044.html#the-cart-algorithm-for-regression" style="font-size: 80%;">The CART algorithm for Regression</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs045.html#computing-the-gini-index" style="font-size: 80%;">Computing the Gini index</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs046.html#simple-python-code-to-read-in-data-and-perform-classification" style="font-size: 80%;">Simple Python Code to read in Data and perform Classification</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs047.html#computing-the-gini-factor" style="font-size: 80%;">Computing the Gini Factor</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs048.html#entropy-and-the-id3-algorithm" style="font-size: 80%;">Entropy and the ID3 algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs049.html#cancer-data-again-now-with-decision-trees-and-other-methods" style="font-size: 80%;">Cancer Data again now with Decision Trees and other Methods</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs050.html#another-example-the-moons-again" style="font-size: 80%;">Another example, the moons again</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs051.html#playing-around-with-regions" style="font-size: 80%;">Playing around with regions</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs052.html#regression-trees" style="font-size: 80%;">Regression trees</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs053.html#final-regressor-code" style="font-size: 80%;">Final regressor code</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs054.html#pros-and-cons-of-trees-pros" style="font-size: 80%;">Pros and cons of trees, pros</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs055.html#disadvantages" style="font-size: 80%;">Disadvantages</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs056.html#ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods" style="font-size: 80%;">Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs057.html#an-overview-of-ensemble-methods" style="font-size: 80%;">An Overview of Ensemble Methods</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs058.html#bagging" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="#more-bagging" style="font-size: 80%;">More bagging</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs060.html#simple-voting-example-head-or-tail" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs061.html#using-the-voting-classifier" style="font-size: 80%;">Using the Voting Classifier</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs062.html#please-not-the-moons-again-voting-and-bagging" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs063.html#bagging-examples" style="font-size: 80%;">Bagging Examples</a></li>
<!-- navigation toc: --> <li><a href="._week44-bs064.html#making-your-own-bootstrap-changing-the-level-of-the-decision-tree" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div> <!-- end of navigation bar -->
<div class="container">
<p>&nbsp;</p><p>&nbsp;</p><p>&nbsp;</p> <!-- add vertical space -->
<a name="part0059"></a>
<!-- !split -->
<h2 id="more-bagging" class="anchor">More bagging </h2>
<p>Bagging typically results in improved accuracy
over prediction using a single tree. Unfortunately, however, it can be
difficult to interpret the resulting model. Recall that one of the
advantages of decision trees is the attractive and easily interpreted
diagram that results.
</p>
<p>However, when we bag a large number of trees, it is no longer
possible to represent the resulting statistical learning procedure
using a single tree, and it is no longer clear which variables are
most important to the procedure. Thus, bagging improves prediction
accuracy at the expense of interpretability. Although the collection
of bagged trees is much more difficult to interpret than a single
tree, one can obtain an overall summary of the importance of each
predictor using the MSE (for bagging regression trees) or the Gini
index (for bagging classification trees). In the case of bagging
regression trees, we can record the total amount that the MSE is
decreased due to splits over a given predictor, averaged over all \( B \) possible
trees. A large value indicates an important predictor. Similarly, in
the context of bagging classification trees, we can add up the total
amount that the Gini index is decreased by splits over a given
predictor, averaged over all \( B \) trees.
</p>
<p>
<!-- navigation buttons at the bottom of the page -->
<ul class="pagination">
<li><a href="._week44-bs058.html">&laquo;</a></li>
<li><a href="._week44-bs000.html">1</a></li>
<li><a href="">...</a></li>
<li><a href="._week44-bs051.html">52</a></li>
<li><a href="._week44-bs052.html">53</a></li>
<li><a href="._week44-bs053.html">54</a></li>
<li><a href="._week44-bs054.html">55</a></li>
<li><a href="._week44-bs055.html">56</a></li>
<li><a href="._week44-bs056.html">57</a></li>
<li><a href="._week44-bs057.html">58</a></li>
<li><a href="._week44-bs058.html">59</a></li>
<li class="active"><a href="._week44-bs059.html">60</a></li>
<li><a href="._week44-bs060.html">61</a></li>
<li><a href="._week44-bs061.html">62</a></li>
<li><a href="._week44-bs062.html">63</a></li>
<li><a href="._week44-bs063.html">64</a></li>
<li><a href="._week44-bs064.html">65</a></li>
<li><a href="._week44-bs060.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->
</div> <!-- end container -->
<!-- include javascript, jQuery *first* -->
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script src="https://netdna.bootstrapcdn.com/bootstrap/3.0.0/js/bootstrap.min.js"></script>
<!-- Bootstrap footer
<footer>
<a href="https://..."><img width="250" align=right src="https://..."></a>
</footer>
-->
<center style="font-size:80%">
<!-- copyright only on the titlepage -->
</center>
</body>
</html>